From 713ed830a38da371cd88390a103ad3045afa4836 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Sep 2026 17:00:02 +0200 Subject: [PATCH] chore: script to move audio backups onto MinIO; record where things stand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same MinIO, its own bucket, as asked. The audio-backups bucket is created; what remains is a MinIO IAM change, which needs root credentials — my permission layer blocked that call, so it is a script to run rather than something I applied. The script is additive and reversible: it attaches a second policy covering only the new bucket and carries the existing generated-images grant over rather than replacing it (attaching only the new one would break image storage). It prints the AUDIO_BACKUPS_S3_* values to set, and how to undo. A bucket policy alone does not work here: MinIO evaluates the user policy first and it denies by default. Verified — the app key gets AccessDenied on the new bucket until its own policy allows it. TODO records that, and the indexer being repointed from Documents to Personal assistant so mail is finally reached. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- TODO.md | 26 ++++- scripts/enable-audio-backup-bucket.js | 133 ++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 scripts/enable-audio-backup-bucket.js diff --git a/TODO.md b/TODO.md index ab1af3c8..c853adea 100644 --- a/TODO.md +++ b/TODO.md @@ -15,6 +15,18 @@ and green (674 tests, three consecutive clean runs). heavier to run) and **Cap** (newer, smaller). hCaptcha is neither Google nor open source, so it trades one third party for another. - [ ] **Kubernetes / CI-CD hardening.** Details under *Deployment readiness*. +- [ ] **Finish pointing audio backups at MinIO — one blocked step.** The + `audio-backups` bucket now exists on the same MinIO the app already uses + (`assets:9000`). The app's key is scoped by a MinIO *user* policy, so a + bucket policy alone grants it nothing — verified, it returns AccessDenied. + The remaining step attaches a second policy covering only that bucket, + leaving the generated-images grant untouched. It needs the MinIO root + credentials, and my permission layer blocked me from running it. Run: + `RU=$(cat ../ped-ai-storage/secrets/minio-root-user) RP=$(cat ../ped-ai-storage/secrets/minio-root-password)` + then `docker compose exec -T -e MINIO_ROOT_USER="$RU" -e MINIO_ROOT_PASSWORD="$RP" pediatric-scribe node scripts/enable-audio-backup-bucket.js`, + set `AUDIO_BACKUPS_S3_*` (the script prints them), and restart. Until then + recordings go to the encrypted Postgres column, which works; rows already + there keep working afterwards. - [ ] **Basic index has no reader.** `MilvusVectorStore.search()` exists, but no tool calls it. Decide where the query path lives: pymilvus inside the deliberately-lean `nextcloud-basic-mcp` image, or a query API from the @@ -31,10 +43,16 @@ and green (674 tests, three consecutive clean runs). per request. Needs an admin setting for which collections to search, then fan-out and merge — `dedupeSources` in `src/utils/clinicalRetrieval.js` already merges and renumbers. See `clinical-assist/COLLECTIONS.md`. -- [ ] **Confirm mail indexing end to end.** Six separate breaks are fixed and the - scan reaches mail, but no full file pass has completed since the Milvus - rebuild, so no mail is indexed yet. Watch for - `[SCAN-*] Mail messages: N seen, M queued`. +- [ ] **Confirm mail indexing end to end.** Six separate breaks are fixed, and + mail is enabled (`BASIC_INDEXING_MAIL_ENABLED=true`), but no mail has been + indexed yet: the file pass was still working through 472 large PDFs (68 + done in six hours, some over 1,500 chunks each) and mail runs after it. + On 2026-09-11 the indexed folder was changed from `Documents` to + `Personal assistant`, which currently lists 0 files, so the file pass is + now trivial and mail should be reached on a scan cycle (every 300s). + Watch for `[SCAN-*] Mail messages: N seen, M queued`. + Note: the 68 files already indexed from `Documents` are still in the + collection. Say the word if they should be cleared. - [ ] **`image-size` DoS advisory (high), no upstream fix.** Every published version up to 2.0.2 is affected; `npm audit fix` only offers a breaking downgrade of pptxgenjs. Not reachable here: the only `addImage` call diff --git a/scripts/enable-audio-backup-bucket.js b/scripts/enable-audio-backup-bucket.js new file mode 100644 index 00000000..55269d27 --- /dev/null +++ b/scripts/enable-audio-backup-bucket.js @@ -0,0 +1,133 @@ +#!/usr/bin/env node +// ============================================================ +// Point audio backups at MinIO — same server, its own bucket. +// +// Recordings are all kept for 24 hours now, not only the failures, so object +// storage is a better home for them than a Postgres column. The bucket lives on +// the MinIO the app already talks to; only the permission is missing. +// +// The app's key is scoped by a MinIO *user* policy, so a bucket policy alone +// does not grant it anything (MinIO evaluates the user policy first, and it +// denies by default). This attaches a second, separate policy covering only +// `audio-backups`, leaving the existing generated-images policy exactly as it +// is — additive, and undone by re-attaching that one policy on its own. +// +// Run it from the app container, which can reach MinIO and holds the app key: +// +// RU=$(cat ../ped-ai-storage/secrets/minio-root-user) +// RP=$(cat ../ped-ai-storage/secrets/minio-root-password) +// docker compose exec -T -e MINIO_ROOT_USER="$RU" -e MINIO_ROOT_PASSWORD="$RP" \ +// pediatric-scribe node scripts/enable-audio-backup-bucket.js +// +// Then set AUDIO_BACKUPS_S3_* (see docs/speech.md) and restart the app. +// Nothing already stored is touched: rows without a storage_key are still read +// from the database column. +// ============================================================ + +var fs = require('fs'); +var { SignatureV4 } = require('@smithy/signature-v4'); +var { HttpRequest } = require('@smithy/protocol-http'); +var { Hash } = require('@smithy/hash-node'); +var { S3Client, CreateBucketCommand } = require('@aws-sdk/client-s3'); + +var BUCKET = process.env.AUDIO_BACKUPS_S3_BUCKET || 'audio-backups'; +var POLICY_NAME = 'audio-backups-app'; + +var rootUser = process.env.MINIO_ROOT_USER; +var rootPassword = process.env.MINIO_ROOT_PASSWORD; +var endpoint = process.env.AUDIO_BACKUPS_S3_ENDPOINT || process.env.GENERATED_IMAGES_S3_ENDPOINT; + +if (!rootUser || !rootPassword) { + console.error('MINIO_ROOT_USER and MINIO_ROOT_PASSWORD are required (they are only used by this script, never by the app).'); + process.exit(1); +} +if (!endpoint) { + console.error('No MinIO endpoint: set AUDIO_BACKUPS_S3_ENDPOINT or GENERATED_IMAGES_S3_ENDPOINT.'); + process.exit(1); +} + +// The key the app already uses. It gains the new bucket; it keeps everything +// it had. +function appAccessKey() { + if (process.env.AUDIO_BACKUPS_S3_ACCESS_KEY) return process.env.AUDIO_BACKUPS_S3_ACCESS_KEY; + if (process.env.AUDIO_BACKUPS_S3_ACCESS_KEY_FILE) return fs.readFileSync(process.env.AUDIO_BACKUPS_S3_ACCESS_KEY_FILE, 'utf8').trim(); + if (process.env.GENERATED_IMAGES_S3_ACCESS_KEY_FILE) return fs.readFileSync(process.env.GENERATED_IMAGES_S3_ACCESS_KEY_FILE, 'utf8').trim(); + throw new Error('Cannot find the application access key'); +} + +var url = new URL(endpoint); +var signer = new SignatureV4({ + service: 's3', region: process.env.AUDIO_BACKUPS_S3_REGION || 'us-east-1', + credentials: { accessKeyId: rootUser, secretAccessKey: rootPassword }, + sha256: Hash.bind(null, 'sha256') +}); + +async function admin(method, path, query, body) { + var qs = new URLSearchParams(query || {}).toString(); + var request = new HttpRequest({ + method: method, protocol: url.protocol, hostname: url.hostname, port: Number(url.port), + path: path, query: query || {}, headers: { host: url.host, 'content-type': 'application/json' }, body: body + }); + var signed = await signer.sign(request); + var response = await fetch(url.origin + path + (qs ? '?' + qs : ''), { method: method, headers: signed.headers, body: body }); + var text = await response.text(); + if (response.status >= 300) throw new Error(method + ' ' + path + ' -> ' + response.status + ' ' + text.slice(0, 200)); + return text; +} + +(async function main() { + var key = appAccessKey(); + console.log('MinIO : ' + url.origin); + console.log('Bucket : ' + BUCKET); + console.log('App key : ' + key); + + var s3 = new S3Client({ + endpoint: endpoint, region: process.env.AUDIO_BACKUPS_S3_REGION || 'us-east-1', forcePathStyle: true, + credentials: { accessKeyId: rootUser, secretAccessKey: rootPassword } + }); + try { + await s3.send(new CreateBucketCommand({ Bucket: BUCKET })); + console.log(' created the bucket'); + } catch (e) { + if (e.name === 'BucketAlreadyOwnedByYou' || e.name === 'BucketAlreadyExists') console.log(' bucket already exists'); + else throw e; + } + + // Only this bucket. A compromise of the key reaches no further than before. + var policy = JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { Effect: 'Allow', Action: ['s3:ListBucket', 's3:GetBucketLocation'], Resource: ['arn:aws:s3:::' + BUCKET] }, + { Effect: 'Allow', Action: ['s3:GetObject', 's3:PutObject', 's3:DeleteObject'], Resource: ['arn:aws:s3:::' + BUCKET + '/*'] } + ] + }); + await admin('PUT', '/minio/admin/v3/add-canned-policy', { name: POLICY_NAME }, policy); + console.log(' wrote the ' + POLICY_NAME + ' policy'); + + // Read what the user has now, so the existing grant is carried over rather + // than replaced. Attaching only the new policy would break image storage. + var current = ''; + try { current = JSON.parse(await admin('GET', '/minio/admin/v3/user-info', { accessKey: key })).policyName || ''; } catch (e) {} + var names = current.split(',').map(function (n) { return n.trim(); }).filter(Boolean); + if (names.indexOf(POLICY_NAME) === -1) names.push(POLICY_NAME); + console.log(' policies before: ' + (current || '(none)')); + + await admin('PUT', '/minio/admin/v3/set-user-or-group-policy', { + userOrGroup: key, isGroup: 'false', policyName: names.join(',') + }); + var after = JSON.parse(await admin('GET', '/minio/admin/v3/user-info', { accessKey: key })).policyName || ''; + console.log(' policies after : ' + after); + + console.log('\nDone. Now set these on the app and restart it:'); + console.log(' AUDIO_BACKUPS_S3_ENDPOINT=' + url.origin); + console.log(' AUDIO_BACKUPS_S3_BUCKET=' + BUCKET); + console.log(' AUDIO_BACKUPS_S3_REGION=' + (process.env.AUDIO_BACKUPS_S3_REGION || 'us-east-1')); + console.log(' AUDIO_BACKUPS_S3_ACCESS_KEY_FILE=' + (process.env.GENERATED_IMAGES_S3_ACCESS_KEY_FILE || '/run/secrets/generated-images-access-key')); + console.log(' AUDIO_BACKUPS_S3_SECRET_KEY_FILE=' + (process.env.GENERATED_IMAGES_S3_SECRET_KEY_FILE || '/run/secrets/generated-images-secret-key')); + console.log('\nAlso give the bucket a 24-hour expiry lifecycle rule, as a backstop for'); + console.log('objects the expiry sweep could not delete.'); + console.log('To undo: re-attach only "' + (current || 'generated-images-app') + '" to ' + key + '.'); +})().catch(function (err) { + console.error('Failed: ' + err.message); + process.exit(1); +});