diff --git a/TODO.md b/TODO.md index ab1af3c8..c853adea 100644 --- a/TODO.md +++ b/TODO.md @@ -15,6 +15,18 @@ and green (674 tests, three consecutive clean runs). heavier to run) and **Cap** (newer, smaller). hCaptcha is neither Google nor open source, so it trades one third party for another. - [ ] **Kubernetes / CI-CD hardening.** Details under *Deployment readiness*. +- [ ] **Finish pointing audio backups at MinIO — one blocked step.** The + `audio-backups` bucket now exists on the same MinIO the app already uses + (`assets:9000`). The app's key is scoped by a MinIO *user* policy, so a + bucket policy alone grants it nothing — verified, it returns AccessDenied. + The remaining step attaches a second policy covering only that bucket, + leaving the generated-images grant untouched. It needs the MinIO root + credentials, and my permission layer blocked me from running it. Run: + `RU=$(cat ../ped-ai-storage/secrets/minio-root-user) RP=$(cat ../ped-ai-storage/secrets/minio-root-password)` + then `docker compose exec -T -e MINIO_ROOT_USER="$RU" -e MINIO_ROOT_PASSWORD="$RP" pediatric-scribe node scripts/enable-audio-backup-bucket.js`, + set `AUDIO_BACKUPS_S3_*` (the script prints them), and restart. Until then + recordings go to the encrypted Postgres column, which works; rows already + there keep working afterwards. - [ ] **Basic index has no reader.** `MilvusVectorStore.search()` exists, but no tool calls it. Decide where the query path lives: pymilvus inside the deliberately-lean `nextcloud-basic-mcp` image, or a query API from the @@ -31,10 +43,16 @@ and green (674 tests, three consecutive clean runs). per request. Needs an admin setting for which collections to search, then fan-out and merge — `dedupeSources` in `src/utils/clinicalRetrieval.js` already merges and renumbers. See `clinical-assist/COLLECTIONS.md`. -- [ ] **Confirm mail indexing end to end.** Six separate breaks are fixed and the - scan reaches mail, but no full file pass has completed since the Milvus - rebuild, so no mail is indexed yet. Watch for - `[SCAN-*] Mail messages: N seen, M queued`. +- [ ] **Confirm mail indexing end to end.** Six separate breaks are fixed, and + mail is enabled (`BASIC_INDEXING_MAIL_ENABLED=true`), but no mail has been + indexed yet: the file pass was still working through 472 large PDFs (68 + done in six hours, some over 1,500 chunks each) and mail runs after it. + On 2026-09-11 the indexed folder was changed from `Documents` to + `Personal assistant`, which currently lists 0 files, so the file pass is + now trivial and mail should be reached on a scan cycle (every 300s). + Watch for `[SCAN-*] Mail messages: N seen, M queued`. + Note: the 68 files already indexed from `Documents` are still in the + collection. Say the word if they should be cleared. - [ ] **`image-size` DoS advisory (high), no upstream fix.** Every published version up to 2.0.2 is affected; `npm audit fix` only offers a breaking downgrade of pptxgenjs. Not reachable here: the only `addImage` call diff --git a/scripts/enable-audio-backup-bucket.js b/scripts/enable-audio-backup-bucket.js new file mode 100644 index 00000000..55269d27 --- /dev/null +++ b/scripts/enable-audio-backup-bucket.js @@ -0,0 +1,133 @@ +#!/usr/bin/env node +// ============================================================ +// Point audio backups at MinIO — same server, its own bucket. +// +// Recordings are all kept for 24 hours now, not only the failures, so object +// storage is a better home for them than a Postgres column. The bucket lives on +// the MinIO the app already talks to; only the permission is missing. +// +// The app's key is scoped by a MinIO *user* policy, so a bucket policy alone +// does not grant it anything (MinIO evaluates the user policy first, and it +// denies by default). This attaches a second, separate policy covering only +// `audio-backups`, leaving the existing generated-images policy exactly as it +// is — additive, and undone by re-attaching that one policy on its own. +// +// Run it from the app container, which can reach MinIO and holds the app key: +// +// RU=$(cat ../ped-ai-storage/secrets/minio-root-user) +// RP=$(cat ../ped-ai-storage/secrets/minio-root-password) +// docker compose exec -T -e MINIO_ROOT_USER="$RU" -e MINIO_ROOT_PASSWORD="$RP" \ +// pediatric-scribe node scripts/enable-audio-backup-bucket.js +// +// Then set AUDIO_BACKUPS_S3_* (see docs/speech.md) and restart the app. +// Nothing already stored is touched: rows without a storage_key are still read +// from the database column. +// ============================================================ + +var fs = require('fs'); +var { SignatureV4 } = require('@smithy/signature-v4'); +var { HttpRequest } = require('@smithy/protocol-http'); +var { Hash } = require('@smithy/hash-node'); +var { S3Client, CreateBucketCommand } = require('@aws-sdk/client-s3'); + +var BUCKET = process.env.AUDIO_BACKUPS_S3_BUCKET || 'audio-backups'; +var POLICY_NAME = 'audio-backups-app'; + +var rootUser = process.env.MINIO_ROOT_USER; +var rootPassword = process.env.MINIO_ROOT_PASSWORD; +var endpoint = process.env.AUDIO_BACKUPS_S3_ENDPOINT || process.env.GENERATED_IMAGES_S3_ENDPOINT; + +if (!rootUser || !rootPassword) { + console.error('MINIO_ROOT_USER and MINIO_ROOT_PASSWORD are required (they are only used by this script, never by the app).'); + process.exit(1); +} +if (!endpoint) { + console.error('No MinIO endpoint: set AUDIO_BACKUPS_S3_ENDPOINT or GENERATED_IMAGES_S3_ENDPOINT.'); + process.exit(1); +} + +// The key the app already uses. It gains the new bucket; it keeps everything +// it had. +function appAccessKey() { + if (process.env.AUDIO_BACKUPS_S3_ACCESS_KEY) return process.env.AUDIO_BACKUPS_S3_ACCESS_KEY; + if (process.env.AUDIO_BACKUPS_S3_ACCESS_KEY_FILE) return fs.readFileSync(process.env.AUDIO_BACKUPS_S3_ACCESS_KEY_FILE, 'utf8').trim(); + if (process.env.GENERATED_IMAGES_S3_ACCESS_KEY_FILE) return fs.readFileSync(process.env.GENERATED_IMAGES_S3_ACCESS_KEY_FILE, 'utf8').trim(); + throw new Error('Cannot find the application access key'); +} + +var url = new URL(endpoint); +var signer = new SignatureV4({ + service: 's3', region: process.env.AUDIO_BACKUPS_S3_REGION || 'us-east-1', + credentials: { accessKeyId: rootUser, secretAccessKey: rootPassword }, + sha256: Hash.bind(null, 'sha256') +}); + +async function admin(method, path, query, body) { + var qs = new URLSearchParams(query || {}).toString(); + var request = new HttpRequest({ + method: method, protocol: url.protocol, hostname: url.hostname, port: Number(url.port), + path: path, query: query || {}, headers: { host: url.host, 'content-type': 'application/json' }, body: body + }); + var signed = await signer.sign(request); + var response = await fetch(url.origin + path + (qs ? '?' + qs : ''), { method: method, headers: signed.headers, body: body }); + var text = await response.text(); + if (response.status >= 300) throw new Error(method + ' ' + path + ' -> ' + response.status + ' ' + text.slice(0, 200)); + return text; +} + +(async function main() { + var key = appAccessKey(); + console.log('MinIO : ' + url.origin); + console.log('Bucket : ' + BUCKET); + console.log('App key : ' + key); + + var s3 = new S3Client({ + endpoint: endpoint, region: process.env.AUDIO_BACKUPS_S3_REGION || 'us-east-1', forcePathStyle: true, + credentials: { accessKeyId: rootUser, secretAccessKey: rootPassword } + }); + try { + await s3.send(new CreateBucketCommand({ Bucket: BUCKET })); + console.log(' created the bucket'); + } catch (e) { + if (e.name === 'BucketAlreadyOwnedByYou' || e.name === 'BucketAlreadyExists') console.log(' bucket already exists'); + else throw e; + } + + // Only this bucket. A compromise of the key reaches no further than before. + var policy = JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { Effect: 'Allow', Action: ['s3:ListBucket', 's3:GetBucketLocation'], Resource: ['arn:aws:s3:::' + BUCKET] }, + { Effect: 'Allow', Action: ['s3:GetObject', 's3:PutObject', 's3:DeleteObject'], Resource: ['arn:aws:s3:::' + BUCKET + '/*'] } + ] + }); + await admin('PUT', '/minio/admin/v3/add-canned-policy', { name: POLICY_NAME }, policy); + console.log(' wrote the ' + POLICY_NAME + ' policy'); + + // Read what the user has now, so the existing grant is carried over rather + // than replaced. Attaching only the new policy would break image storage. + var current = ''; + try { current = JSON.parse(await admin('GET', '/minio/admin/v3/user-info', { accessKey: key })).policyName || ''; } catch (e) {} + var names = current.split(',').map(function (n) { return n.trim(); }).filter(Boolean); + if (names.indexOf(POLICY_NAME) === -1) names.push(POLICY_NAME); + console.log(' policies before: ' + (current || '(none)')); + + await admin('PUT', '/minio/admin/v3/set-user-or-group-policy', { + userOrGroup: key, isGroup: 'false', policyName: names.join(',') + }); + var after = JSON.parse(await admin('GET', '/minio/admin/v3/user-info', { accessKey: key })).policyName || ''; + console.log(' policies after : ' + after); + + console.log('\nDone. Now set these on the app and restart it:'); + console.log(' AUDIO_BACKUPS_S3_ENDPOINT=' + url.origin); + console.log(' AUDIO_BACKUPS_S3_BUCKET=' + BUCKET); + console.log(' AUDIO_BACKUPS_S3_REGION=' + (process.env.AUDIO_BACKUPS_S3_REGION || 'us-east-1')); + console.log(' AUDIO_BACKUPS_S3_ACCESS_KEY_FILE=' + (process.env.GENERATED_IMAGES_S3_ACCESS_KEY_FILE || '/run/secrets/generated-images-access-key')); + console.log(' AUDIO_BACKUPS_S3_SECRET_KEY_FILE=' + (process.env.GENERATED_IMAGES_S3_SECRET_KEY_FILE || '/run/secrets/generated-images-secret-key')); + console.log('\nAlso give the bucket a 24-hour expiry lifecycle rule, as a backstop for'); + console.log('objects the expiry sweep could not delete.'); + console.log('To undo: re-attach only "' + (current || 'generated-images-app') + '" to ' + key + '.'); +})().catch(function (err) { + console.error('Failed: ' + err.message); + process.exit(1); +});