From d74f51633626cb90ed7bc691510f6d6db19bb278 Mon Sep 17 00:00:00 2001 From: Nico <47644445+nicotsx@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:31:42 +0100 Subject: [PATCH] feat: file-based provisionning of volumes and repos (#656) * feat: file-based provisionning of volumes and repos docs: provisioning example chore: ui improvements * chore: ci issues --- README.md | 15 +- app/client/api-client/types.gen.ts | 15 + app/client/components/managed-badge.tsx | 23 + .../repositories/routes/edit-repository.tsx | 6 +- .../repositories/routes/repositories.tsx | 6 +- app/client/modules/repositories/tabs/info.tsx | 10 +- .../modules/settings/routes/settings.tsx | 23 +- app/client/modules/volumes/routes/volumes.tsx | 6 +- app/client/modules/volumes/tabs/info.tsx | 6 +- .../20260312141708_loose_gideon/migration.sql | 4 + .../20260312141708_loose_gideon/snapshot.json | 2271 +++++++++++++++++ app/server/core/config.ts | 2 + app/server/db/schema.ts | 71 +- app/server/modules/lifecycle/startup.ts | 7 + .../modules/provisioning/provisioning.test.ts | 391 +++ .../modules/provisioning/provisioning.ts | 262 ++ .../__tests__/repositories.controller.test.ts | 90 + .../modules/repositories/repositories.dto.ts | 1 + .../repositories/repositories.service.ts | 89 +- .../repositories/repository-config-secrets.ts | 79 + .../__tests__/volumes.controller.test.ts | 67 + .../modules/volumes/volume-config-secrets.ts | 47 + app/server/modules/volumes/volume.dto.ts | 1 + app/server/modules/volumes/volume.service.ts | 35 +- app/server/utils/crypto.ts | 104 +- bun.lock | 1 + examples/README.md | 2 +- examples/provisioned-resources/.env.example | 5 + examples/provisioned-resources/.gitignore | 3 + examples/provisioned-resources/README.md | 89 + .../provisioned-resources/docker-compose.yml | 28 + .../provisioning.example.json | 36 + .../secrets/aws_secret_access_key.example | 1 + examples/secrets-placeholders/.env.example | 6 - examples/secrets-placeholders/.gitignore | 5 - examples/secrets-placeholders/README.md | 61 - .../secrets-placeholders/docker-compose.yml | 26 - 37 files changed, 3546 insertions(+), 348 deletions(-) create mode 100644 app/client/components/managed-badge.tsx create mode 100644 app/drizzle/20260312141708_loose_gideon/migration.sql create mode 100644 app/drizzle/20260312141708_loose_gideon/snapshot.json create mode 100644 app/server/modules/provisioning/provisioning.test.ts create mode 100644 app/server/modules/provisioning/provisioning.ts create mode 100644 app/server/modules/repositories/repository-config-secrets.ts create mode 100644 app/server/modules/volumes/volume-config-secrets.ts create mode 100644 examples/provisioned-resources/.env.example create mode 100644 examples/provisioned-resources/.gitignore create mode 100644 examples/provisioned-resources/README.md create mode 100644 examples/provisioned-resources/docker-compose.yml create mode 100644 examples/provisioned-resources/provisioning.example.json create mode 100644 examples/provisioned-resources/secrets/aws_secret_access_key.example delete mode 100644 examples/secrets-placeholders/.env.example delete mode 100644 examples/secrets-placeholders/.gitignore delete mode 100644 examples/secrets-placeholders/README.md delete mode 100644 examples/secrets-placeholders/docker-compose.yml diff --git a/README.md b/README.md index 5f652a48..e9cdfaa3 100644 --- a/README.md +++ b/README.md @@ -100,16 +100,19 @@ Zerobyte can be customized using environment variables. Below are the available | `LOG_LEVEL` | Logging verbosity. Options: `debug`, `info`, `warn`, `error`. | `info` | | `SERVER_IDLE_TIMEOUT` | Idle timeout for the server in seconds. | `60` | | `RCLONE_CONFIG_DIR` | Path to the rclone config directory inside the container. Change this if running as a non-root user. | `/root/.config/rclone` | +| `PROVISIONING_PATH` | Path to a JSON file with operator-managed repositories and volumes to sync at startup. | (none) | -### Secret References +### Provisioned Resources -For enhanced security, Zerobyte supports dynamic secret resolution for sensitive fields (like passwords, access keys, etc.) in volume and repository configurations. Instead of storing the encrypted secret in the database, you can use one of the following prefixes: +Zerobyte can sync operator-managed repositories and volumes from a JSON file at startup. This is useful when you want credentials or connection details to live in deployment-time configuration instead of being entered through the UI. -- `env://VAR_NAME`: Reads the secret from the environment variable `VAR_NAME`. -- `file://SECRET_NAME`: Reads the secret from `/run/secrets/SECRET_NAME` (standard Docker Secrets path). +Provisioned resources: -**Example:** -When configuring an S3 repository, you can set the Secret Access Key to `env://S3_SECRET_KEY` and then provide that variable in your `docker-compose.yml`. +- appear in the normal repositories and volumes screens +- stay read-only in the UI +- can resolve credential fields from environment variables or `/run/secrets/*` during startup sync + +See `examples/provisioned-resources/README.md` for a full example. ### Simplified setup (No remote mounts) diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index c16fdd64..216a4740 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -337,6 +337,7 @@ export type ListVolumesResponses = { 200: Array<{ id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -462,6 +463,7 @@ export type CreateVolumeResponses = { 201: { id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -635,6 +637,7 @@ export type GetVolumeResponses = { volume: { id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -776,6 +779,7 @@ export type UpdateVolumeResponses = { 200: { id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -986,6 +990,7 @@ export type ListRepositoriesResponses = { 200: Array<{ id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -1406,6 +1411,7 @@ export type GetRepositoryResponses = { 200: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -1779,6 +1785,7 @@ export type UpdateRepositoryResponses = { 200: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -2435,6 +2442,7 @@ export type ListBackupSchedulesResponses = { volume: { id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -2495,6 +2503,7 @@ export type ListBackupSchedulesResponses = { repository: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -2806,6 +2815,7 @@ export type GetBackupScheduleResponses = { volume: { id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -2866,6 +2876,7 @@ export type GetBackupScheduleResponses = { repository: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -3158,6 +3169,7 @@ export type GetBackupScheduleForVolumeResponses = { volume: { id: number; shortId: string; + provisioningId: string | null; name: string; type: 'nfs' | 'smb' | 'directory' | 'webdav' | 'rclone' | 'sftp'; status: 'mounted' | 'unmounted' | 'error'; @@ -3218,6 +3230,7 @@ export type GetBackupScheduleForVolumeResponses = { repository: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -3687,6 +3700,7 @@ export type GetScheduleMirrorsResponses = { repository: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { @@ -3898,6 +3912,7 @@ export type UpdateScheduleMirrorsResponses = { repository: { id: string; shortId: string; + provisioningId: string | null; name: string; type: 'local' | 's3' | 'r2' | 'gcs' | 'azure' | 'rclone' | 'rest' | 'sftp'; config: { diff --git a/app/client/components/managed-badge.tsx b/app/client/components/managed-badge.tsx new file mode 100644 index 00000000..c58d178c --- /dev/null +++ b/app/client/components/managed-badge.tsx @@ -0,0 +1,23 @@ +import { Badge } from "~/client/components/ui/badge"; +import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip"; + +type ManagedBadgeProps = { + label?: string; + message?: string; +}; + +const defaultMessage = + "This resource is provisioned at startup. Changes are useful for testing, but the next provisioning sync can overwrite or recreate it."; + +export const ManagedBadge = ({ label = "Managed", message = defaultMessage }: ManagedBadgeProps) => { + return ( + + + {label} + + +

{message}

+ + + ); +}; diff --git a/app/client/modules/repositories/routes/edit-repository.tsx b/app/client/modules/repositories/routes/edit-repository.tsx index d45022b1..5463b31a 100644 --- a/app/client/modules/repositories/routes/edit-repository.tsx +++ b/app/client/modules/repositories/routes/edit-repository.tsx @@ -24,6 +24,7 @@ import { parseError } from "~/client/lib/errors"; import { Alert, AlertDescription } from "~/client/components/ui/alert"; import type { RepositoryConfig } from "@zerobyte/core/restic"; import { useNavigate } from "@tanstack/react-router"; +import { ManagedBadge } from "~/client/components/managed-badge"; const riskyLocationFieldsByBackend = { local: ["path"], @@ -124,7 +125,10 @@ export function EditRepositoryPage({ repositoryId }: { repositoryId: string }) {
- Edit Repository +
+ Edit Repository + {repository.provisioningId && } +
diff --git a/app/client/modules/repositories/routes/repositories.tsx b/app/client/modules/repositories/routes/repositories.tsx index 9e821b14..eeed85c8 100644 --- a/app/client/modules/repositories/routes/repositories.tsx +++ b/app/client/modules/repositories/routes/repositories.tsx @@ -126,7 +126,11 @@ export function RepositoriesPage() { className="hover:bg-accent/50 hover:cursor-pointer h-12" onClick={() => navigate({ to: `/repositories/${repository.shortId}` })} > - {repository.name} + +
+ {repository.name} +
+ diff --git a/app/client/modules/repositories/tabs/info.tsx b/app/client/modules/repositories/tabs/info.tsx index 5ae1b63e..6b73e69b 100644 --- a/app/client/modules/repositories/tabs/info.tsx +++ b/app/client/modules/repositories/tabs/info.tsx @@ -28,6 +28,7 @@ import { parseError } from "~/client/lib/errors"; import { useNavigate } from "@tanstack/react-router"; import { CompressionStatsChart } from "../components/compression-stats-chart"; import { cn } from "~/client/lib/utils"; +import { ManagedBadge } from "~/client/components/managed-badge"; type Props = { repository: Repository; @@ -109,7 +110,10 @@ export const RepositoryInfoTabContent = ({ repository, initialStats }: Props) =>
-

Repository Settings

+
+

Repository Settings

+ {repository.provisioningId && } +
+
+
Management
+

{repository.provisioningId ? "Provisioned" : "Manual"}

+
Compression Mode

{repository.compressionMode || "off"}

diff --git a/app/client/modules/settings/routes/settings.tsx b/app/client/modules/settings/routes/settings.tsx index 5916d7ab..908fb3a7 100644 --- a/app/client/modules/settings/routes/settings.tsx +++ b/app/client/modules/settings/routes/settings.tsx @@ -1,5 +1,5 @@ import { useMutation } from "@tanstack/react-query"; -import { Download, KeyRound, User, X, Settings as SettingsIcon, Building2 } from "lucide-react"; +import { Download, Fingerprint, KeyRound, User, X, Settings as SettingsIcon, Building2 } from "lucide-react"; import { useState } from "react"; import { toast } from "sonner"; import { downloadResticPasswordMutation } from "~/client/api-client/@tanstack/react-query.gen"; @@ -45,7 +45,7 @@ export function SettingsPage({ appContext, initialMembers, initialSsoSettings, i const activeTab = tab || "account"; const navigate = useNavigate(); - const { activeMember } = useOrganizationContext(); + const { activeMember, activeOrganization } = useOrganizationContext(); const isOrgAdmin = activeMember?.role === "owner" || activeMember?.role === "admin"; const handleLogout = async () => { @@ -295,6 +295,25 @@ export function SettingsPage({ appContext, initialMembers, initialSsoSettings, i {isOrgAdmin && ( + +
+ + + Organization Details + + Reference details for the active organization +
+ + + + + +
diff --git a/app/client/modules/volumes/routes/volumes.tsx b/app/client/modules/volumes/routes/volumes.tsx index cce8a554..db2f146b 100644 --- a/app/client/modules/volumes/routes/volumes.tsx +++ b/app/client/modules/volumes/routes/volumes.tsx @@ -137,7 +137,11 @@ export function VolumesPage() { className="hover:bg-muted/50 hover:cursor-pointer transition-colors h-12" onClick={() => navigate({ to: `/volumes/${volume.shortId}` })} > - {volume.name} + +
+ {volume.name} +
+ diff --git a/app/client/modules/volumes/tabs/info.tsx b/app/client/modules/volumes/tabs/info.tsx index 29dbc6e7..b6bebcc1 100644 --- a/app/client/modules/volumes/tabs/info.tsx +++ b/app/client/modules/volumes/tabs/info.tsx @@ -27,6 +27,7 @@ import { import type { UpdateVolumeResponse } from "~/client/api-client/types.gen"; import { useNavigate } from "@tanstack/react-router"; import { parseError } from "~/client/lib/errors"; +import { ManagedBadge } from "~/client/components/managed-badge"; type Props = { volume: Volume; @@ -122,7 +123,10 @@ export const VolumeInfoTabContent = ({ volume, statfs }: Props) => {
- Volume Configuration +
+ Volume Configuration + {volume.provisioningId && } +
{volume.status !== "mounted" ? ( diff --git a/app/drizzle/20260312141708_loose_gideon/migration.sql b/app/drizzle/20260312141708_loose_gideon/migration.sql new file mode 100644 index 00000000..dbe25ed8 --- /dev/null +++ b/app/drizzle/20260312141708_loose_gideon/migration.sql @@ -0,0 +1,4 @@ +ALTER TABLE `repositories_table` ADD `provisioning_id` text;--> statement-breakpoint +ALTER TABLE `volumes_table` ADD `provisioning_id` text;--> statement-breakpoint +CREATE UNIQUE INDEX `repositories_table_org_provisioning_id_uidx` ON `repositories_table` (`organization_id`,`provisioning_id`);--> statement-breakpoint +CREATE UNIQUE INDEX `volumes_table_org_provisioning_id_uidx` ON `volumes_table` (`organization_id`,`provisioning_id`); \ No newline at end of file diff --git a/app/drizzle/20260312141708_loose_gideon/snapshot.json b/app/drizzle/20260312141708_loose_gideon/snapshot.json new file mode 100644 index 00000000..b926b734 --- /dev/null +++ b/app/drizzle/20260312141708_loose_gideon/snapshot.json @@ -0,0 +1,2271 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "7cbce296-a3ec-4b25-8930-32d5796a8ced", + "prevIds": ["c84e81d9-8676-45ec-990f-530f98531c2d"], + "ddl": [ + { + "name": "account", + "entityType": "tables" + }, + { + "name": "app_metadata", + "entityType": "tables" + }, + { + "name": "backup_schedule_mirrors_table", + "entityType": "tables" + }, + { + "name": "backup_schedule_notifications_table", + "entityType": "tables" + }, + { + "name": "backup_schedules_table", + "entityType": "tables" + }, + { + "name": "invitation", + "entityType": "tables" + }, + { + "name": "member", + "entityType": "tables" + }, + { + "name": "notification_destinations_table", + "entityType": "tables" + }, + { + "name": "organization", + "entityType": "tables" + }, + { + "name": "repositories_table", + "entityType": "tables" + }, + { + "name": "sessions_table", + "entityType": "tables" + }, + { + "name": "sso_provider", + "entityType": "tables" + }, + { + "name": "two_factor", + "entityType": "tables" + }, + { + "name": "users_table", + "entityType": "tables" + }, + { + "name": "verification", + "entityType": "tables" + }, + { + "name": "volumes_table", + "entityType": "tables" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "account_id", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "provider_id", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "user_id", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "access_token", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "refresh_token", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id_token", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "access_token_expires_at", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "refresh_token_expires_at", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "scope", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "password", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "key", + "entityType": "columns", + "table": "app_metadata" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "value", + "entityType": "columns", + "table": "app_metadata" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "app_metadata" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "app_metadata" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "schedule_id", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "repository_id", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_copy_at", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_copy_status", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_copy_error", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "backup_schedule_mirrors_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "schedule_id", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "destination_id", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "notify_on_start", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "notify_on_success", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "notify_on_warning", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "notify_on_failure", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "backup_schedule_notifications_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "short_id", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "volume_id", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "repository_id", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cron_expression", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "retention_policy", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'[]'", + "generated": null, + "name": "exclude_patterns", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'[]'", + "generated": null, + "name": "exclude_if_present", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'[]'", + "generated": null, + "name": "include_patterns", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_backup_at", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_backup_status", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_backup_error", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "next_backup_at", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "one_file_system", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'[]'", + "generated": null, + "name": "custom_restic_params", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "sort_order", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "backup_schedules_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "email", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "role", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'pending'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "expires_at", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "inviter_id", + "entityType": "columns", + "table": "invitation" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "member" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "member" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "user_id", + "entityType": "columns", + "table": "member" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'member'", + "generated": null, + "name": "role", + "entityType": "columns", + "table": "member" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "member" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "config", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "notification_destinations_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "organization" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "organization" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "slug", + "entityType": "columns", + "table": "organization" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "logo", + "entityType": "columns", + "table": "organization" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "organization" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "metadata", + "entityType": "columns", + "table": "organization" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "short_id", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "provisioning_id", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "config", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'auto'", + "generated": null, + "name": "compression_mode", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'unknown'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_checked", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_error", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "doctor_result", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "stats", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "stats_updated_at", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "upload_limit_enabled", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "real", + "notNull": true, + "autoincrement": false, + "default": "1", + "generated": null, + "name": "upload_limit_value", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'Mbps'", + "generated": null, + "name": "upload_limit_unit", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "download_limit_enabled", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "real", + "notNull": true, + "autoincrement": false, + "default": "1", + "generated": null, + "name": "download_limit_value", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'Mbps'", + "generated": null, + "name": "download_limit_unit", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "repositories_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "user_id", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "token", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "expires_at", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "ip_address", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "user_agent", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "impersonated_by", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "active_organization_id", + "entityType": "columns", + "table": "sessions_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "provider_id", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "user_id", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "issuer", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "domain", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "auto_link_matching_emails", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "oidc_config", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "saml_config", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "sso_provider" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "two_factor" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "secret", + "entityType": "columns", + "table": "two_factor" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backup_codes", + "entityType": "columns", + "table": "two_factor" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "user_id", + "entityType": "columns", + "table": "two_factor" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "username", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "password_hash", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "has_downloaded_restic_password", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "email", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "email_verified", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "image", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "display_username", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "two_factor_enabled", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'user'", + "generated": null, + "name": "role", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "banned", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "ban_reason", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "ban_expires", + "entityType": "columns", + "table": "users_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "verification" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "identifier", + "entityType": "columns", + "table": "verification" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "value", + "entityType": "columns", + "table": "verification" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "expires_at", + "entityType": "columns", + "table": "verification" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "verification" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "verification" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "short_id", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "provisioning_id", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'unmounted'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_error", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "last_health_check", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "(unixepoch() * 1000)", + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "config", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "auto_remount", + "entityType": "columns", + "table": "volumes_table" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "organization_id", + "entityType": "columns", + "table": "volumes_table" + }, + { + "columns": ["user_id"], + "tableTo": "users_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "account_user_id_users_table_id_fk", + "entityType": "fks", + "table": "account" + }, + { + "columns": ["schedule_id"], + "tableTo": "backup_schedules_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedule_mirrors_table_schedule_id_backup_schedules_table_id_fk", + "entityType": "fks", + "table": "backup_schedule_mirrors_table" + }, + { + "columns": ["repository_id"], + "tableTo": "repositories_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedule_mirrors_table_repository_id_repositories_table_id_fk", + "entityType": "fks", + "table": "backup_schedule_mirrors_table" + }, + { + "columns": ["schedule_id"], + "tableTo": "backup_schedules_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedule_notifications_table_schedule_id_backup_schedules_table_id_fk", + "entityType": "fks", + "table": "backup_schedule_notifications_table" + }, + { + "columns": ["destination_id"], + "tableTo": "notification_destinations_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedule_notifications_table_destination_id_notification_destinations_table_id_fk", + "entityType": "fks", + "table": "backup_schedule_notifications_table" + }, + { + "columns": ["volume_id"], + "tableTo": "volumes_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedules_table_volume_id_volumes_table_id_fk", + "entityType": "fks", + "table": "backup_schedules_table" + }, + { + "columns": ["repository_id"], + "tableTo": "repositories_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedules_table_repository_id_repositories_table_id_fk", + "entityType": "fks", + "table": "backup_schedules_table" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "backup_schedules_table_organization_id_organization_id_fk", + "entityType": "fks", + "table": "backup_schedules_table" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "invitation_organization_id_organization_id_fk", + "entityType": "fks", + "table": "invitation" + }, + { + "columns": ["inviter_id"], + "tableTo": "users_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "invitation_inviter_id_users_table_id_fk", + "entityType": "fks", + "table": "invitation" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "member_organization_id_organization_id_fk", + "entityType": "fks", + "table": "member" + }, + { + "columns": ["user_id"], + "tableTo": "users_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "member_user_id_users_table_id_fk", + "entityType": "fks", + "table": "member" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "notification_destinations_table_organization_id_organization_id_fk", + "entityType": "fks", + "table": "notification_destinations_table" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "repositories_table_organization_id_organization_id_fk", + "entityType": "fks", + "table": "repositories_table" + }, + { + "columns": ["user_id"], + "tableTo": "users_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "sessions_table_user_id_users_table_id_fk", + "entityType": "fks", + "table": "sessions_table" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_sso_provider_organization_id_organization_id_fk", + "entityType": "fks", + "table": "sso_provider" + }, + { + "columns": ["user_id"], + "tableTo": "users_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "nameExplicit": false, + "name": "fk_sso_provider_user_id_users_table_id_fk", + "entityType": "fks", + "table": "sso_provider" + }, + { + "columns": ["user_id"], + "tableTo": "users_table", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "two_factor_user_id_users_table_id_fk", + "entityType": "fks", + "table": "two_factor" + }, + { + "columns": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "volumes_table_organization_id_organization_id_fk", + "entityType": "fks", + "table": "volumes_table" + }, + { + "columns": ["schedule_id", "destination_id"], + "nameExplicit": false, + "name": "backup_schedule_notifications_table_schedule_id_destination_id_pk", + "entityType": "pks", + "table": "backup_schedule_notifications_table" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "account_pk", + "table": "account", + "entityType": "pks" + }, + { + "columns": ["key"], + "nameExplicit": false, + "name": "app_metadata_pk", + "table": "app_metadata", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "backup_schedule_mirrors_table_pk", + "table": "backup_schedule_mirrors_table", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "backup_schedules_table_pk", + "table": "backup_schedules_table", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "invitation_pk", + "table": "invitation", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "member_pk", + "table": "member", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "notification_destinations_table_pk", + "table": "notification_destinations_table", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "organization_pk", + "table": "organization", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "repositories_table_pk", + "table": "repositories_table", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "sessions_table_pk", + "table": "sessions_table", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "sso_provider_pk", + "table": "sso_provider", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "two_factor_pk", + "table": "two_factor", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "users_table_pk", + "table": "users_table", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "verification_pk", + "table": "verification", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "volumes_table_pk", + "table": "volumes_table", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "user_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "account_userId_idx", + "entityType": "indexes", + "table": "account" + }, + { + "columns": [ + { + "value": "organization_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "invitation_organizationId_idx", + "entityType": "indexes", + "table": "invitation" + }, + { + "columns": [ + { + "value": "email", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "invitation_email_idx", + "entityType": "indexes", + "table": "invitation" + }, + { + "columns": [ + { + "value": "organization_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "member_organizationId_idx", + "entityType": "indexes", + "table": "member" + }, + { + "columns": [ + { + "value": "user_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "member_userId_idx", + "entityType": "indexes", + "table": "member" + }, + { + "columns": [ + { + "value": "organization_id", + "isExpression": false + }, + { + "value": "user_id", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "member_org_user_uidx", + "entityType": "indexes", + "table": "member" + }, + { + "columns": [ + { + "value": "slug", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "organization_slug_uidx", + "entityType": "indexes", + "table": "organization" + }, + { + "columns": [ + { + "value": "organization_id", + "isExpression": false + }, + { + "value": "provisioning_id", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "repositories_table_org_provisioning_id_uidx", + "entityType": "indexes", + "table": "repositories_table" + }, + { + "columns": [ + { + "value": "user_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "sessionsTable_userId_idx", + "entityType": "indexes", + "table": "sessions_table" + }, + { + "columns": [ + { + "value": "secret", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "twoFactor_secret_idx", + "entityType": "indexes", + "table": "two_factor" + }, + { + "columns": [ + { + "value": "user_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "twoFactor_userId_idx", + "entityType": "indexes", + "table": "two_factor" + }, + { + "columns": [ + { + "value": "identifier", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "verification_identifier_idx", + "entityType": "indexes", + "table": "verification" + }, + { + "columns": [ + { + "value": "organization_id", + "isExpression": false + }, + { + "value": "provisioning_id", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "volumes_table_org_provisioning_id_uidx", + "entityType": "indexes", + "table": "volumes_table" + }, + { + "columns": ["schedule_id", "repository_id"], + "nameExplicit": false, + "name": "backup_schedule_mirrors_table_schedule_id_repository_id_unique", + "entityType": "uniques", + "table": "backup_schedule_mirrors_table" + }, + { + "columns": ["name", "organization_id"], + "nameExplicit": false, + "name": "volumes_table_name_organization_id_unique", + "entityType": "uniques", + "table": "volumes_table" + }, + { + "columns": ["short_id"], + "nameExplicit": false, + "name": "backup_schedules_table_short_id_unique", + "entityType": "uniques", + "table": "backup_schedules_table" + }, + { + "columns": ["short_id"], + "nameExplicit": false, + "name": "repositories_table_short_id_unique", + "entityType": "uniques", + "table": "repositories_table" + }, + { + "columns": ["token"], + "nameExplicit": false, + "name": "sessions_table_token_unique", + "entityType": "uniques", + "table": "sessions_table" + }, + { + "columns": ["provider_id"], + "nameExplicit": false, + "name": "sso_provider_provider_id_unique", + "entityType": "uniques", + "table": "sso_provider" + }, + { + "columns": ["username"], + "nameExplicit": false, + "name": "users_table_username_unique", + "entityType": "uniques", + "table": "users_table" + }, + { + "columns": ["email"], + "nameExplicit": false, + "name": "users_table_email_unique", + "entityType": "uniques", + "table": "users_table" + }, + { + "columns": ["short_id"], + "nameExplicit": false, + "name": "volumes_table_short_id_unique", + "entityType": "uniques", + "table": "volumes_table" + } + ], + "renames": [] +} diff --git a/app/server/core/config.ts b/app/server/core/config.ts index 18284b95..1b3b006f 100644 --- a/app/server/core/config.ts +++ b/app/server/core/config.ts @@ -38,6 +38,7 @@ const envSchema = z APP_SECRET: z.string().min(32).max(256), BASE_URL: z.string(), ENABLE_DEV_PANEL: z.string().default("false"), + PROVISIONING_PATH: z.string().optional(), }) .transform((s) => ({ __prod__: s.NODE_ENV === "production", @@ -57,6 +58,7 @@ const envSchema = z baseUrl: s.BASE_URL, isSecure: s.BASE_URL?.startsWith("https://") ?? false, enableDevPanel: s.ENABLE_DEV_PANEL === "true", + provisioningPath: s.PROVISIONING_PATH, })); const parseConfig = (env: unknown) => { diff --git a/app/server/db/schema.ts b/app/server/db/schema.ts index 38186b03..6b7f72db 100644 --- a/app/server/db/schema.ts +++ b/app/server/db/schema.ts @@ -203,6 +203,7 @@ export const volumesTable = sqliteTable( { id: int().primaryKey({ autoIncrement: true }), shortId: text("short_id").$type().notNull().unique(), + provisioningId: text("provisioning_id"), name: text().notNull(), type: text().$type().notNull(), status: text().$type().notNull().default("unmounted"), @@ -223,7 +224,10 @@ export const volumesTable = sqliteTable( .notNull() .references(() => organization.id, { onDelete: "cascade" }), }, - (table) => [unique().on(table.name, table.organizationId)], + (table) => [ + unique().on(table.name, table.organizationId), + uniqueIndex("volumes_table_org_provisioning_id_uidx").on(table.organizationId, table.provisioningId), + ], ); export type Volume = typeof volumesTable.$inferSelect; export type VolumeInsert = typeof volumesTable.$inferInsert; @@ -231,35 +235,42 @@ export type VolumeInsert = typeof volumesTable.$inferInsert; /** * Repositories Table */ -export const repositoriesTable = sqliteTable("repositories_table", { - id: text().primaryKey(), - shortId: text("short_id").$type().notNull().unique(), - name: text().notNull(), - type: text().$type().notNull(), - config: text("config", { mode: "json" }).$type().notNull(), - compressionMode: text("compression_mode").$type().default("auto"), - status: text().$type().default("unknown"), - lastChecked: int("last_checked", { mode: "number" }), - lastError: text("last_error"), - doctorResult: text("doctor_result", { mode: "json" }).$type(), - stats: text("stats", { mode: "json" }).$type(), - statsUpdatedAt: int("stats_updated_at", { mode: "number" }), - uploadLimitEnabled: int("upload_limit_enabled", { mode: "boolean" }).notNull().default(false), - uploadLimitValue: real("upload_limit_value").notNull().default(1), - uploadLimitUnit: text("upload_limit_unit").$type().notNull().default("Mbps"), - downloadLimitEnabled: int("download_limit_enabled", { mode: "boolean" }).notNull().default(false), - downloadLimitValue: real("download_limit_value").notNull().default(1), - downloadLimitUnit: text("download_limit_unit").$type().notNull().default("Mbps"), - createdAt: int("created_at", { mode: "number" }) - .notNull() - .default(sql`(unixepoch() * 1000)`), - updatedAt: int("updated_at", { mode: "number" }) - .notNull() - .default(sql`(unixepoch() * 1000)`), - organizationId: text("organization_id") - .notNull() - .references(() => organization.id, { onDelete: "cascade" }), -}); +export const repositoriesTable = sqliteTable( + "repositories_table", + { + id: text().primaryKey(), + shortId: text("short_id").$type().notNull().unique(), + provisioningId: text("provisioning_id"), + name: text().notNull(), + type: text().$type().notNull(), + config: text("config", { mode: "json" }).$type().notNull(), + compressionMode: text("compression_mode").$type().default("auto"), + status: text().$type().default("unknown"), + lastChecked: int("last_checked", { mode: "number" }), + lastError: text("last_error"), + doctorResult: text("doctor_result", { mode: "json" }).$type(), + stats: text("stats", { mode: "json" }).$type(), + statsUpdatedAt: int("stats_updated_at", { mode: "number" }), + uploadLimitEnabled: int("upload_limit_enabled", { mode: "boolean" }).notNull().default(false), + uploadLimitValue: real("upload_limit_value").notNull().default(1), + uploadLimitUnit: text("upload_limit_unit").$type().notNull().default("Mbps"), + downloadLimitEnabled: int("download_limit_enabled", { mode: "boolean" }).notNull().default(false), + downloadLimitValue: real("download_limit_value").notNull().default(1), + downloadLimitUnit: text("download_limit_unit").$type().notNull().default("Mbps"), + createdAt: int("created_at", { mode: "number" }) + .notNull() + .default(sql`(unixepoch() * 1000)`), + updatedAt: int("updated_at", { mode: "number" }) + .notNull() + .default(sql`(unixepoch() * 1000)`), + organizationId: text("organization_id") + .notNull() + .references(() => organization.id, { onDelete: "cascade" }), + }, + (table) => [ + uniqueIndex("repositories_table_org_provisioning_id_uidx").on(table.organizationId, table.provisioningId), + ], +); export type Repository = typeof repositoriesTable.$inferSelect; export type RepositoryInsert = typeof repositoriesTable.$inferInsert; diff --git a/app/server/modules/lifecycle/startup.ts b/app/server/modules/lifecycle/startup.ts index 507a0db0..bd55caac 100644 --- a/app/server/modules/lifecycle/startup.ts +++ b/app/server/modules/lifecycle/startup.ts @@ -14,6 +14,9 @@ import { VolumeAutoRemountJob } from "~/server/jobs/auto-remount"; import { cache } from "~/server/utils/cache"; import { withContext } from "~/server/core/request-context"; import { backupsService } from "../backups/backups.service"; +import { config } from "~/server/core/config"; +import { syncProvisionedResources } from "../provisioning/provisioning"; +import { toMessage } from "~/server/utils/errors"; const ensureLatestConfigurationSchema = async () => { const volumes = await db.query.volumesTable.findMany({}); @@ -53,6 +56,10 @@ export const startup = async () => { await Scheduler.start(); await Scheduler.clear(); + await syncProvisionedResources(config.provisioningPath).catch((error) => { + logger.error(`Provisioning sync failed: ${toMessage(error)}`); + }); + await ensureLatestConfigurationSchema(); const { deletedSchedules } = await backupsService.cleanupOrphanedSchedules().catch((err) => { diff --git a/app/server/modules/provisioning/provisioning.test.ts b/app/server/modules/provisioning/provisioning.test.ts new file mode 100644 index 00000000..97fa9c99 --- /dev/null +++ b/app/server/modules/provisioning/provisioning.test.ts @@ -0,0 +1,391 @@ +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { db } from "~/server/db/db"; +import { backupSchedulesTable, repositoriesTable } from "~/server/db/schema"; +import { restic } from "~/server/core/restic"; +import { createTestSession } from "~/test/helpers/auth"; +import { generateShortId } from "~/server/utils/id"; +import { provisionedResourcesSchema, readProvisionedResourcesFile, syncProvisionedResources } from "./provisioning"; + +describe("provisioning", () => { + afterEach(() => { + mock.restore(); + }); + + test("rejects duplicate ids for the same organization", () => { + expect(() => + provisionedResourcesSchema.parse({ + version: 1, + repositories: [ + { + id: "shared-id", + organizationId: "acme", + name: "Repository one", + backend: "local", + config: { + backend: "local", + path: "/tmp/one", + }, + }, + { + id: "shared-id", + organizationId: "acme", + name: "Repository two", + backend: "local", + config: { + backend: "local", + path: "/tmp/two", + }, + }, + ], + volumes: [], + }), + ).toThrow("Duplicate provisioned repository id for organization acme: shared-id"); + }); + + test("syncs provisioned repositories and volumes into the database", async () => { + const { organizationId } = await createTestSession(); + + process.env.ZEROBYTE_PROVISIONED_ACCESS_KEY = "access-key-from-env"; + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-provisioning-")); + const provisioningPath = path.join(tempDir, "provisioning.json"); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [ + { + id: "aws-prod", + organizationId, + name: "AWS Production", + backend: "s3", + compressionMode: "auto", + config: { + backend: "s3", + endpoint: "https://s3.amazonaws.com", + bucket: "company-backups", + accessKeyId: "env://ZEROBYTE_PROVISIONED_ACCESS_KEY", + secretAccessKey: "plain-secret-key", + }, + }, + ], + volumes: [ + { + id: "shared-directory", + organizationId, + name: "Shared Directory", + backend: "directory", + autoRemount: true, + config: { + backend: "directory", + path: tempDir, + }, + }, + ], + }), + ); + + const parsed = await readProvisionedResourcesFile(provisioningPath); + expect(parsed.repositories).toHaveLength(1); + + await syncProvisionedResources(provisioningPath); + + const repositories = await db.query.repositoriesTable.findMany({ where: { organizationId } }); + const volumes = await db.query.volumesTable.findMany({ where: { organizationId } }); + + const repository = repositories.find((item) => item.name === "AWS Production"); + const volume = volumes.find((item) => item.name === "Shared Directory"); + + expect(repository).toBeTruthy(); + if (!repository || repository.config.backend !== "s3") { + throw new Error("Expected provisioned repository to be stored as an s3 repository"); + } + expect(repository.config.accessKeyId).toBe("access-key-from-env"); + expect(repository.provisioningId).toBeDefined(); + + expect(volume).toBeTruthy(); + expect(volume?.status).toBe("mounted"); + expect(volume?.provisioningId).toBeDefined(); + }); + + test("removes managed resources when delete is set", async () => { + const { organizationId } = await createTestSession(); + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-provisioning-")); + const provisioningPath = path.join(tempDir, "provisioning.json"); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [ + { + id: "repo-to-remove", + organizationId, + name: "Repo to remove", + backend: "local", + config: { + backend: "local", + path: tempDir, + isExistingRepository: true, + }, + }, + ], + volumes: [ + { + id: "volume-to-remove", + organizationId, + name: "Volume to remove", + backend: "directory", + config: { + backend: "directory", + path: tempDir, + }, + }, + ], + }), + ); + + await syncProvisionedResources(provisioningPath); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [ + { + id: "repo-to-remove", + organizationId, + name: "Repo to remove", + backend: "local", + delete: true, + config: { + backend: "local", + path: tempDir, + isExistingRepository: true, + }, + }, + ], + volumes: [ + { + id: "volume-to-remove", + organizationId, + name: "Volume to remove", + backend: "directory", + delete: true, + config: { + backend: "directory", + path: tempDir, + }, + }, + ], + }), + ); + + await syncProvisionedResources(provisioningPath); + + const repositories = await db.query.repositoriesTable.findMany({ where: { organizationId } }); + const volumes = await db.query.volumesTable.findMany({ where: { organizationId } }); + + expect(repositories.filter((item) => item.provisioningId)).toHaveLength(0); + expect(volumes.filter((item) => item.provisioningId)).toHaveLength(0); + }); + + test("renaming a provisioned volume updates it in place", async () => { + const { organizationId } = await createTestSession(); + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-provisioning-")); + const provisioningPath = path.join(tempDir, "provisioning.json"); + const provisionedVolumeId = "shared-directory"; + + await db.insert(repositoriesTable).values({ + id: crypto.randomUUID(), + shortId: generateShortId(), + name: "Schedule Repository", + type: "local", + config: { + backend: "local", + path: tempDir, + isExistingRepository: true, + }, + organizationId, + }); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [], + volumes: [ + { + id: provisionedVolumeId, + organizationId, + name: "Shared Directory", + backend: "directory", + config: { + backend: "directory", + path: tempDir, + }, + }, + ], + }), + ); + + await syncProvisionedResources(provisioningPath); + + const initialVolume = await db.query.volumesTable.findFirst({ + where: { organizationId, provisioningId: `provisioned:${organizationId}:${provisionedVolumeId}` }, + }); + expect(initialVolume).toBeTruthy(); + if (!initialVolume) { + throw new Error("Expected initial provisioned volume to exist"); + } + + const repository = await db.query.repositoriesTable.findFirst({ where: { organizationId } }); + expect(repository).toBeTruthy(); + if (!repository) { + throw new Error("Expected repository to exist"); + } + + const [schedule] = await db + .insert(backupSchedulesTable) + .values({ + shortId: generateShortId(), + name: "Daily Backup", + volumeId: initialVolume.id, + repositoryId: repository.id, + cronExpression: "0 0 * * *", + organizationId, + }) + .returning(); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [], + volumes: [ + { + id: provisionedVolumeId, + organizationId, + name: "Shared Directory Renamed", + backend: "directory", + config: { + backend: "directory", + path: tempDir, + }, + }, + ], + }), + ); + + await syncProvisionedResources(provisioningPath); + + const volumes = await db.query.volumesTable.findMany({ where: { organizationId } }); + expect(volumes).toHaveLength(1); + expect(volumes[0]?.id).toBe(initialVolume.id); + expect(volumes[0]?.name).toBe("Shared Directory Renamed"); + expect(volumes[0]?.provisioningId).toBe(`provisioned:${organizationId}:${provisionedVolumeId}`); + + const persistedSchedule = await db.query.backupSchedulesTable.findFirst({ where: { id: schedule.id } }); + expect(persistedSchedule).toBeTruthy(); + expect(persistedSchedule?.volumeId).toBe(initialVolume.id); + }); + + test("does not partially sync resources when resolving a provisioned secret fails", async () => { + const { organizationId } = await createTestSession(); + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-provisioning-")); + const provisioningPath = path.join(tempDir, "provisioning.json"); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [ + { + id: "broken-secret-repo", + organizationId, + name: "Broken Secret Repo", + backend: "rest", + config: { + backend: "rest", + url: "https://rest.example.test", + password: "file://ZEROBYTE_WEBDAV_PASSWORD", + }, + }, + ], + volumes: [ + { + id: "shared-directory", + organizationId, + name: "Shared Directory", + backend: "directory", + config: { + backend: "directory", + path: tempDir, + }, + }, + ], + }), + ); + + await expect(syncProvisionedResources(provisioningPath)).rejects.toThrow( + "Provisioned secret file not found: /run/secrets/ZEROBYTE_WEBDAV_PASSWORD", + ); + + const repositories = await db.query.repositoriesTable.findMany({ where: { organizationId } }); + const volumes = await db.query.volumesTable.findMany({ where: { organizationId } }); + + expect(repositories.filter((repository) => repository.provisioningId)).toHaveLength(0); + expect(volumes.filter((volume) => volume.provisioningId)).toHaveLength(0); + }); + + test("initializes a non-existing provisioned repository on first sync", async () => { + const { organizationId } = await createTestSession(); + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-provisioning-")); + const provisioningPath = path.join(tempDir, "provisioning.json"); + const initMock = mock(() => Promise.resolve({ success: true, error: null })); + + spyOn(restic, "init").mockImplementation(initMock); + + await fs.writeFile( + provisioningPath, + JSON.stringify({ + version: 1, + repositories: [ + { + id: "new-local-repo", + organizationId, + name: "New Local Repo", + backend: "local", + config: { + backend: "local", + path: tempDir, + isExistingRepository: false, + }, + }, + ], + volumes: [], + }), + ); + + await syncProvisionedResources(provisioningPath); + + expect(initMock).toHaveBeenCalledTimes(1); + + const repository = await db.query.repositoriesTable.findFirst({ + where: { + organizationId, + provisioningId: `provisioned:${organizationId}:new-local-repo`, + }, + }); + + expect(repository).toBeTruthy(); + expect(repository?.status).toBe("healthy"); + expect(repository?.lastChecked).toEqual(expect.any(Number)); + expect(repository?.lastError).toBeNull(); + }); +}); diff --git a/app/server/modules/provisioning/provisioning.ts b/app/server/modules/provisioning/provisioning.ts new file mode 100644 index 00000000..ea9f4d7d --- /dev/null +++ b/app/server/modules/provisioning/provisioning.ts @@ -0,0 +1,262 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { + COMPRESSION_MODES, + REPOSITORY_BACKENDS, + repositoryConfigSchema, + type RepositoryConfig, +} from "@zerobyte/core/restic"; +import { logger } from "@zerobyte/core/node"; +import { z } from "zod"; +import { config as appConfig } from "~/server/core/config"; +import { restic } from "~/server/core/restic"; +import { db } from "~/server/db/db"; +import { repositoriesTable, volumesTable } from "~/server/db/schema"; +import { mapRepositoryConfigSecrets } from "~/server/modules/repositories/repository-config-secrets"; +import { mapVolumeConfigSecrets } from "~/server/modules/volumes/volume-config-secrets"; +import { BACKEND_TYPES, volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; +import { cryptoUtils } from "~/server/utils/crypto"; +import { toMessage } from "~/server/utils/errors"; +import { generateShortId } from "~/server/utils/id"; + +const envSecretPrefix = "env://"; +const fileSecretPrefix = "file://"; + +const provisionedRepositorySchema = z.object({ + id: z.string().min(1), + organizationId: z.string().min(1), + name: z.string().min(1), + compressionMode: z.enum(COMPRESSION_MODES).optional(), + config: repositoryConfigSchema, + backend: z.enum(REPOSITORY_BACKENDS), + delete: z.boolean().default(false), +}); +type ProvisionedRepository = z.infer; + +const provisionedVolumeSchema = z.object({ + id: z.string().min(1), + organizationId: z.string().min(1), + name: z.string().min(1), + autoRemount: z.boolean().default(true), + config: volumeConfigSchema, + delete: z.boolean().default(false), + backend: z.enum(BACKEND_TYPES), +}); +type ProvisionedVolume = z.infer; + +export const provisionedResourcesSchema = z + .object({ + version: z.literal(1).default(1), + repositories: z.array(provisionedRepositorySchema).default([]), + volumes: z.array(provisionedVolumeSchema).default([]), + }) + .superRefine((value, ctx) => { + const repositoryIds = new Set(); + for (const repository of value.repositories) { + const key = `${repository.organizationId}:${repository.id}`; + if (repositoryIds.has(key)) { + ctx.addIssue({ + code: "custom", + message: `Duplicate provisioned repository id for organization ${repository.organizationId}: ${repository.id}`, + path: ["repositories"], + }); + } + repositoryIds.add(key); + } + + const volumeIds = new Set(); + for (const volume of value.volumes) { + const key = `${volume.organizationId}:${volume.id}`; + if (volumeIds.has(key)) { + ctx.addIssue({ + code: "custom", + message: `Duplicate provisioned volume id for organization ${volume.organizationId}: ${volume.id}`, + path: ["volumes"], + }); + } + volumeIds.add(key); + } + }); + +type ProvisionedResources = z.infer; + +export const readProvisionedResourcesFile = async (filePath: string): Promise => { + const content = await fs.readFile(filePath, "utf-8"); + const parsed = JSON.parse(content) as unknown; + + return provisionedResourcesSchema.parse(parsed); +}; + +const resolveProvisioningSecret = async (value: string): Promise => { + if (!value) { + return value; + } + + if (value.startsWith(envSecretPrefix)) { + const name = value.slice(envSecretPrefix.length); + if (!name) { + throw new Error("Provisioned env secret reference is missing a variable name"); + } + + const resolved = process.env[name]; + if (resolved === undefined) { + throw new Error(`Environment variable not set: ${name}`); + } + + return resolved; + } + + if (value.startsWith(fileSecretPrefix)) { + const secretName = value.slice(fileSecretPrefix.length).replace(/^\/+/, ""); + if (!secretName) { + throw new Error("Provisioned file secret reference is missing a secret name"); + } + if (secretName.includes("/") || secretName.includes("\\") || secretName.includes("\0")) { + throw new Error("Provisioned file secret reference must be a single path segment"); + } + + const secretPath = path.join("/run/secrets", secretName); + const content = await fs.readFile(secretPath, "utf-8").catch(() => { + throw new Error(`Provisioned secret file not found: ${secretPath}`); + }); + return content.trimEnd(); + } + + return value; +}; + +const sealProvisionedSecret = async (value: string): Promise => { + const resolved = await resolveProvisioningSecret(value); + return cryptoUtils.sealSecret(resolved); +}; + +const encryptProvisionedRepositoryConfig = async (config: RepositoryConfig): Promise => { + return await mapRepositoryConfigSecrets(config, sealProvisionedSecret); +}; + +const encryptProvisionedVolumeConfig = async (config: BackendConfig): Promise => { + return await mapVolumeConfigSecrets(config, sealProvisionedSecret); +}; + +const syncProvisionedRepositories = async (repositories: ProvisionedRepository[]) => { + const existingRepositories = await db.query.repositoriesTable.findMany({ + where: { AND: [{ provisioningId: { isNotNull: true } }] }, + }); + + for (const repository of repositories) { + const provisioningId = `provisioned:${repository.organizationId}:${repository.id}`; + + if (repository.delete) { + await db.delete(repositoriesTable).where(eq(repositoriesTable.provisioningId, provisioningId)); + continue; + } + + const existing = existingRepositories.find((r) => r.provisioningId === provisioningId); + const encryptedConfig = await encryptProvisionedRepositoryConfig(repository.config); + + if (!existing) { + const id = Bun.randomUUIDv7(); + + await db.insert(repositoriesTable).values({ + id, + provisioningId: provisioningId, + shortId: generateShortId(), + name: repository.name, + type: repository.backend, + config: encryptedConfig, + compressionMode: repository.compressionMode, + status: "unknown", + organizationId: repository.organizationId, + }); + + if (!repository.config.isExistingRepository) { + const result = await restic + .init(encryptedConfig, repository.organizationId, { timeoutMs: appConfig.serverIdleTimeout * 1000 }) + .catch((error) => ({ success: false, error })); + + await db + .update(repositoriesTable) + .set({ + status: result.error ? "error" : "healthy", + lastChecked: Date.now(), + lastError: result.error ? toMessage(result.error) : null, + updatedAt: Date.now(), + }) + .where(eq(repositoriesTable.id, id)); + + if (result.error) { + logger.error(`Provisioned repository ${repository.name} failed to initialize: ${toMessage(result.error)}`); + } + } + continue; + } + + const updatePayload = { + name: repository.name, + type: repository.backend, + config: encryptedConfig, + compressionMode: repository.compressionMode, + organizationId: repository.organizationId, + updatedAt: Date.now(), + }; + + await db.update(repositoriesTable).set(updatePayload).where(eq(repositoriesTable.id, existing.id)); + } +}; + +const syncProvisionedVolumes = async (volumes: ProvisionedVolume[]) => { + const existingVolumes = await db.query.volumesTable.findMany({ + where: { AND: [{ provisioningId: { isNotNull: true } }] }, + }); + + for (const volume of volumes) { + const provisioningId = `provisioned:${volume.organizationId}:${volume.id}`; + + if (volume.delete) { + await db.delete(volumesTable).where(eq(volumesTable.provisioningId, provisioningId)); + continue; + } + + const existing = existingVolumes.find((v) => v.provisioningId === provisioningId); + + if (!existing) { + await db.insert(volumesTable).values({ + shortId: generateShortId(), + provisioningId: provisioningId, + name: volume.name, + type: volume.backend, + config: await encryptProvisionedVolumeConfig(volume.config), + autoRemount: volume.autoRemount, + status: volume.autoRemount ? "mounted" : "unmounted", + organizationId: volume.organizationId, + }); + continue; + } + + const updatePayload = { + name: volume.name, + type: volume.backend, + config: await encryptProvisionedVolumeConfig(volume.config), + autoRemount: volume.autoRemount, + organizationId: volume.organizationId, + updatedAt: Date.now(), + }; + + await db.update(volumesTable).set(updatePayload).where(eq(volumesTable.id, existing.id)); + } +}; + +export const syncProvisionedResources = async (filePath?: string) => { + if (!filePath) { + return; + } + + const resources = await readProvisionedResourcesFile(filePath); + await syncProvisionedRepositories(resources.repositories); + await syncProvisionedVolumes(resources.volumes); + + logger.info( + `Synchronized ${resources.repositories.length} provisioned repositor${resources.repositories.length === 1 ? "y" : "ies"} and ${resources.volumes.length} provisioned volume${resources.volumes.length === 1 ? "" : "s"}`, + ); +}; diff --git a/app/server/modules/repositories/__tests__/repositories.controller.test.ts b/app/server/modules/repositories/__tests__/repositories.controller.test.ts index 50bff3a1..25655f49 100644 --- a/app/server/modules/repositories/__tests__/repositories.controller.test.ts +++ b/app/server/modules/repositories/__tests__/repositories.controller.test.ts @@ -40,6 +40,29 @@ const createRepositoryRecord = async (organizationId: string) => { return repository; }; +const createManagedRepositoryRecord = async (organizationId: string) => { + const [repository] = await db + .insert(repositoriesTable) + .values({ + id: Bun.randomUUIDv7(), + provisioningId: `provisioned:${crypto.randomUUID()}`, + shortId: generateShortId(), + name: `Managed-${crypto.randomUUID()}`, + type: "local", + config: { + backend: "local", + path: `/tmp/repository-${crypto.randomUUID()}`, + isExistingRepository: true, + }, + compressionMode: "off", + status: "healthy", + organizationId, + }) + .returning(); + + return repository; +}; + describe("repositories security", () => { test("should return 401 if no session cookie is provided", async () => { const res = await app.request("/api/v1/repositories"); @@ -175,6 +198,30 @@ describe("repositories security", () => { expect(res.status).toBe(400); }); + + test("should accept env:// values as plain secrets on create", async () => { + const { headers } = await createTestSession(); + const res = await app.request("/api/v1/repositories", { + method: "POST", + headers: { + ...headers, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + name: "S3 repo", + compressionMode: "auto", + config: { + backend: "s3", + endpoint: "https://s3.amazonaws.com", + bucket: "bucket-name", + accessKeyId: "access-key", + secretAccessKey: "env://ZEROBYTE_REPOSITORY_SECRET", + }, + }), + }); + + expect(res.status).not.toBe(400); + }); }); }); @@ -338,4 +385,47 @@ describe("repositories updates", () => { } }); }); + + test("GET marks provisioned repositories as managed", async () => { + const { headers, organizationId } = await createTestSession(); + const repository = await createManagedRepositoryRecord(organizationId); + + const res = await app.request(`/api/v1/repositories/${repository.shortId}`, { headers }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.provisioningId).toBeDefined(); + }); + + test("PATCH allows updates for managed repositories", async () => { + const { headers, organizationId } = await createTestSession(); + const repository = await createManagedRepositoryRecord(organizationId); + + const res = await app.request(`/api/v1/repositories/${repository.shortId}`, { + method: "PATCH", + headers: { + ...headers, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + name: "Updated repository", + }), + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.name).toBe("Updated repository"); + }); + + test("DELETE allows managed repositories", async () => { + const { headers, organizationId } = await createTestSession(); + const repository = await createManagedRepositoryRecord(organizationId); + + const res = await app.request(`/api/v1/repositories/${repository.shortId}`, { + method: "DELETE", + headers, + }); + + expect(res.status).toBe(200); + }); }); diff --git a/app/server/modules/repositories/repositories.dto.ts b/app/server/modules/repositories/repositories.dto.ts index 29b8432b..f066f6ad 100644 --- a/app/server/modules/repositories/repositories.dto.ts +++ b/app/server/modules/repositories/repositories.dto.ts @@ -14,6 +14,7 @@ import { export const repositorySchema = z.object({ id: z.string(), shortId: z.string(), + provisioningId: z.string().nullable(), name: z.string(), type: z.enum(REPOSITORY_BACKENDS), config: repositoryConfigSchema, diff --git a/app/server/modules/repositories/repositories.service.ts b/app/server/modules/repositories/repositories.service.ts index 5e701f8f..0555b2a5 100644 --- a/app/server/modules/repositories/repositories.service.ts +++ b/app/server/modules/repositories/repositories.service.ts @@ -1,4 +1,3 @@ -import crypto from "node:crypto"; import nodePath from "node:path"; import { and, eq } from "drizzle-orm"; import { BadRequestError, ConflictError, InternalServerError, NotFoundError } from "http-errors-enhanced"; @@ -18,7 +17,6 @@ import { repoMutex } from "../../core/repository-mutex"; import { db } from "../../db/db"; import { repositoriesTable, type Repository } from "../../db/schema"; import { cache, cacheKeys } from "../../utils/cache"; -import { cryptoUtils } from "../../utils/crypto"; import { toMessage } from "../../utils/errors"; import { generateShortId } from "../../utils/id"; import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "@zerobyte/core/restic/server"; @@ -30,6 +28,7 @@ import { findCommonAncestor } from "@zerobyte/core/utils"; import { prepareSnapshotDump } from "./helpers/dump"; import { executeDoctor } from "./helpers/doctor"; import type { ShortId } from "~/server/utils/branded"; +import { decryptRepositoryConfig, encryptRepositoryConfig } from "./repository-config-secrets"; const runningDoctors = new Map(); @@ -57,94 +56,16 @@ const listRepositories = async () => { return repositories; }; -const encryptConfig = async (config: RepositoryConfig): Promise => { - const encryptedConfig: Record = { ...config }; - - if (config.customPassword) { - encryptedConfig.customPassword = await cryptoUtils.sealSecret(config.customPassword); - } - - if (config.cacert) { - encryptedConfig.cacert = await cryptoUtils.sealSecret(config.cacert); - } - - switch (config.backend) { - case "s3": - case "r2": - encryptedConfig.accessKeyId = await cryptoUtils.sealSecret(config.accessKeyId); - encryptedConfig.secretAccessKey = await cryptoUtils.sealSecret(config.secretAccessKey); - break; - case "gcs": - encryptedConfig.credentialsJson = await cryptoUtils.sealSecret(config.credentialsJson); - break; - case "azure": - encryptedConfig.accountKey = await cryptoUtils.sealSecret(config.accountKey); - break; - case "rest": - if (config.username) { - encryptedConfig.username = await cryptoUtils.sealSecret(config.username); - } - if (config.password) { - encryptedConfig.password = await cryptoUtils.sealSecret(config.password); - } - break; - case "sftp": - encryptedConfig.privateKey = await cryptoUtils.sealSecret(config.privateKey); - break; - } - - return encryptedConfig as RepositoryConfig; -}; - -const decryptConfig = async (config: RepositoryConfig): Promise => { - const decryptedConfig: Record = { ...config }; - - if (config.customPassword) { - decryptedConfig.customPassword = await cryptoUtils.resolveSecret(config.customPassword); - } - - if (config.cacert) { - decryptedConfig.cacert = await cryptoUtils.resolveSecret(config.cacert); - } - - switch (config.backend) { - case "s3": - case "r2": - decryptedConfig.accessKeyId = await cryptoUtils.resolveSecret(config.accessKeyId); - decryptedConfig.secretAccessKey = await cryptoUtils.resolveSecret(config.secretAccessKey); - break; - case "gcs": - decryptedConfig.credentialsJson = await cryptoUtils.resolveSecret(config.credentialsJson); - break; - case "azure": - decryptedConfig.accountKey = await cryptoUtils.resolveSecret(config.accountKey); - break; - case "rest": - if (config.username) { - decryptedConfig.username = await cryptoUtils.resolveSecret(config.username); - } - if (config.password) { - decryptedConfig.password = await cryptoUtils.resolveSecret(config.password); - } - break; - case "sftp": - decryptedConfig.privateKey = await cryptoUtils.resolveSecret(config.privateKey); - break; - } - - return decryptedConfig as RepositoryConfig; -}; - const createRepository = async (name: string, config: RepositoryConfig, compressionMode?: CompressionMode) => { const organizationId = getOrganizationId(); - const id = crypto.randomUUID(); + const id = Bun.randomUUIDv7(); const shortId = generateShortId(); if (config.backend === "local" && !config.isExistingRepository) { config.path = `${config.path}/${shortId}`; } - const encryptedConfig = await encryptConfig(config); + const encryptedConfig = await encryptRepositoryConfig(config); const [created] = await db .insert(repositoriesTable) @@ -761,9 +682,9 @@ const updateRepository = async (shortId: ShortId, updates: UpdateRepositoryBody) parsedConfig = nextConfig; } - const decryptedExisting = await decryptConfig(existingConfig); + const decryptedExisting = await decryptRepositoryConfig(existingConfig); const configChanged = updates.config && JSON.stringify(decryptedExisting) !== JSON.stringify(parsedConfig); - const encryptedConfig = updates.config ? await encryptConfig(parsedConfig) : existingConfig; + const encryptedConfig = updates.config ? await encryptRepositoryConfig(parsedConfig) : existingConfig; const updatedAt = Date.now(); const updatePayload = { diff --git a/app/server/modules/repositories/repository-config-secrets.ts b/app/server/modules/repositories/repository-config-secrets.ts new file mode 100644 index 00000000..823ef5cc --- /dev/null +++ b/app/server/modules/repositories/repository-config-secrets.ts @@ -0,0 +1,79 @@ +import type { RepositoryConfig } from "@zerobyte/core/restic"; +import { cryptoUtils } from "~/server/utils/crypto"; + +type SecretTransformer = (value: string) => Promise; + +const transformOptionalSecret = async ( + value: string | undefined, + transformSecret: SecretTransformer, +): Promise => { + if (!value) { + return value; + } + + return await transformSecret(value); +}; + +export const mapRepositoryConfigSecrets = async ( + config: RepositoryConfig, + transformSecret: SecretTransformer, +): Promise => { + const customPassword = await transformOptionalSecret(config.customPassword, transformSecret); + const cacert = await transformOptionalSecret(config.cacert, transformSecret); + + switch (config.backend) { + case "s3": + case "r2": + return { + ...config, + customPassword, + cacert, + accessKeyId: await transformSecret(config.accessKeyId), + secretAccessKey: await transformSecret(config.secretAccessKey), + }; + case "gcs": + return { + ...config, + customPassword, + cacert, + credentialsJson: await transformSecret(config.credentialsJson), + }; + case "azure": + return { + ...config, + customPassword, + cacert, + accountKey: await transformSecret(config.accountKey), + }; + case "rest": + return { + ...config, + customPassword, + cacert, + username: await transformOptionalSecret(config.username, transformSecret), + password: await transformOptionalSecret(config.password, transformSecret), + }; + case "sftp": + return { + ...config, + customPassword, + cacert, + privateKey: await transformSecret(config.privateKey), + }; + case "local": + case "rclone": + return { + ...config, + customPassword, + cacert, + }; + } +}; + +export const encryptRepositoryConfig = async (config: RepositoryConfig): Promise => { + return await mapRepositoryConfigSecrets(config, cryptoUtils.sealSecret); +}; + +export const decryptRepositoryConfig = async (config: RepositoryConfig): Promise => { + return await mapRepositoryConfigSecrets(config, cryptoUtils.resolveSecret); +}; diff --git a/app/server/modules/volumes/__tests__/volumes.controller.test.ts b/app/server/modules/volumes/__tests__/volumes.controller.test.ts index ff94a73f..822cfd7e 100644 --- a/app/server/modules/volumes/__tests__/volumes.controller.test.ts +++ b/app/server/modules/volumes/__tests__/volumes.controller.test.ts @@ -1,9 +1,33 @@ import { test, describe, expect } from "bun:test"; +import { db } from "~/server/db/db"; +import { volumesTable } from "~/server/db/schema"; import { createApp } from "~/server/app"; import { createTestSession, getAuthHeaders } from "~/test/helpers/auth"; +import { generateShortId } from "~/server/utils/id"; const app = createApp(); +const createManagedVolumeRecord = async (organizationId: string) => { + const [volume] = await db + .insert(volumesTable) + .values({ + shortId: generateShortId(), + provisioningId: `provisioned:${organizationId}:${generateShortId()}`, + name: `Managed-${Date.now()}`, + type: "directory", + status: "mounted", + config: { + backend: "directory", + path: "/tmp", + }, + autoRemount: true, + organizationId, + }) + .returning(); + + return volume; +}; + describe("volumes security", () => { test("should return 401 if no session cookie is provided", async () => { const res = await app.request("/api/v1/volumes"); @@ -92,5 +116,48 @@ describe("volumes security", () => { expect(res.status).toBe(400); }); + + test("should mark provisioned volumes as managed", async () => { + const { headers, organizationId } = await createTestSession(); + const volume = await createManagedVolumeRecord(organizationId); + + const res = await app.request(`/api/v1/volumes/${volume.shortId}`, { headers }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.volume.provisioningId).toBeDefined(); + }); + + test("should allow updates for managed volumes", async () => { + const { headers, organizationId } = await createTestSession(); + const volume = await createManagedVolumeRecord(organizationId); + + const res = await app.request(`/api/v1/volumes/${volume.shortId}`, { + method: "PUT", + headers: { + ...headers, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + name: "Updated volume", + }), + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.name).toBe("Updated volume"); + }); + + test("should allow deletion for managed volumes", async () => { + const { headers, organizationId } = await createTestSession(); + const volume = await createManagedVolumeRecord(organizationId); + + const res = await app.request(`/api/v1/volumes/${volume.shortId}`, { + method: "DELETE", + headers, + }); + + expect(res.status).toBe(200); + }); }); }); diff --git a/app/server/modules/volumes/volume-config-secrets.ts b/app/server/modules/volumes/volume-config-secrets.ts new file mode 100644 index 00000000..b8bf25fa --- /dev/null +++ b/app/server/modules/volumes/volume-config-secrets.ts @@ -0,0 +1,47 @@ +import type { BackendConfig } from "~/schemas/volumes"; +import { cryptoUtils } from "~/server/utils/crypto"; + +type SecretTransformer = (value: string) => Promise; + +const transformOptionalSecret = async ( + value: string | undefined, + transformSecret: SecretTransformer, +): Promise => { + if (!value) { + return value; + } + + return await transformSecret(value); +}; + +export const mapVolumeConfigSecrets = async ( + config: BackendConfig, + transformSecret: SecretTransformer, +): Promise => { + switch (config.backend) { + case "smb": + return { + ...config, + password: await transformOptionalSecret(config.password, transformSecret), + }; + case "webdav": + return { + ...config, + password: await transformOptionalSecret(config.password, transformSecret), + }; + case "sftp": + return { + ...config, + password: await transformOptionalSecret(config.password, transformSecret), + privateKey: await transformOptionalSecret(config.privateKey, transformSecret), + }; + case "nfs": + case "directory": + case "rclone": + return config; + } +}; + +export const encryptVolumeConfig = async (config: BackendConfig): Promise => { + return await mapVolumeConfigSecrets(config, cryptoUtils.sealSecret); +}; diff --git a/app/server/modules/volumes/volume.dto.ts b/app/server/modules/volumes/volume.dto.ts index e407e535..6bab8bf4 100644 --- a/app/server/modules/volumes/volume.dto.ts +++ b/app/server/modules/volumes/volume.dto.ts @@ -5,6 +5,7 @@ import { BACKEND_STATUS, BACKEND_TYPES, volumeConfigSchema } from "~/schemas/vol export const volumeSchema = z.object({ id: z.number(), shortId: z.string(), + provisioningId: z.string().nullable(), name: z.string(), type: z.enum(BACKEND_TYPES), status: z.enum(BACKEND_STATUS), diff --git a/app/server/modules/volumes/volume.service.ts b/app/server/modules/volumes/volume.service.ts index dfd64777..9005918c 100644 --- a/app/server/modules/volumes/volume.service.ts +++ b/app/server/modules/volumes/volume.service.ts @@ -5,7 +5,6 @@ import { and, eq } from "drizzle-orm"; import { BadRequestError, InternalServerError, NotFoundError } from "http-errors-enhanced"; import { db } from "../../db/db"; import { volumesTable } from "../../db/schema"; -import { cryptoUtils } from "../../utils/crypto"; import { toMessage } from "../../utils/errors"; import { generateShortId } from "../../utils/id"; import { getStatFs, type StatFs } from "../../utils/mountinfo"; @@ -19,29 +18,7 @@ import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { getOrganizationId } from "~/server/core/request-context"; import { isNodeJSErrnoException } from "~/server/utils/fs"; import { asShortId, type ShortId } from "~/server/utils/branded"; - -async function encryptSensitiveFields(config: BackendConfig): Promise { - switch (config.backend) { - case "smb": - return { - ...config, - password: config.password ? await cryptoUtils.sealSecret(config.password) : undefined, - }; - case "webdav": - return { - ...config, - password: config.password ? await cryptoUtils.sealSecret(config.password) : undefined, - }; - case "sftp": - return { - ...config, - password: config.password ? await cryptoUtils.sealSecret(config.password) : undefined, - privateKey: config.privateKey ? await cryptoUtils.sealSecret(config.privateKey) : undefined, - }; - default: - return config; - } -} +import { encryptVolumeConfig } from "./volume-config-secrets"; const listVolumes = async () => { const organizationId = getOrganizationId(); @@ -70,7 +47,7 @@ const createVolume = async (name: string, backendConfig: BackendConfig) => { } const shortId = generateShortId(); - const encryptedConfig = await encryptSensitiveFields(backendConfig); + const encryptedConfig = await encryptVolumeConfig(backendConfig); const [created] = await db .insert(volumesTable) @@ -206,7 +183,7 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => { } const newConfig = newConfigResult.data; - const encryptedConfig = await encryptSensitiveFields(newConfig); + const encryptedConfig = await encryptVolumeConfig(newConfig); const [updated] = await db .update(volumesTable) @@ -240,19 +217,21 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => { const testConnection = async (backendConfig: BackendConfig) => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-test-")); + const encryptedConfig = await encryptVolumeConfig(backendConfig); const mockVolume = { id: 0, shortId: asShortId("test"), name: "test-connection", path: tempDir, - config: backendConfig, + config: encryptedConfig, createdAt: Date.now(), updatedAt: Date.now(), lastHealthCheck: Date.now(), - type: backendConfig.backend, + type: encryptedConfig.backend, status: "unmounted" as const, lastError: null, + provisioningId: null, autoRemount: true, organizationId: "test-org", }; diff --git a/app/server/utils/crypto.ts b/app/server/utils/crypto.ts index aed32a24..a8aa74ad 100644 --- a/app/server/utils/crypto.ts +++ b/app/server/utils/crypto.ts @@ -1,8 +1,5 @@ import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; import { config } from "../core/config"; -import { isNodeJSErrnoException } from "./fs"; import { promisify } from "node:util"; const hkdf = promisify(crypto.hkdf); @@ -11,9 +8,6 @@ const algorithm = "aes-256-gcm" as const; const keyLength = 32; const encryptionPrefix = "encv1:"; -const envSecretPrefix = "env://"; -const fileSecretPrefix = "file://"; - /** * Checks if a given string is encrypted by looking for the encryption prefix. */ @@ -21,68 +15,6 @@ const isEncrypted = (val?: string): boolean => { return typeof val === "string" && val.startsWith(encryptionPrefix); }; -/** - * Checks if a string looks like a supported secret reference. - * - env://VAR_NAME -> reads process.env.VAR_NAME - * - file://name -> reads a file /run/secrets/name - */ -const isSecretReference = (val?: string): boolean => { - return typeof val === "string" && (val.startsWith(envSecretPrefix) || val.startsWith(fileSecretPrefix)); -}; - -/** - * Resolves an environment variable secret reference. - */ -const resolveEnvSecret = (ref: string): string => { - const name = ref.slice(envSecretPrefix.length); - if (!name) { - throw new Error("env:// reference is missing variable name"); - } - - const value = process.env[name]; - if (value === undefined) { - throw new Error(`Environment variable not set: ${name}`); - } - - return value; -}; - -/** - * Resolves a file-based secret reference. - * Reads the secret from /run/secrets/{name} - */ -const resolveFileSecret = async (ref: string): Promise => { - const secretName = ref.slice(fileSecretPrefix.length); - if (!secretName) { - throw new Error("file:// reference is missing secret name"); - } - - const normalizedName = secretName.replace(/^\/+/, ""); - if (!normalizedName) { - throw new Error("file:// reference is missing secret name"); - } - if (normalizedName.includes("\0") || normalizedName.includes("/") || normalizedName.includes("\\")) { - throw new Error("Invalid secret reference: secret name must be a single path segment"); - } - - const resolvedPath = path.join("/run/secrets", normalizedName); - - try { - const content = await fs.readFile(resolvedPath, "utf-8"); - return content.trimEnd(); - } catch (error) { - if (isNodeJSErrnoException(error)) { - if (error.code === "ENOENT") { - throw new Error(`Secret file not found: ${resolvedPath}`); - } - if (error.code === "EACCES") { - throw new Error(`Permission denied reading secret file: ${resolvedPath}`); - } - } - throw new Error(`Failed to read secret file ${resolvedPath}: ${(error as Error).message}`); - } -}; - /** * Given a string, encrypts it using a randomly generated salt and the APP_SECRET. * Returns the input unchanged if it's empty or already encrypted. @@ -137,51 +69,34 @@ const decrypt = async (encryptedData: string) => { /** * Resolves secret references and encrypted database values. - * - * - encv1:... -> decrypt - * - env://VAR -> read process.env.VAR - * - file://name -> read /run/secrets/name - * - otherwise returns value unchanged */ const resolveSecret = async (value: string): Promise => { - if (!value) { - return value; - } - if (isEncrypted(value)) { return decrypt(value); } - if (value.startsWith(envSecretPrefix)) { - return resolveEnvSecret(value); - } - - if (value.startsWith(fileSecretPrefix)) { - return resolveFileSecret(value); - } - return value; }; /** * Prepares a secret value for storage. - * - * - env://... and file://... are stored as-is (references) - * - encv1:... is stored as-is (already encrypted) - * - otherwise encrypt before storing */ const sealSecret = async (value: string): Promise => { - if (!value) { - return value; - } - - if (isEncrypted(value) || isSecretReference(value)) { + if (isEncrypted(value)) { return value; } return encrypt(value); }; +const sealOptionalSecret = async (value?: string): Promise => { + if (typeof value === "undefined") { + return value; + } + + return sealSecret(value); +}; + async function deriveSecret(label: string) { const derivedKey = await hkdf("sha256", config.appSecret, "", label, 32); @@ -195,6 +110,7 @@ function generateResticPassword(): string { export const cryptoUtils = { resolveSecret, sealSecret, + sealOptionalSecret, deriveSecret, generateResticPassword, isEncrypted, diff --git a/bun.lock b/bun.lock index 3668dc2d..ed673bbd 100644 --- a/bun.lock +++ b/bun.lock @@ -34,6 +34,7 @@ "@tanstack/react-router": "^1.166.7", "@tanstack/react-router-ssr-query": "^1.166.7", "@tanstack/react-start": "^1.166.7", + "@zerobyte/core": "workspace:*", "better-auth": "^1.5.4", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/examples/README.md b/examples/README.md index 8d2e7657..c576859f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,7 +10,7 @@ This folder contains runnable, copy/paste-friendly examples for running Zerobyte - [Simplified Docker Compose (no remote mounts)](simplified-docker-compose/README.md) — reduced-privilege deployment (no `SYS_ADMIN`, no `/dev/fuse`). - [Bind-mount a local directory](directory-bind-mount/README.md) — back up a host folder by mounting it into the container. - [Mount an rclone config](rclone-config-mount/README.md) — use rclone-based repository backends by mounting your rclone config. -- [Secret placeholders + Docker secrets](secrets-placeholders/README.md) — keep secrets out of the DB using `env://...` and `file://...` references. +- [Provisioned repositories and volumes](provisioned-resources/README.md) — operator-managed provisioning with env/file references kept in a mounted JSON file. ### Advanced setups diff --git a/examples/provisioned-resources/.env.example b/examples/provisioned-resources/.env.example new file mode 100644 index 00000000..d0aec48e --- /dev/null +++ b/examples/provisioned-resources/.env.example @@ -0,0 +1,5 @@ +TZ=UTC +BASE_URL=http://localhost:4096 +APP_SECRET=replace-with-openssl-rand-hex-32 +ZEROBYTE_AWS_ACCESS_KEY_ID=replace-with-your-s3-access-key-id +ZEROBYTE_WEBDAV_PASSWORD=replace-with-your-webdav-password diff --git a/examples/provisioned-resources/.gitignore b/examples/provisioned-resources/.gitignore new file mode 100644 index 00000000..e2cd6237 --- /dev/null +++ b/examples/provisioned-resources/.gitignore @@ -0,0 +1,3 @@ +.env +provisioning.json +secrets/aws_secret_access_key diff --git a/examples/provisioned-resources/README.md b/examples/provisioned-resources/README.md new file mode 100644 index 00000000..c64e9442 --- /dev/null +++ b/examples/provisioned-resources/README.md @@ -0,0 +1,89 @@ +# Provisioned repositories and volumes + +This example shows how to keep operator-managed repositories and volumes in a mounted JSON file instead of creating them through the UI. + +At startup, Zerobyte reads the provisioning file, resolves any `env://...` or `file://...` references, encrypts the resolved secrets into the database, and syncs the resources into the normal repositories and volumes lists as managed entries. + +## Why this model + +- Secret references stay in deployment-time config instead of the UI. +- Provisioned repositories and volumes show up in the normal UI and API. +- Secret rotation is just an env/secret update plus a restart. + +## What this example includes + +- `docker-compose.yml` mounts a provisioning file and a Docker secret. +- `.env.example` provides the environment variables used by `env://...` references. +- `provisioning.example.json` provisions one S3 repository and one WebDAV volume. +- `secrets/aws_secret_access_key.example` shows the file consumed by `file://aws_secret_access_key`. + +## Prerequisites + +- Docker + Docker Compose +- An existing Zerobyte organization ID (found in the UI under Settings > Organization) +- An S3-compatible repository target and a WebDAV share, or your own equivalent values + +If this is a brand-new Zerobyte instance, finish first-run setup first so you have a real organization ID, then enable provisioning and restart the container. + +## Setup + +1. Copy the example files: + +```bash +cp .env.example .env +cp provisioning.example.json provisioning.json +cp secrets/aws_secret_access_key.example secrets/aws_secret_access_key +``` + +2. Edit `.env`: + +- Set `APP_SECRET` to a real secret, for example `openssl rand -hex 32` +- Set `ZEROBYTE_AWS_ACCESS_KEY_ID` +- Set `ZEROBYTE_WEBDAV_PASSWORD` +- Adjust `BASE_URL` and `TZ` if needed + +3. Edit `provisioning.json`: + +- Replace `organizationId` with your existing Zerobyte organization ID +- Update the S3 endpoint/bucket values +- Update the WebDAV server, path, and username + +4. Edit `secrets/aws_secret_access_key` and replace the placeholder value with the real secret access key. + +5. Start the stack: + +```bash +docker compose up -d +``` + +## How secret references work + +- `env://ZEROBYTE_AWS_ACCESS_KEY_ID` reads from a container environment variable. +- `env://ZEROBYTE_WEBDAV_PASSWORD` reads from a container environment variable. +- `file://aws_secret_access_key` reads `/run/secrets/aws_secret_access_key` inside the container. +- The resolved values are encrypted before Zerobyte stores them in the database. + +`file://...` references are always resolved from `/run/secrets` and must be a single filename, not a nested path. + +## Access + +- UI/API: `http://:4096` + +## What you'll see in Zerobyte + +- `AWS Production Backups` appears in the repositories list as a managed repository. +- `Team A WebDAV` appears in the volumes list as a managed volume. +- Both resources are read-only in the UI, but can still be used by backup schedules. +- Changes to `provisioning.json`, `.env`, or mounted secret files apply on the next container restart. + +## Rotating or removing provisioned resources + +- Rotate an env-based secret: update `.env`, then restart Zerobyte. +- Rotate a file-based secret: update `secrets/aws_secret_access_key`, then restart Zerobyte. +- Remove a resource: add `delete: true`, then restart Zerobyte. + +## Notes + +- This example keeps `SYS_ADMIN` and `/dev/fuse` enabled because the sample volume uses WebDAV. +- Each provisioned entry must reference an existing `organizationId`. +- Each entry includes both a top-level `backend` and the matching `config.backend`. diff --git a/examples/provisioned-resources/docker-compose.yml b/examples/provisioned-resources/docker-compose.yml new file mode 100644 index 00000000..3b87a2bf --- /dev/null +++ b/examples/provisioned-resources/docker-compose.yml @@ -0,0 +1,28 @@ +services: + zerobyte: + image: ghcr.io/nicotsx/zerobyte:latest + container_name: zerobyte + restart: unless-stopped + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse:/dev/fuse + ports: + - "4096:4096" + environment: + - TZ=${TZ:-UTC} + - BASE_URL=${BASE_URL:?BASE_URL must be set} + - APP_SECRET=${APP_SECRET:?APP_SECRET must be set} + - PROVISIONING_PATH=/config/provisioning.json + - ZEROBYTE_AWS_ACCESS_KEY_ID=${ZEROBYTE_AWS_ACCESS_KEY_ID:?ZEROBYTE_AWS_ACCESS_KEY_ID must be set} + - ZEROBYTE_WEBDAV_PASSWORD=${ZEROBYTE_WEBDAV_PASSWORD:?ZEROBYTE_WEBDAV_PASSWORD must be set} + volumes: + - /etc/localtime:/etc/localtime:ro + - /var/lib/zerobyte:/var/lib/zerobyte + - ./provisioning.json:/config/provisioning.json:ro + secrets: + - aws_secret_access_key + +secrets: + aws_secret_access_key: + file: ./secrets/aws_secret_access_key diff --git a/examples/provisioned-resources/provisioning.example.json b/examples/provisioned-resources/provisioning.example.json new file mode 100644 index 00000000..84d88479 --- /dev/null +++ b/examples/provisioned-resources/provisioning.example.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "repositories": [ + { + "id": "aws-prod", + "organizationId": "replace-with-existing-organization-id", + "name": "AWS Production Backups", + "backend": "s3", + "compressionMode": "auto", + "config": { + "backend": "s3", + "endpoint": "https://s3.amazonaws.com", + "bucket": "company-backups", + "accessKeyId": "env://ZEROBYTE_AWS_ACCESS_KEY_ID", + "secretAccessKey": "file://aws_secret_access_key" + } + } + ], + "volumes": [ + { + "id": "webdav-team-a", + "organizationId": "replace-with-existing-organization-id", + "name": "Team A WebDAV", + "backend": "webdav", + "config": { + "backend": "webdav", + "server": "backup.internal.example.com", + "path": "/team-a", + "username": "team-a", + "password": "env://ZEROBYTE_WEBDAV_PASSWORD", + "port": 443, + "ssl": true + } + } + ] +} diff --git a/examples/provisioned-resources/secrets/aws_secret_access_key.example b/examples/provisioned-resources/secrets/aws_secret_access_key.example new file mode 100644 index 00000000..ee2f5403 --- /dev/null +++ b/examples/provisioned-resources/secrets/aws_secret_access_key.example @@ -0,0 +1 @@ +replace-with-your-s3-secret-access-key diff --git a/examples/secrets-placeholders/.env.example b/examples/secrets-placeholders/.env.example deleted file mode 100644 index e1ed0423..00000000 --- a/examples/secrets-placeholders/.env.example +++ /dev/null @@ -1,6 +0,0 @@ -# Timezone used by the container -TZ=UTC - -# Optional example secret injected via environment variable. -# Use it in Zerobyte config fields as: env://ZEROBYTE_SMB_PASSWORD -ZEROBYTE_SMB_PASSWORD= diff --git a/examples/secrets-placeholders/.gitignore b/examples/secrets-placeholders/.gitignore deleted file mode 100644 index 55d6353f..00000000 --- a/examples/secrets-placeholders/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Secret files - do not commit -smb-password.txt -*-password.txt -*.secret -*.key diff --git a/examples/secrets-placeholders/README.md b/examples/secrets-placeholders/README.md deleted file mode 100644 index dac55772..00000000 --- a/examples/secrets-placeholders/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Secret placeholders (env:// and file://) + Docker secrets - -Zerobyte supports **secret placeholders** in many configuration fields (repositories, volumes, notifications). -Instead of storing raw secrets in the database, you can store a reference that gets resolved at runtime. - -Supported formats: - -- `env://VAR_NAME` → reads `process.env.VAR_NAME` -- `file://name` → reads `/run/secrets/name` (Docker Compose / Docker secrets) - -This example shows how to run Zerobyte with: - -- an environment variable you can reference via `env://...` -- a Docker secret you can reference via `file://...` - -## Prerequisites - -- Docker + Docker Compose - -This example includes `SYS_ADMIN` and `/dev/fuse` because it’s intended for SMB volumes (remote mounts). - -## Setup - -1. Copy the env file: - -```bash -cp .env.example .env -``` - -2. Create a Docker secret file. - - ⚠️ **Important**: never commit real credentials. This folder includes a `.gitignore` to help prevent accidentally committing secret files. - -```bash -printf "your-smb-password" > smb-password.txt -``` - -3. Start Zerobyte: - -```bash -docker compose up -d -``` - -## Using placeholders in Zerobyte - -You can now use the placeholders for example in these Zerobyte configuration fields: - -| UI section | Type | Field | Example value | -| --- | --- | --- | --- | -| Volumes → Create volume | SMB | Password | `file://smb_password` or `env://ZEROBYTE_SMB_PASSWORD` | -| Volumes → Create volume | WebDAV | Password | `file://webdav_password` or `env://ZEROBYTE_WEBDAV_PASSWORD` | -| Repositories → Create repository | S3 | Secret access key | `file://aws_secret_access_key` or `env://AWS_SECRET_ACCESS_KEY` | -| Repositories → Create repository | SFTP | SSH Private key | `file://sftp_private_key` or `env://SFTP_PRIVATE_KEY` | -| Notifications → Create notification | Telegram | Bot token | `file://telegram_bot_token` or `env://TELEGRAM_BOT_TOKEN` | - -Notes: - -- Placeholder names used in these examples are arbitrary; you can choose any valid name. -- Placeholder names are case-sensitive. -- With `file://...`, the secret name must be a single path segment (no `/` or `\\`). -- You can still paste a raw secret, but placeholders can be considered safer and easier to rotate. diff --git a/examples/secrets-placeholders/docker-compose.yml b/examples/secrets-placeholders/docker-compose.yml deleted file mode 100644 index 96f3d86d..00000000 --- a/examples/secrets-placeholders/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -services: - zerobyte: - image: ghcr.io/nicotsx/zerobyte:latest - container_name: zerobyte - restart: unless-stopped - # Required for mounting remote volumes (SMB/NFS/WebDAV) from inside the container - cap_add: - - SYS_ADMIN - devices: - - /dev/fuse:/dev/fuse - ports: - - "4096:4096" - environment: - - TZ=${TZ:-UTC} - # Example env-backed secret (refer to it in Zerobyte as env://ZEROBYTE_SMB_PASSWORD) - - ZEROBYTE_SMB_PASSWORD=${ZEROBYTE_SMB_PASSWORD:-} - secrets: - # Example file-backed secret (refer to it in Zerobyte as file://smb_password) - - smb_password - volumes: - - /etc/localtime:/etc/localtime:ro - - /var/lib/zerobyte:/var/lib/zerobyte - -secrets: - smb_password: - file: ./smb-password.txt