From 16fcec4d407c6759ce09d7f2ff07d5865fb24d31 Mon Sep 17 00:00:00 2001 From: ak2k Date: Fri, 6 Jun 2025 20:27:52 -0400 Subject: [PATCH] feat: add documentation infrastructure and service implementation guide ## Documentation Infrastructure & Service Implementation Guide This PR introduces documentation tooling and contributor resources to help maintain and expand the SelfHostBlocks project: ### Service Implementation Guide - Add comprehensive guide for implementing new SHB services - Covers contracts, testing, documentation, and integration patterns - Provides step-by-step instructions and best practices for contributors ### Redirect Management Tool - Add automated update-redirects tool to maintain documentation links - Extracts heading IDs from markdown files across docs/, modules/, demo/ - Maps source files to target HTML pages using project routing logic - Generates missing redirects in format expected by nixos-render-docs --- docs/generate-redirects-nixos-render-docs.py | 208 +++++++ docs/manual.md | 4 + docs/redirects.json | 419 ++++++++----- docs/service-implementation-guide.md | 611 +++++++++++++++++++ flake.nix | 78 +++ 5 files changed, 1172 insertions(+), 148 deletions(-) create mode 100644 docs/generate-redirects-nixos-render-docs.py create mode 100644 docs/service-implementation-guide.md diff --git a/docs/generate-redirects-nixos-render-docs.py b/docs/generate-redirects-nixos-render-docs.py new file mode 100644 index 0000000..c3333bf --- /dev/null +++ b/docs/generate-redirects-nixos-render-docs.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +Generate redirects.json by scanning actual HTML files produced by nixos-render-docs. + +This script implements a runtime patching mechanism to automatically generate a +complete redirects.json file by scanning generated HTML files for real anchor +locations, eliminating manual maintenance and ensuring accuracy. + +ARCHITECTURE OVERVIEW: +The script works by monkey-patching nixos-render-docs at runtime to: +1. Disable redirect validation during HTML generation +2. Generate HTML documentation normally +3. Scan all generated HTML files to extract anchor IDs and their file locations +4. Apply filtering logic to exclude system-generated anchors +5. Generate and write redirects.json with accurate mappings + +KEY COMPONENTS: +- Runtime patching: Modifies nixos-render-docs behavior without source changes +- HTML scanning: Extracts anchor IDs using regex pattern matching +- Filtering: Excludes NixOS options (opt-*) and extra options (selfhostblock*) +- Output generation: Creates both debug information and production redirects.json + +IMPORTANT NOTES: +- Uses atexit handler to ensure output is generated even if process is interrupted +- Patches are applied on module import, making this a side-effect import +- Error handling preserves original validation behavior in case of failure +""" + +import sys +import json +import atexit +import os +import re + +# Global storage for anchor-to-file mappings discovered during HTML scanning +# Structure: {anchor_id: html_filename} +file_target_mapping = {} + +def scan_html_files(output_dir, html_files): + """ + Scan HTML files to extract anchor IDs and build anchor-to-file mappings. + + Discovers all HTML files in output_dir and extracts id attributes to populate + the global file_target_mapping. Filters out NixOS system options during scanning. + + Args: + output_dir: Directory containing generated HTML files + html_files: Unused parameter (always discovers files from filesystem) + """ + # Always discover HTML files from the output directory + if not os.path.exists(output_dir): + print(f"DEBUG: Output directory {output_dir} does not exist", file=sys.stderr) + return + + html_files = [f for f in os.listdir(output_dir) if f.endswith('.html')] + print(f"DEBUG: Discovered {len(html_files)} HTML files in {output_dir}", file=sys.stderr) + + # Process each HTML file to extract anchor IDs + for html_file in html_files: + html_path = os.path.join(output_dir, html_file) + try: + with open(html_path, 'r', encoding='utf-8') as f: + html_content = f.read() + + # Extract all id attributes using regex pattern matching + # Matches: id="anchor-name" and captures anchor-name + anchor_matches = re.findall(r'id="([^"]+)"', html_content) + + # Filter and record anchor mappings + non_opt_count = 0 + for anchor_id in anchor_matches: + # Skip NixOS system option anchors (opt-* prefix) + if not anchor_id.startswith('opt-'): + file_target_mapping[anchor_id] = html_file + non_opt_count += 1 + + if non_opt_count > 0: + print(f"Found {non_opt_count} anchors in {html_file}", file=sys.stderr) + + except Exception as e: + # Log errors but continue processing other files + print(f"Failed to scan {html_path}: {e}", file=sys.stderr) + +def output_collected_refs(): + """ + Generate and write the final redirects.json file from collected anchor mappings. + + This function is registered as an atexit handler to ensure output is generated + even if the process is interrupted. It processes the global file_target_mapping + to create the final redirects file with appropriate filtering. + + Output files: + - out/redirects.json: Production redirects mapping + """ + import os + + # Generate redirects from discovered HTML anchor mappings + if file_target_mapping: + print(f"Creating redirects from {len(file_target_mapping)} HTML mappings", file=sys.stderr) + redirects = {} + filtered_count = 0 + + # Apply filtering logic to exclude system-generated anchors + for anchor_id, html_file in file_target_mapping.items(): + # Filter out: + # - opt-*: NixOS system options + # - selfhostblock*: Extra options from this project + if not anchor_id.startswith('opt-') and not anchor_id.startswith('selfhostblock'): + redirects[anchor_id] = [f"{html_file}#{anchor_id}"] + else: + filtered_count += 1 + + print(f"Generated {len(redirects)} redirects (filtered out {filtered_count} system options)", file=sys.stderr) + else: + # Fallback case - should not occur during normal operation + print("Warning: No HTML mappings available", file=sys.stderr) + redirects = {} + + # Ensure output directory exists + os.makedirs('out', exist_ok=True) + + # Write production redirects file + try: + redirects_file = 'out/redirects.json' + + with open(redirects_file, 'w') as f: + json.dump(redirects, f, indent=2, sort_keys=True) + + print(f"Generated redirects.json with {len(redirects)} redirects", file=sys.stderr) + + except Exception as e: + print(f"Failed to write redirects.json: {e}", file=sys.stderr) + +# Register output generation to run on process exit +atexit.register(output_collected_refs) + +def apply_patches(): + """ + Apply runtime monkey patches to nixos-render-docs modules. + + This function modifies the behavior of nixos-render-docs by: + 1. Hooking into the HTML generation CLI command + 2. Temporarily disabling redirect validation during HTML generation + 3. Scanning generated HTML files to extract anchor mappings + 4. Restoring original validation behavior + + The patching approach allows us to extract anchor information without + modifying the nixos-render-docs source code directly. + + Raises: + ImportError: If nixos-render-docs modules cannot be imported + """ + try: + # Import required nixos-render-docs modules + import nixos_render_docs.html as html_module + import nixos_render_docs.redirects as redirects_module + import nixos_render_docs.manual as manual_module + + # Store reference to original HTML CLI function + original_run_cli_html = manual_module._run_cli_html + + def patched_run_cli_html(args): + """ + Patched version of _run_cli_html that disables validation and scans output. + + This wrapper function: + 1. Temporarily disables redirect validation to prevent errors + 2. Runs normal HTML generation + 3. Scans generated HTML files for anchor mappings + 4. Restores original validation behavior + """ + print("Generating HTML documentation...", file=sys.stderr) + + # Temporarily disable redirect validation + original_validate = redirects_module.Redirects.validate + redirects_module.Redirects.validate = lambda self, targets: None + + try: + # Run original HTML generation + result = original_run_cli_html(args) + + # Determine output directory from CLI arguments + if hasattr(args, 'outfile') and args.outfile: + output_dir = os.path.dirname(args.outfile) + else: + output_dir = '.' + + # Scan generated HTML files for anchor mappings + scan_html_files(output_dir, None) + print(f"Scanned {len(file_target_mapping)} anchor mappings", file=sys.stderr) + + finally: + # Always restore original validation function + redirects_module.Redirects.validate = original_validate + + return result + + # Replace the original function with our patched version + manual_module._run_cli_html = patched_run_cli_html + + print("Applied patches to nixos-render-docs", file=sys.stderr) + + except ImportError as e: + print(f"Failed to apply patches: {e}", file=sys.stderr) + +# Apply patches immediately when this module is imported +# This ensures the patches are active before nixos-render-docs CLI runs +apply_patches() \ No newline at end of file diff --git a/docs/manual.md b/docs/manual.md index 060bf09..a2bbfef 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -32,6 +32,10 @@ demos.md contributing.md ``` +```{=include=} chapters html:into-file=//service-implementation-guide.html +service-implementation-guide.md +``` + ```{=include=} appendix html:into-file=//options.html options.md ``` diff --git a/docs/redirects.json b/docs/redirects.json index 7e7dd82..9db5e93 100644 --- a/docs/redirects.json +++ b/docs/redirects.json @@ -1,7 +1,49 @@ { + "adding-new-service-documentation": [ + "service-implementation-guide.html#adding-new-service-documentation" + ], "all-options": [ "options.html#all-options" ], + "analyze-existing-services": [ + "service-implementation-guide.html#analyze-existing-services" + ], + "api-health-check": [ + "service-implementation-guide.html#api-health-check" + ], + "authentication-integration": [ + "service-implementation-guide.html#authentication-integration" + ], + "authentication-integration-pitfalls": [ + "service-implementation-guide.html#authentication-integration-pitfalls" + ], + "automated-redirect-generation": [ + "service-implementation-guide.html#automated-redirect-generation" + ], + "best-practices-summary": [ + "service-implementation-guide.html#best-practices-summary" + ], + "block-ssl": [ + "blocks-ssl.html#block-ssl" + ], + "block-ssl-debug": [ + "blocks-ssl.html#block-ssl-debug" + ], + "block-ssl-impl-lets-encrypt": [ + "blocks-ssl.html#block-ssl-impl-lets-encrypt" + ], + "block-ssl-impl-self-signed": [ + "blocks-ssl.html#block-ssl-impl-self-signed" + ], + "block-ssl-options": [ + "blocks-ssl.html#block-ssl-options" + ], + "block-ssl-tests": [ + "blocks-ssl.html#block-ssl-tests" + ], + "block-ssl-usage": [ + "blocks-ssl.html#block-ssl-usage" + ], "blocks": [ "blocks.html#blocks" ], @@ -14,20 +56,14 @@ "blocks-monitoring-configuration": [ "blocks-monitoring.html#blocks-monitoring-configuration" ], - "blocks-monitoring-nextcloud-dashboard": [ - "blocks-monitoring.html#blocks-monitoring-nextcloud-dashboard" - ], "blocks-monitoring-deluge-dashboard": [ "blocks-monitoring.html#blocks-monitoring-deluge-dashboard" ], "blocks-monitoring-error-dashboard": [ "blocks-monitoring.html#blocks-monitoring-error-dashboard" ], - "blocks-monitoring-performance-dashboard": [ - "blocks-monitoring.html#blocks-monitoring-performance-dashboard" - ], - "blocks-monitoring-provisioning": [ - "blocks-monitoring.html#blocks-monitoring-provisioning" + "blocks-monitoring-nextcloud-dashboard": [ + "blocks-monitoring.html#blocks-monitoring-nextcloud-dashboard" ], "blocks-monitoring-options": [ "blocks-monitoring.html#blocks-monitoring-options" @@ -149,6 +185,12 @@ "blocks-monitoring-options-shb.monitoring.subdomain": [ "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.subdomain" ], + "blocks-monitoring-performance-dashboard": [ + "blocks-monitoring.html#blocks-monitoring-performance-dashboard" + ], + "blocks-monitoring-provisioning": [ + "blocks-monitoring.html#blocks-monitoring-provisioning" + ], "blocks-postgresql": [ "blocks-postgresql.html#blocks-postgresql" ], @@ -473,32 +515,11 @@ "blocks-sops-usage": [ "blocks-sops.html#blocks-sops-usage" ], - "blocks-sops-usage-requester": [ - "blocks-sops.html#blocks-sops-usage-requester" - ], "blocks-sops-usage-manual": [ "blocks-sops.html#blocks-sops-usage-manual" ], - "block-ssl": [ - "blocks-ssl.html#block-ssl" - ], - "block-ssl-debug": [ - "blocks-ssl.html#block-ssl-debug" - ], - "block-ssl-impl-lets-encrypt": [ - "blocks-ssl.html#block-ssl-impl-lets-encrypt" - ], - "block-ssl-impl-self-signed": [ - "blocks-ssl.html#block-ssl-impl-self-signed" - ], - "block-ssl-options": [ - "blocks-ssl.html#block-ssl-options" - ], - "block-ssl-tests": [ - "blocks-ssl.html#block-ssl-tests" - ], - "block-ssl-usage": [ - "blocks-ssl.html#block-ssl-usage" + "blocks-sops-usage-requester": [ + "blocks-sops.html#blocks-sops-usage-requester" ], "blocks-ssl-options-shb.certs.cas.selfsigned": [ "blocks-ssl.html#blocks-ssl-options-shb.certs.cas.selfsigned" @@ -617,6 +638,27 @@ "blocks-ssl-options-shb.certs.systemdService": [ "blocks-ssl.html#blocks-ssl-options-shb.certs.systemdService" ], + "build-time-validation": [ + "service-implementation-guide.html#build-time-validation" + ], + "check-nixos-integration": [ + "service-implementation-guide.html#check-nixos-integration" + ], + "common-pitfalls-and-solutions": [ + "service-implementation-guide.html#common-pitfalls-and-solutions" + ], + "complete-shb-service": [ + "service-implementation-guide.html#complete-shb-service" + ], + "complete-workflow": [ + "service-implementation-guide.html#complete-workflow" + ], + "configuration-issues": [ + "service-implementation-guide.html#configuration-issues" + ], + "configuration-management": [ + "service-implementation-guide.html#configuration-management" + ], "contract-backup": [ "contracts-backup.html#contract-backup" ], @@ -686,27 +728,6 @@ "contracts": [ "contracts.html#contracts" ], - "contracts-nixpkgs": [ - "contracts.html#contracts-nixpkgs" - ], - "contracts-provided": [ - "contracts.html#contracts-provided" - ], - "contracts-concept": [ - "contracts.html#contracts-concept" - ], - "contracts-schema": [ - "contracts.html#contracts-schema" - ], - "contracts-test": [ - "contracts.html#contracts-test" - ], - "contracts-videos": [ - "contracts.html#contracts-videos" - ], - "contracts-why": [ - "contracts.html#contracts-why" - ], "contracts-backup-options-shb.contracts.backup": [ "contracts-backup.html#contracts-backup-options-shb.contracts.backup" ], @@ -743,6 +764,9 @@ "contracts-backup-options-shb.contracts.backup.settings": [ "contracts-backup.html#contracts-backup-options-shb.contracts.backup.settings" ], + "contracts-concept": [ + "contracts.html#contracts-concept" + ], "contracts-databasebackup-options-shb.contracts.databasebackup": [ "contracts-databasebackup.html#contracts-databasebackup-options-shb.contracts.databasebackup" ], @@ -773,6 +797,15 @@ "contracts-databasebackup-options-shb.contracts.databasebackup.settings": [ "contracts-databasebackup.html#contracts-databasebackup-options-shb.contracts.databasebackup.settings" ], + "contracts-nixpkgs": [ + "contracts.html#contracts-nixpkgs" + ], + "contracts-provided": [ + "contracts.html#contracts-provided" + ], + "contracts-schema": [ + "contracts.html#contracts-schema" + ], "contracts-secret-options-shb.contracts.secret": [ "contracts-secret.html#contracts-secret-options-shb.contracts.secret" ], @@ -815,6 +848,15 @@ "contracts-ssl-options-shb.contracts.ssl.systemdService": [ "contracts-ssl.html#contracts-ssl-options-shb.contracts.ssl.systemdService" ], + "contracts-test": [ + "contracts.html#contracts-test" + ], + "contracts-videos": [ + "contracts.html#contracts-videos" + ], + "contracts-why": [ + "contracts.html#contracts-why" + ], "contributing": [ "contributing.html#contributing" ], @@ -857,6 +899,15 @@ "contributing-upstream": [ "contributing.html#contributing-upstream" ], + "create-comprehensive-tests": [ + "service-implementation-guide.html#create-comprehensive-tests" + ], + "create-service-documentation": [ + "service-implementation-guide.html#create-service-documentation" + ], + "create-service-module": [ + "service-implementation-guide.html#create-service-module" + ], "demo-homeassistant": [ "demo-homeassistant.html#demo-homeassistant" ], @@ -866,6 +917,9 @@ "demo-homeassistant-deploy-basic": [ "demo-homeassistant.html#demo-homeassistant-deploy-basic" ], + "demo-homeassistant-deploy-colmena": [ + "demo-homeassistant.html#demo-homeassistant-deploy-colmena" + ], "demo-homeassistant-deploy-ldap": [ "demo-homeassistant.html#demo-homeassistant-deploy-ldap" ], @@ -896,9 +950,6 @@ "demo-homeassistant-virtual-machine": [ "demo-homeassistant.html#demo-homeassistant-virtual-machine" ], - "demo-homeassistant-deploy-colmena": [ - "demo-homeassistant.html#demo-homeassistant-deploy-colmena" - ], "demo-nextcloud": [ "demo-nextcloud.html#demo-nextcloud" ], @@ -947,12 +998,69 @@ "demos": [ "demos.html#demos" ], + "external-exporter": [ + "service-implementation-guide.html#external-exporter" + ], + "handle-unfree-dependencies": [ + "service-implementation-guide.html#handle-unfree-dependencies" + ], + "how-redirects-work": [ + "service-implementation-guide.html#how-redirects-work" + ], + "implementation-considerations": [ + "service-implementation-guide.html#implementation-considerations" + ], + "implementation-steps": [ + "service-implementation-guide.html#implementation-steps" + ], + "iterative-development-approach": [ + "service-implementation-guide.html#iterative-development-approach" + ], + "local-testing": [ + "service-implementation-guide.html#local-testing" + ], + "monitoring-failures": [ + "service-implementation-guide.html#monitoring-failures" + ], + "monitoring-implementation": [ + "service-implementation-guide.html#monitoring-implementation" + ], + "native-prometheus-metrics": [ + "service-implementation-guide.html#native-prometheus-metrics" + ], + "nixpkgs-integration": [ + "service-implementation-guide.html#nixpkgs-integration" + ], + "pre-implementation-research": [ + "service-implementation-guide.html#pre-implementation-research" + ], "preface": [ "index.html#preface" ], + "quick-reference": [ + "service-implementation-guide.html#quick-reference" + ], + "redirect-management": [ + "service-implementation-guide.html#redirect-management" + ], + "redirect-patterns": [ + "service-implementation-guide.html#redirect-patterns" + ], + "required-test-variants": [ + "service-implementation-guide.html#required-test-variants" + ], + "resources": [ + "service-implementation-guide.html#resources" + ], + "security-best-practices": [ + "service-implementation-guide.html#security-best-practices" + ], "self-host-blocks-manual": [ "index.html#self-host-blocks-manual" ], + "service-implementation-guide": [ + "service-implementation-guide.html#service-implementation-guide" + ], "services": [ "services.html#services" ], @@ -968,39 +1076,6 @@ "services-forgejo-options": [ "services-forgejo.html#services-forgejo-options" ], - "services-forgejo-options-shb.forgejo.users": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users" - ], - "services-forgejo-options-shb.forgejo.users._name_.email": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.email" - ], - "services-forgejo-options-shb.forgejo.users._name_.isAdmin": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.isAdmin" - ], - "services-forgejo-options-shb.forgejo.users._name_.password": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.request": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.request.group": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.group" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.request.mode": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.mode" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.request.owner": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.owner" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.request.restartUnits": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.restartUnits" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.result": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.result" - ], - "services-forgejo-options-shb.forgejo.users._name_.password.result.path": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.result.path" - ], "services-forgejo-options-shb.forgejo.backup": [ "services-forgejo.html#services-forgejo-options-shb.forgejo.backup" ], @@ -1187,6 +1262,27 @@ "services-forgejo-options-shb.forgejo.sso.sharedSecret": [ "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret" ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.request": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request" + ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.group": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.group" + ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.mode": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.mode" + ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.owner": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.owner" + ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.restartUnits": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.restartUnits" + ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.result": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.result" + ], + "services-forgejo-options-shb.forgejo.sso.sharedSecret.result.path": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.result.path" + ], "services-forgejo-options-shb.forgejo.sso.sharedSecretForAuthelia": [ "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecretForAuthelia" ], @@ -1211,30 +1307,42 @@ "services-forgejo-options-shb.forgejo.sso.sharedSecretForAuthelia.result.path": [ "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecretForAuthelia.result.path" ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.request": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request" - ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.group": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.group" - ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.mode": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.mode" - ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.owner": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.owner" - ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.request.restartUnits": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.request.restartUnits" - ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.result": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.result" - ], - "services-forgejo-options-shb.forgejo.sso.sharedSecret.result.path": [ - "services-forgejo.html#services-forgejo-options-shb.forgejo.sso.sharedSecret.result.path" - ], "services-forgejo-options-shb.forgejo.subdomain": [ "services-forgejo.html#services-forgejo-options-shb.forgejo.subdomain" ], + "services-forgejo-options-shb.forgejo.users": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users" + ], + "services-forgejo-options-shb.forgejo.users._name_.email": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.email" + ], + "services-forgejo-options-shb.forgejo.users._name_.isAdmin": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.isAdmin" + ], + "services-forgejo-options-shb.forgejo.users._name_.password": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.request": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.request.group": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.group" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.request.mode": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.mode" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.request.owner": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.owner" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.request.restartUnits": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.request.restartUnits" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.result": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.result" + ], + "services-forgejo-options-shb.forgejo.users._name_.password.result.path": [ + "services-forgejo.html#services-forgejo-options-shb.forgejo.users._name_.password.result.path" + ], "services-forgejo-usage": [ "services-forgejo.html#services-forgejo-usage" ], @@ -1244,12 +1352,12 @@ "services-forgejo-usage-configuration": [ "services-forgejo.html#services-forgejo-usage-configuration" ], - "services-forgejo-usage-https": [ - "services-forgejo.html#services-forgejo-usage-https" - ], "services-forgejo-usage-extra-settings": [ "services-forgejo.html#services-forgejo-usage-extra-settings" ], + "services-forgejo-usage-https": [ + "services-forgejo.html#services-forgejo-usage-https" + ], "services-forgejo-usage-ldap": [ "services-forgejo.html#services-forgejo-usage-ldap" ], @@ -1259,6 +1367,9 @@ "services-nextcloudserver": [ "services-nextcloud.html#services-nextcloudserver" ], + "services-nextcloudserver-dashboard": [ + "services-nextcloud.html#services-nextcloudserver-dashboard" + ], "services-nextcloudserver-debug": [ "services-nextcloud.html#services-nextcloudserver-debug" ], @@ -1268,9 +1379,6 @@ "services-nextcloudserver-features": [ "services-nextcloud.html#services-nextcloudserver-features" ], - "services-nextcloudserver-dashboard": [ - "services-nextcloud.html#services-nextcloudserver-dashboard" - ], "services-nextcloudserver-options": [ "services-nextcloud.html#services-nextcloudserver-options" ], @@ -1373,20 +1481,14 @@ "services-nextcloudserver-options-shb.nextcloud.apps.memories": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.memories" ], - "services-nextcloudserver-options-shb.nextcloud.apps.memories.vaapi": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.memories.vaapi" - ], "services-nextcloudserver-options-shb.nextcloud.apps.memories.enable": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.memories.enable" ], "services-nextcloudserver-options-shb.nextcloud.apps.memories.photosPath": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.memories.photosPath" ], - "services-nextcloudserver-options-shb.nextcloud.apps.recognize": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.recognize" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.recognize.enable": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.recognize.enable" + "services-nextcloudserver-options-shb.nextcloud.apps.memories.vaapi": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.memories.vaapi" ], "services-nextcloudserver-options-shb.nextcloud.apps.onlyoffice": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.onlyoffice" @@ -1430,6 +1532,12 @@ "services-nextcloudserver-options-shb.nextcloud.apps.previewgenerator.recommendedSettings": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.previewgenerator.recommendedSettings" ], + "services-nextcloudserver-options-shb.nextcloud.apps.recognize": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.recognize" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.recognize.enable": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.recognize.enable" + ], "services-nextcloudserver-options-shb.nextcloud.apps.sso": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso" ], @@ -1457,6 +1565,27 @@ "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret" ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.group": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.group" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.mode": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.mode" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.owner": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.owner" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.restartUnits": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.restartUnits" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result" + ], + "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result.path": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result.path" + ], "services-nextcloudserver-options-shb.nextcloud.apps.sso.secretForAuthelia": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secretForAuthelia" ], @@ -1481,27 +1610,6 @@ "services-nextcloudserver-options-shb.nextcloud.apps.sso.secretForAuthelia.result.path": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secretForAuthelia.result.path" ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.group": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.group" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.mode": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.mode" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.owner": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.owner" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.restartUnits": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.request.restartUnits" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result" - ], - "services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result.path": [ - "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.apps.sso.secret.result.path" - ], "services-nextcloudserver-options-shb.nextcloud.autoDisableMaintenanceModeOnStart": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.autoDisableMaintenanceModeOnStart" ], @@ -1835,6 +1943,21 @@ "services-vaultwarden-zfs": [ "services-vaultwarden.html#services-vaultwarden-zfs" ], + "test-failures": [ + "service-implementation-guide.html#test-failures" + ], + "testing-and-validation": [ + "service-implementation-guide.html#testing-and-validation" + ], + "understand-target-service": [ + "service-implementation-guide.html#understand-target-service" + ], + "update-flake-configuration": [ + "service-implementation-guide.html#update-flake-configuration" + ], + "update-redirects-automatically": [ + "service-implementation-guide.html#update-redirects-automatically" + ], "usage": [ "usage.html#usage" ], @@ -1850,9 +1973,6 @@ "usage-example-nixosrebuild": [ "usage.html#usage-example-nixosrebuild" ], - "usage-lib": [ - "usage.html#usage-lib" - ], "usage-flake": [ "usage.html#usage-flake" ], @@ -1868,7 +1988,10 @@ "usage-flake-tag": [ "usage.html#usage-flake-tag" ], + "usage-lib": [ + "usage.html#usage-lib" + ], "usage-secrets": [ "usage.html#usage-secrets" ] -} +} \ No newline at end of file diff --git a/docs/service-implementation-guide.md b/docs/service-implementation-guide.md new file mode 100644 index 0000000..69e3031 --- /dev/null +++ b/docs/service-implementation-guide.md @@ -0,0 +1,611 @@ +# SelfHostBlocks Service Implementation Guide {#service-implementation-guide} + +This guide documents the complete process for implementing a new service in SelfHostBlocks, based on lessons learned from the nzbget implementation and analysis of existing service patterns. + +**Note**: SelfHostBlocks aims to be "the smallest amount of code above what is available in nixpkgs" (see `docs/contributing.md`). Services should leverage existing nixpkgs options when possible and focus on providing contract integrations rather than reimplementing configuration. + +## What Makes a "Complete" SHB Service {#complete-shb-service} + +According to the project maintainer's criteria, a service is considered fully supported if it includes: + +1. **SSL block integration** - HTTPS/TLS certificate management +2. **Backup block integration** - Automated backup of service data +3. **Monitoring integration** - Prometheus metrics and health checks +4. **LDAP (LLDAP) integration** - Directory-based authentication +5. **SSO (Authelia) integration** - Single sign-on authentication +6. **Comprehensive tests** - All integration variants tested + +## Pre-Implementation Research {#pre-implementation-research} + +### 1. Analyze Existing Services {#analyze-existing-services} + +Before starting, study existing services to understand patterns: + +```bash +# Study service patterns +ls modules/services/ # List all services +cat modules/services/deluge.nix # Best practice example +cat modules/services/vaultwarden.nix # Another good example +``` + +**Key patterns to identify:** +- Configuration structure and options +- How contracts are used (SSL, backup, monitoring, secrets) +- Authentication integration approaches +- Service-specific settings and defaults + +### 2. Understand the Target Service {#understand-target-service} + +Research the service you're implementing: +- **Configuration format** (YAML, INI, JSON, etc.) +- **Authentication methods** (built-in users, LDAP, OIDC/OAuth) +- **API endpoints** (for monitoring/health checks) +- **Data directories** (what needs backing up) +- **Network requirements** (ports, protocols) +- **Dependencies** (databases, external tools) + +### 3. Check NixOS Integration {#check-nixos-integration} + +Verify nixpkgs support: +```bash +# Check if NixOS service exists +nix eval --impure --expr '(import { configuration = {...}: {}; }).options.services' --apply 'builtins.attrNames' --json | jq -r '.[]' | grep -i servicename +# or search online: https://search.nixos.org/options?query=services.servicename +``` + +If no nixpkgs integration exists, you may need to: +- Package the service first +- Use containerized approach +- Request upstream nixpkgs integration + +## Implementation Steps {#implementation-steps} + +### 1. Create the Service Module {#create-service-module} + +Location: `modules/services/servicename.nix` + +**Basic structure:** +```nix +{ config, pkgs, lib, ... }: + +let + cfg = config.shb.servicename; + contracts = pkgs.callPackage ../contracts {}; + shblib = pkgs.callPackage ../../lib {}; + fqdn = "${cfg.subdomain}.${cfg.domain}"; + + # Choose appropriate format based on service config + settingsFormat = pkgs.formats.yaml {}; # or .ini, .json, etc. +in +{ + options.shb.servicename = { + # Core options (always required) + enable = lib.mkEnableOption "selfhostblocks.servicename"; + subdomain = lib.mkOption { ... }; + domain = lib.mkOption { ... }; + + # SSL integration (always include) + ssl = lib.mkOption { + description = "Path to SSL files"; + type = lib.types.nullOr contracts.ssl.certs; + default = null; + }; + + # Service-specific options + port = lib.mkOption { ... }; + dataDir = lib.mkOption { ... }; + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = settingsFormat.type; + options = { + # Define key options with descriptions + }; + }; + }; + + # Authentication options + authEndpoint = lib.mkOption { + type = lib.types.nullOr lib.types.str; + description = "OIDC endpoint for SSO"; + default = null; + }; + + ldap = lib.mkOption { ... }; # LDAP integration + users = lib.mkOption { ... }; # Local user management + + # Integration options + backup = lib.mkOption { + type = lib.types.submodule { + options = contracts.backup.mkRequester { + user = "servicename"; + sourceDirectories = [ cfg.dataDir ]; + }; + }; + }; + + monitoring = lib.mkOption { + type = lib.types.nullOr (lib.types.submodule { + options = { + # Service-specific monitoring options + }; + }); + default = null; + }; + + # System options + extraServiceConfig = lib.mkOption { ... }; + logLevel = lib.mkOption { ... }; + }; + + config = lib.mkIf cfg.enable (lib.mkMerge [ + { + # Base service configuration + services.servicename = { + enable = true; + # Map SHB options to nixpkgs service options + }; + + # Nginx reverse proxy + shb.nginx.vhosts = [{ + inherit (cfg) subdomain domain ssl; + upstream = "http://127.0.0.1:${toString cfg.port}"; + + # SSO integration + autheliaRules = lib.mkIf (cfg.authEndpoint != null) [ + { + domain = fqdn; + policy = "bypass"; + resources = [ "^/api" ]; # API endpoints + } + { + domain = fqdn; + policy = "two_factor"; + resources = [ "^.*" ]; # Everything else + } + ]; + }]; + + # User/group setup + users.users.servicename = { + extraGroups = [ "media" ]; # If needed for file access + }; + + # Directory permissions + systemd.tmpfiles.rules = [ + "d ${cfg.dataDir} 0755 servicename servicename - -" + ]; + } + + # Monitoring configuration (conditional) + (lib.mkIf (cfg.monitoring != null) { + services.prometheus.scrapeConfigs = [{ + job_name = "servicename"; + static_configs = [{ + targets = [ "127.0.0.1:${toString cfg.port}" ]; + labels = { + hostname = config.networking.hostName; + domain = cfg.domain; + }; + }]; + metrics_path = "/metrics"; # or appropriate endpoint + scrape_interval = "30s"; + }]; + }) + ]); +} +``` + +### 2. Key Implementation Considerations {#implementation-considerations} + +#### Configuration Management {#configuration-management} +- **Use freeform settings** when possible: `freeformType = settingsFormat.type` +- **Provide sensible defaults** for common options +- **Use lib.mkDefault** for user-overridable settings +- **Use lib.mkForce** for security-critical settings + +#### Authentication Integration {#authentication-integration} +- **SSO (Authelia)**: Use `autheliaRules` with appropriate bypass policies +- **LDAP**: Follow the patterns from existing services +- **Local users**: Use SHB secret contracts for password management + +#### Security Best Practices {#security-best-practices} +- **Bind to localhost**: Services should listen on `127.0.0.1` only +- **Use nginx for TLS**: Don't configure TLS in the service itself +- **Proper file permissions**: Use systemd.tmpfiles.rules +- **Secret management**: Always use SHB secret contracts + +### 3. Monitoring Implementation {#monitoring-implementation} + +Choose the appropriate monitoring approach: + +#### Option A: Native Prometheus Metrics {#native-prometheus-metrics} +If the service supports Prometheus natively: +```nix +services.prometheus.scrapeConfigs = [{ + job_name = "servicename"; + static_configs = [{ targets = [ "127.0.0.1:${toString cfg.port}" ]; }]; + metrics_path = "/metrics"; +}]; +``` + +#### Option B: API Health Check {#api-health-check} +If no native metrics, monitor API endpoints: +```nix +services.prometheus.scrapeConfigs = [{ + job_name = "servicename"; + static_configs = [{ targets = [ "127.0.0.1:${toString cfg.port}" ]; }]; + metrics_path = "/api/status"; # or appropriate endpoint +}]; +``` + +#### Option C: External Exporter {#external-exporter} +For services requiring dedicated exporters (like Deluge): +```nix +services.prometheus.exporters.servicename = { + enable = true; + # exporter-specific configuration +}; +``` + +### 4. Create Comprehensive Tests {#create-comprehensive-tests} + +Location: `test/services/servicename.nix` + +**Test structure:** +```nix +{ pkgs, ... }: +let + testLib = pkgs.callPackage ../common.nix {}; + + # Common test scripts + commonTestScript = testLib.mkScripts { + hasSSL = { node, ... }: !(isNull node.config.shb.servicename.ssl); + waitForServices = { ... }: [ "nginx.service" "servicename.service" ]; + waitForPorts = { node, ... }: [ node.config.services.servicename.port ]; + + # Service-specific connectivity test + extraScript = { node, proto_fqdn, ... }: '' + with subtest("service connectivity"): + response = curl(client, "", "${proto_fqdn}/api/health") + # Add service-specific checks + ''; + }; + + # Monitoring test script + prometheusTestScript = { nodes, ... }: '' + server.wait_for_open_port(${toString nodes.server.config.services.servicename.port}) + with subtest("prometheus monitoring"): + # Test the actual monitoring endpoint + response = server.succeed("curl -sSf http://localhost:${port}/metrics") + # Validate response format + ''; + + # Base configuration + basic = { config, ... }: { + imports = [ + testLib.baseModule + ../../modules/services/servicename.nix + ]; + + shb.servicename = { + enable = true; + inherit (config.test) domain subdomain; + # Basic configuration + }; + }; + +in { + # Test variants (all 6 required) + basic = pkgs.testers.runNixOSTest { ... }; + backup = pkgs.testers.runNixOSTest { ... }; + https = pkgs.testers.runNixOSTest { ... }; + ldap = pkgs.testers.runNixOSTest { ... }; + monitoring = pkgs.testers.runNixOSTest { ... }; + sso = pkgs.testers.runNixOSTest { ... }; +} +``` + +#### Required Test Variants {#required-test-variants} + +1. **basic**: Core functionality without authentication +2. **backup**: Tests backup integration +3. **https**: Tests SSL/TLS integration +4. **ldap**: Tests LDAP authentication +5. **monitoring**: Tests Prometheus integration +6. **sso**: Tests Authelia SSO integration + +### 5. Update Flake Configuration {#update-flake-configuration} + +Add to `flake.nix`: + +```nix +allModules = [ + # ... existing modules + modules/services/servicename.nix +]; +``` + +```nix +checks = { + # ... existing checks + // (vm_test "servicename" ./test/services/servicename.nix) +}; +``` + +### 6. Create Service Documentation {#create-service-documentation} + +Create comprehensive documentation for the new service: + +**Location**: `modules/services/servicename/docs/default.md` + +```markdown +# ServiceName Service {\#services-servicename} + +Brief description of what the service does. + +## Features {\#services-servicename-features} + +- Feature 1 +- Feature 2 + +## Usage {\#services-servicename-usage} + +### Basic Configuration {\#services-servicename-basic} + +shb.servicename = { + enable = true; + domain = "example.com"; + subdomain = "servicename"; +}; + +### SSL Configuration {\#services-servicename-ssl} + +shb.servicename.ssl.paths = { + cert = /path/to/cert; + key = /path/to/key; +}; + +## Options Reference {\#services-servicename-options} + +{=include=} options +id-prefix: services-servicename-options- +list-id: selfhostblocks-servicename-options +source: @OPTIONS_JSON@ +``` + +**Important**: Use consistent heading ID patterns: +- Service overview: `{\#services-servicename}` +- Features: `{\#services-servicename-features}` +- Usage sections: `{\#services-servicename-basic}`, `{\#services-servicename-ssl}`, etc. +- Options: `{\#services-servicename-options}` + +Note: Replace `servicename` with your actual service name (e.g., `nzbget`, `jellyfin`). + +### 7. Update Redirects Automatically {#update-redirects-automatically} + +After creating documentation, generate the required redirects: + +```bash +# Scan documentation and add missing redirects +nix run .#update-redirects + +# Review the changes +git diff docs/redirects.json + +# The tool will show what redirects were added +``` + +The automation will: +- Find all heading IDs in your documentation +- Generate appropriate redirect entries +- Add them to `docs/redirects.json` +- Follow established naming patterns + +### 8. Handle Unfree Dependencies {#handle-unfree-dependencies} + +If the service requires unfree packages: + +```nix +# In flake.nix +config = { + allowUnfree = true; + permittedInsecurePackages = [ + # List any required insecure packages + ]; +}; +``` + +Update CI workflow if needed: +```yaml +# In .github/workflows/build.yaml +- name: Setup Nix + uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + allow-unfree = true +``` + +## Testing and Validation {#testing-and-validation} + +### Local Testing {#local-testing} +```bash +# Test redirect automation +nix run .#update-redirects + +# Test all service variants (replace ${system} with your system, e.g., x86_64-linux) +nix build .#checks.${system}.vm_servicename_basic +nix build .#checks.${system}.vm_servicename_backup +nix build .#checks.${system}.vm_servicename_https +nix build .#checks.${system}.vm_servicename_ldap +nix build .#checks.${system}.vm_servicename_monitoring +nix build .#checks.${system}.vm_servicename_sso + +# Or run all tests (as recommended in docs/contributing.md) +nix flake check + +# For interactive testing and debugging, see docs/contributing.md: +# nix run .#checks.${system}.vm_servicename_basic.driverInteractive + +# Test documentation build (includes redirect validation) +nix build .#manualHtml +``` + +### Iterative Development Approach {#iterative-development-approach} + +1. **Start with basic functionality** - get core service working +2. **Add SSL integration** - enable HTTPS +3. **Add backup integration** - ensure data protection +4. **Add monitoring** - implement health checks +5. **Add authentication** - LDAP and SSO integration +6. **Create documentation** - write service documentation with heading IDs +7. **Update redirects** - run `nix run .#update-redirects` to generate redirects +8. **Comprehensive testing** - all 6 test variants +9. **Final validation** - ensure documentation builds correctly + +## Common Pitfalls and Solutions {#common-pitfalls-and-solutions} + +### Configuration Issues {#configuration-issues} +- **Problem**: Service doesn't start due to config validation +- **Solution**: Use `lib.mkDefault` for user settings, `lib.mkForce` for security settings + +### Authentication Integration {#authentication-integration-pitfalls} +- **Problem**: SSO redirect loops or access denied +- **Solution**: Check `autheliaRules` bypass patterns for API endpoints + +### Monitoring Failures {#monitoring-failures} +- **Problem**: Prometheus scraping fails with 404 +- **Solution**: Verify the actual API endpoints the service provides + +### Test Failures {#test-failures} +- **Problem**: VM tests timeout or fail connectivity +- **Solution**: Check `waitForServices` and `waitForPorts` configurations + +### Nixpkgs Integration {#nixpkgs-integration} +- **Problem**: Service options don't match SHB needs +- **Solution**: Map SHB options to nixpkgs options, use `extraConfig` for overrides + +## Best Practices Summary {#best-practices-summary} + +1. **Follow existing patterns** - study deluge.nix and vaultwarden.nix +2. **Use freeform configuration** - maximum flexibility with typed key options +3. **Implement all contracts** - SSL, backup, monitoring, secrets +4. **Test comprehensively** - all 6 integration variants +5. **Security first** - localhost binding, proper permissions, secret management +6. **Document thoroughly** - clear descriptions for all options +7. **Iterative development** - build complexity gradually +8. **CI/CD validation** - ensure all tests pass before submission + +## Redirect Management {#redirect-management} + +SelfHostBlocks uses `nixos-render-docs` for documentation generation, which includes built-in redirect validation. The `docs/redirects.json` file maps documentation identifiers to their target URLs. + +### Automated Redirect Generation {#automated-redirect-generation} + +SelfHostBlocks includes an automated redirect management tool that leverages the official `nixos-render-docs` ecosystem: + +```bash +# Generate fresh redirects from HTML documentation +nix run .#update-redirects +``` + +This tool: +- **Generates HTML documentation** using `nixos-render-docs` with redirect collection enabled +- **Scans actual HTML files** for anchor IDs to ensure perfect accuracy +- **Creates fresh redirects** from scratch by mapping anchors to their real file locations +- **Filters system-generated anchors** (excludes `opt-*` and `selfhostblock*` entries) +- **Provides interactive confirmation** before updating `docs/redirects.json` + +### How Redirects Work {#how-redirects-work} + +1. **nixos-render-docs validation**: During documentation builds, `nixos-render-docs` automatically validates that all heading IDs have corresponding redirect entries +2. **Automated maintenance**: The `update-redirects` tool automatically maintains `redirects.json` by: + - Building HTML documentation with patched `nixos-render-docs` + - Scanning generated HTML files for actual anchor IDs and their file locations + - Creating accurate redirect mappings without guesswork or pattern matching +3. **Manual override**: You can still manually edit `docs/redirects.json` for special cases + +### Redirect Patterns {#redirect-patterns} + +The automation follows these patterns when mapping headings to redirect targets: + +| Heading ID | Source File | Redirect Target | +|------------|-------------|-----------------| +| `services-nzbget-basic` | `modules/services/nzbget/docs/default.md` | `["services-nzbget.html#services-nzbget-basic"]` | +| `blocks-monitoring` | `modules/blocks/monitoring/docs/default.md` | `["blocks-monitoring.html#blocks-monitoring"]` | +| `demo-nextcloud` | `demo/nextcloud/README.md` | `["demo-nextcloud.html#demo-nextcloud"]` | +| `contracts` | `docs/contracts.md` | `["contracts.html#contracts"]` | + +Note: Redirects always include the anchor link (`#heading-id`) to jump to the specific heading within the target page. + +### Adding New Service Documentation {#adding-new-service-documentation} + +When implementing a new service, the redirect workflow is now automated: + +1. **Write documentation** with heading IDs: + ```markdown + # NewService {\#services-newservice} + ## Basic Configuration {\#services-newservice-basic} + ``` + +2. **Update redirects automatically**: + ```bash + nix run .#update-redirects + ``` + +3. **Review and commit** the changes: + ```bash + git add docs/redirects.json modules/services/newservice/docs/default.md + git commit -m "Add newservice documentation" + ``` + +### Build-time Validation {#build-time-validation} + +The documentation build process will fail if: +- Any documentation heading ID lacks a corresponding redirect entry +- Redirect targets point to non-existent content +- There are formatting errors in the redirects file + +This ensures documentation links remain functional when content is moved or reorganized. + +## Resources {#resources} + +- **Contributing guide**: `docs/contributing.md` for authoritative development workflows and testing procedures +- **Existing services**: `modules/services/` for patterns and implementation examples +- **Contracts documentation**: `modules/contracts/` for understanding integration interfaces +- **Test framework**: `test/common.nix` for testing utilities and patterns +- **NixOS options**: https://search.nixos.org/options for upstream service options +- **SHB documentation**: Generated docs showing existing service patterns +- **Redirect automation**: `nix run .#update-redirects` for automated redirect management +- **nixos-render-docs**: Built-in redirect validation and documentation generation + +## Quick Reference {#quick-reference} + +### Complete Workflow {#complete-workflow} +```bash +# 1. Implement service module +vim modules/services/SERVICENAME.nix + +# 2. Create tests +vim test/services/SERVICENAME.nix + +# 3. Update flake +vim flake.nix # Add to allModules and checks + +# 4. Write documentation +vim modules/services/SERVICENAME/docs/default.md + +# 5. Generate redirects +nix run .#update-redirects + +# 6. Test everything +nix flake check # Run all tests (recommended) +# Or test specific variants: +# nix build .#checks.${system}.vm_SERVICENAME_basic +nix build .#manualHtml + +# 7. Commit changes +git add . +git commit -m "Add SERVICENAME with full integration" +``` + +This guide provides a complete roadmap for implementing production-ready SelfHostBlocks services that meet the project's quality standards. \ No newline at end of file diff --git a/flake.nix b/flake.nix index 6a52bf7..ecc8b29 100644 --- a/flake.nix +++ b/flake.nix @@ -104,6 +104,58 @@ release = builtins.readFile ./VERSION; }; + # Documentation redirect generation tool - scans HTML files for anchor mappings + packages.generateRedirects = + let + # Python patch to inject redirect collector + pythonPatch = pkgs.writeText "nixos-render-docs-patch.py" '' + # Load redirect collector patch + try: + import sys, os + sys.path.insert(0, os.path.dirname(__file__) + '/..') + import missing_refs_collector + except Exception as e: + print(f"Warning: Failed to load redirect collector: {e}", file=sys.stderr) + ''; + + # Patched nixos-render-docs that collects redirects during HTML generation + nixos-render-docs-patched = pkgs.writeShellApplication { + name = "nixos-render-docs"; + runtimeInputs = [ pkgs.nixos-render-docs ]; + text = '' + TEMP_DIR=$(mktemp -d); trap 'rm -rf "$TEMP_DIR"' EXIT + + cp -r ${pkgs.nixos-render-docs}/${pkgs.python3.sitePackages}/nixos_render_docs "$TEMP_DIR/" + chmod -R +w "$TEMP_DIR" + cp ${./docs/generate-redirects-nixos-render-docs.py} "$TEMP_DIR/missing_refs_collector.py" + echo '{}' > "$TEMP_DIR/empty_redirects.json" + cat ${pythonPatch} >> "$TEMP_DIR/nixos_render_docs/__init__.py" + + ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --redirects) ARGS+=("$1" "$TEMP_DIR/empty_redirects.json"); shift 2 ;; + *) ARGS+=("$1"); shift ;; + esac + done + + export PYTHONPATH="$TEMP_DIR:''${PYTHONPATH:-}" + nixos-render-docs "''${ARGS[@]}" + ''; + }; + in + (pkgs.callPackage ./docs { + inherit nmdsrc; + allModules = allModules ++ contractDummyModules; + release = builtins.readFile ./VERSION; + nixos-render-docs = nixos-render-docs-patched; + }).overrideAttrs (old: { + installPhase = '' + ${old.installPhase} + ln -sf share/doc/selfhostblocks/redirects.json $out/redirects.json + ''; + }); + lib = (pkgs.callPackage ./lib {}) // { @@ -193,6 +245,32 @@ --set PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS true ''; }) {}; + + # Run "nix run .#update-redirects" to regenerate docs/redirects.json + apps.update-redirects = { + type = "app"; + program = "${pkgs.writeShellApplication { + name = "update-redirects"; + runtimeInputs = [ pkgs.nix pkgs.jq ]; + text = '' + echo "=== SelfHostBlocks Redirects Updater ===" + echo "Generating fresh ./docs/redirects.json..." + + nix build .#generateRedirects || { echo "Error: Failed to generate redirects" >&2; exit 1; } + [[ -f result/redirects.json ]] || { echo "Error: Generated redirects file not found" >&2; exit 1; } + + echo "Generated $(jq 'keys | length' result/redirects.json) redirects" + echo + read -p "Update docs/redirects.json? This will backup the current file [y/N] " -r response + [[ "$response" =~ ^[Yy] ]] || { echo "Aborted - no changes made"; exit 0; } + + [[ -f docs/redirects.json ]] && cp docs/redirects.json docs/redirects.json.backup && echo "Created backup" + cp result/redirects.json docs/redirects.json + echo " Updated docs/redirects.json" + echo "To verify: nix build .#manualHtml" + ''; + }}/bin/update-redirects"; + }; } ) // { herculesCI.ciSystems = [ "x86_64-linux" ];