add playwright test for forgejo

README.md

@@ -105,6 +105,10 @@ Self Host Blocks takes care of common self-hosting needs:
 - Automatic reverse proxy and certificate management for HTTPS.
 - VPN and proxy tunneling services.
 
+Great care is taken to make the proposed stack robust.
+This translates into a test suite comprised of automated NixOS VM tests
+including playwright tests.
+
 ### Services
 
 [Provided services](https://shb.skarabox.com/services.html) are:
@@ -305,7 +309,8 @@ the services on your own server.
 ## Community
 
 All issues and PRs are welcome. For PRs, if they are substantial changes, please open an issue to
-discuss the details first. More details in [here](https://shb.skarabox.com/contributing.html).
+discuss the details first. More details in [the contributing section](https://shb.skarabox.com/contributing.html)
+of the manual.
 
 Come hang out in the [Matrix channel](https://matrix.to/#/%23selfhostblocks%3Amatrix.org). :)
 

docs/contributing.md

@@ -43,6 +43,12 @@ When you get to the shell, run either `start_all()` or `test_script()`. The form
 the VMs and service, then you can introspect. The latter also starts the VMs if they are not yet and
 then will run the test script.
 
+If the test includes playwright tests, you can see the playwright trace with:
+
+```bash
+$ nix run .#playwright -- show-trace path/to/trace.zip
+```
+
 ## Upload test results to CI {#contributing-upload}
 
 Github actions do now have hardware acceleration, so running them there is not slow anymore. If

docs/redirects.json

@@ -968,6 +968,9 @@
   "services-forgejo-options": [
     "services-forgejo.html#services-forgejo-options"
   ],
+  "services-forgejo-options-shb.forgejo.adminUsername": [
+    "services-forgejo.html#services-forgejo-options-shb.forgejo.adminUsername"
+  ],
   "services-forgejo-options-shb.forgejo.adminPassword": [
     "services-forgejo.html#services-forgejo-options-shb.forgejo.adminPassword"
   ],

flake.nix

@@ -155,6 +155,25 @@
           // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
           // (vm_test "contracts-secret" ./test/contracts/secret.nix)
           ));
+
+        # Run nix .#playwright -- show-trace path/to/trace.zip
+        packages.playwright =
+          pkgs.callPackage ({ stdenvNoCC, makeWrapper, playwright }: stdenvNoCC.mkDerivation {
+            name = "playwright";
+
+            src = playwright;
+
+            nativeBuildInputs = [
+              makeWrapper
+            ];
+
+            # No quotes around the value for LLDAP_PASSWORD because we want the value to not be enclosed in quotes.
+            installPhase = ''
+              makeWrapper ${pkgs.python3Packages.playwright}/bin/playwright $out/bin/playwright \
+                --set PLAYWRIGHT_BROWSERS_PATH ${pkgs.playwright-driver.browsers} \
+                --set PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS true
+            '';
+          }) {};
       }
   ) // {
     herculesCI.ciSystems = [ "x86_64-linux" ];

modules/services/forgejo.nix

@@ -81,7 +81,7 @@ in
 
           adminName = mkOption {
             type = str;
-            description = "Admin user of the LDAP server.";
+            description = "Admin user of the LDAP server. Cannot be reserved word 'admin'.";
             default = "admin";
           };
 
@@ -170,8 +170,14 @@ in
       };
     };
 
+    adminUsername = mkOption {
+      type = str;
+      description = "Forgejo admin user name.";
+      default = "admin";
+    };
+
     adminPassword = mkOption {
-      description = "File containing the Forgejo admin user password.";
+      description = "Forgejo admin user password.";
       type = submodule {
         options = contracts.secret.mkRequester {
           mode = "0440";
@@ -495,10 +501,17 @@ in
     })
 
     (mkIf cfg.enable {
+      assertions = [
+        {
+          assertion = cfg.adminUsername != "admin";
+          message = "Admin username cannot be 'admin'.";
+        }
+      ];
+
       systemd.services.forgejo.preStart = ''
         admin="${getExe config.services.forgejo.package} admin user"
-        $admin create --admin --email "root@localhost" --username meadmin --password "$(tr -d '\n' < ${cfg.adminPassword.result.path})" || true
+        $admin create --admin --must-change-password=false --email "root@localhost" --username "${cfg.adminUsername}" --password "$(tr -d '\n' < ${cfg.adminPassword.result.path})" || true
-        $admin change-password --username meadmin --password "$(tr -d '\n' < ${cfg.adminPassword.result.path})" || true
+        $admin change-password --must-change-password=false --username "${cfg.adminUsername}" --password "$(tr -d '\n' < ${cfg.adminPassword.result.path})" || true
 '';
     })
 

test/common.nix

@@ -1,7 +1,7 @@
 { pkgs, lib }:
 let
   inherit (lib) hasAttr mkOption optionalString;
-  inherit (lib.types) bool listOf nullOr submodule str;
+  inherit (lib.types) listOf nullOr submodule str;
 
   baseImports = {
     imports = [
@@ -93,7 +93,11 @@ let
     '')
     + (optionalString (hasAttr "test" nodes.client && hasAttr "login" nodes.client.test) ''
     with subtest("Login"):
-        print(client.succeed("login_playwright firefox"))
+        code, logs = client.execute("login_playwright firefox")
+        client.copy_from_vm("trace")
+        print(logs)
+        if code != 0:
+            raise Exception("login_playwright did not succeed")
     '')
     + (let
       script = extraScript args;
@@ -201,8 +205,8 @@ in
           (let
             testCfg = pkgs.writeText "users.json" (builtins.toJSON cfg);
           in ''
-            # import re
             import json
+            import re
             import sys
             from playwright.sync_api import expect
             from playwright.sync_api import sync_playwright
@@ -226,24 +230,33 @@ in
             with sync_playwright() as p:
                 browser = getattr(p, browser_name).launch(args=browser_args)
 
-                for u in testCfg["testLoginWith"]:
+                for i, u in enumerate(testCfg["testLoginWith"]):
                     print(f"Testing for user {u['username']} and password {u['password']}")
 
                     context = browser.new_context(ignore_https_errors=True)
                     context.tracing.start(screenshots=True, snapshots=True, sources=True)
-                    page = context.new_page()
+                    try:
-                    page.goto(testCfg['startUrl'])
+                        page = context.new_page()
+                        print(f"Going to {testCfg['startUrl']}")
+                        page.goto(testCfg['startUrl'])
       
-                    page.screenshot(path="example.png")
+                        if u['username'] is not None:
-                    if u['username'] is not None:
+                            print(f"Filling field {testCfg['usernameFieldLabel']} with {u['username']}")
-                        page.get_by_label(testCfg['usernameFieldLabel']).fill(u['username'])
+                            page.get_by_label(testCfg['usernameFieldLabel']).fill(u['username'])
-                    if u['password'] is not None:
+                        if u['password'] is not None:
-                        page.get_by_label(testCfg['passwordFieldLabel']).fill(u['password'])
+                            print(f"Filling field {testCfg['passwordFieldLabel']} with {u['password']}")
-                    page.get_by_role("button", name=testCfg['loginButtonName']).click()
+                            page.get_by_label(testCfg['passwordFieldLabel']).fill(u['password'])
-                    for line in u['nextPageExpect']:
-                        eval(line)
 
-                    context.tracing.stop(path="trace.zip")
+                        print(f"Clicking button {testCfg['loginButtonName']}")
+                        page.get_by_role("button", name=testCfg['loginButtonName']).click()
+
+                        for line in u['nextPageExpect']:
+                            print(f"Running: {line}")
+                            print(f"Page has title: {page.title()}")
+                            eval(line)
+                    finally:
+                        page.screenshot(path=f"trace/{i}/final.png")
+                        context.tracing.stop(path=f"trace/{i}.zip")
 
                 browser.close()
           '')

test/services/forgejo.nix

@@ -20,6 +20,12 @@ let
   };
 
   basic = { config, ... }: {
+    imports = [
+      testLib.baseModule
+      ../../modules/blocks/hardcodedsecret.nix
+      ../../modules/services/forgejo.nix
+    ];
+
     test = {
       subdomain = "f";
     };
@@ -28,6 +34,7 @@ let
       enable = true;
       inherit (config.test) subdomain domain;
 
+      adminUsername = "theadmin";
       adminPassword.result = config.shb.hardcodedsecret.forgejoAdminPassword.result;
       databasePassword.result = config.shb.hardcodedsecret.forgejoDatabasePassword.result;
     };
@@ -48,6 +55,33 @@ let
     };
   };
 
+  clientLogin = { config, ... }: {
+    imports = [
+      testLib.baseModule
+      testLib.clientLoginModule
+    ];
+    test = {
+      subdomain = "f";
+    };
+
+    test.login = {
+      startUrl = "http://${config.test.fqdn}/user/login";
+      usernameFieldLabel = "Username or email address";
+      passwordFieldLabel = "Password";
+      loginButtonName = "Sign In";
+      testLoginWith = [
+        { username = "theadmin"; password = adminPassword + "oops"; nextPageExpect = [
+            "expect(page.get_by_text('Username or password is incorrect.')).to_be_visible()"
+          ]; }
+        { username = "theadmin"; password = adminPassword; nextPageExpect = [
+            "expect(page.get_by_text('Username or password is incorrect.')).not_to_be_visible()"
+            "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
+            "expect(page).to_have_title(re.compile('Dashboard'))"
+          ]; }
+      ];
+    };
+  };
+
   https = { config, ... }: {
     shb.forgejo = {
       ssl = config.shb.certs.certs.selfsigned.n;
@@ -96,16 +130,17 @@ in
   basic = pkgs.testers.runNixOSTest {
     name = "forgejo_basic";
 
+    nodes.client = {
+      imports = [
+        clientLogin
+      ];
+    };
     nodes.server = {
       imports = [
-        testLib.baseModule
-        ../../modules/services/forgejo.nix
         basic
       ];
     };
 
-    nodes.client = {};
-
     testScript = commonTestScript.access;
   };
 
@@ -114,8 +149,6 @@ in
 
     nodes.server = { config, ... }: {
       imports = [
-        testLib.baseModule
-        ../../modules/services/forgejo.nix
         basic
         (testLib.backup config.shb.forgejo.backup)
       ];
@@ -131,10 +164,8 @@ in
 
     nodes.server = {
       imports = [
-        testLib.baseModule
-        ../../modules/services/forgejo.nix
-        testLib.certs
         basic
+        testLib.certs
         https
       ];
     };
@@ -149,8 +180,6 @@ in
 
     nodes.server = {
       imports = [
-        testLib.baseModule
-        ../../modules/services/forgejo.nix
         basic
         testLib.ldap
         ldap
@@ -167,10 +196,8 @@ in
 
     nodes.server = { config, pkgs, ... }: {
       imports = [
-        testLib.baseModule
-        ../../modules/services/forgejo.nix
-        testLib.certs
         basic
+        testLib.certs
         https
         testLib.ldap
         (testLib.sso config.shb.certs.certs.selfsigned.n)