ci: use bisect flake lock workflow

.github/workflows/auto-merge.yaml

@@ -23,7 +23,7 @@ on:
         required: false
 
 jobs:
-  automerge:
+  automerge-rebase:
     runs-on: ubuntu-latest
     steps:
       - uses: reitermarkus/automerge@v2
@@ -35,3 +35,16 @@ jobs:
           pull-request: ${{ github.event.inputs.pull-request }}
           review: ${{ github.event.inputs.review }}
           dry-run: false
+
+  automerge-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: reitermarkus/automerge@v2
+        with:
+          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          merge-method: merge
+          do-not-merge-labels: never-merge
+          required-labels: automerge-merge
+          pull-request: ${{ github.event.inputs.pull-request }}
+          review: ${{ github.event.inputs.review }}
+          dry-run: false

.github/workflows/lock-update.yaml

@@ -1,9 +1,16 @@
+# See shb.skarabox.com/misc.html
 name: Update Flake Lock
 
 on:
-  workflow_dispatch:
   schedule:
-    - cron: '0 0 * * *' # runs daily at 00:00
+    - cron: '0 */3 * * *' # runs every 3 hours at :00
+
+  workflow_dispatch:
+    inputs:
+      bisect_future:
+        description: "Bisect in the future instead of the past"
+        required: false
+        default: "false"
 
 jobs:
   lockfile:
@@ -15,9 +22,12 @@ jobs:
         uses: cachix/install-nix-action@v31
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Update flake.lock
-        uses: DeterminateSystems/update-flake-lock@main
+      - name: Cache nixpkgs mirror
+        uses: actions/cache@v4
         with:
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
-          pr-labels: |
-            automerge
+          path: .cache/nixpkgs.git
+          key: nixpkgs-mirror-v1
+      - name: Update flake.lock
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+        script: nix .#update-flake-lock-pr

.github/workflows/update-flake-lock-pr.nix

@@ -0,0 +1,219 @@
+{
+  gh,
+  writeShellApplication,
+}:
+writeShellApplication {
+  name = "update-flake-lock-pr";
+  runtimeInputs = [
+    gh
+  ];
+  text = ''
+    branch=ci/nixpkgs-update
+    if [ "''${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
+      branch=ci/nixpkgs-update-auto
+    fi
+
+    goto_future=no
+    if [ -n "''${1:-}" ]; then
+      goto_future=yes
+    fi
+
+    main () {
+      PR=$(get_pr)
+
+      if [ -z "$PR" ]; then
+        create_pr
+        exit 0
+      fi
+
+      case "$(get_pr_check_status "$PR")" in
+        pending)
+          echo "Checks are running on PR $PR, nothing to do yet."
+          exit 0
+          ;;
+
+        succeeded)
+          echo "Checks succeeded, nothing to do."
+          exit 0
+          ;;
+
+        failing)
+          echo "Checks failed, updating PR to midpoint commit."
+
+          local CURRENT_COMMIT
+          local LAST_WORKING_COMMIT
+          local FAILING_COMMIT
+          local NEW_COMMIT
+
+          CURRENT_COMMIT="$(get_main_branch_commit)"
+          if [ "$goto_future" = "no" ]; then
+            LAST_WORKING_COMMIT="$CURRENT_COMMIT"
+            FAILING_COMMIT="$(get_pr_commit "$PR")"
+          else
+            LAST_WORKING_COMMIT="$(get_pr_commit "$PR")"
+            FAILING_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+          NEW_COMMIT="$(midpoint_commit "$LAST_WORKING_COMMIT" "$FAILING_COMMIT")"
+
+          if [ "$NEW_COMMIT" = "$LAST_WORKING_COMMIT" ] || [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            NEW_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+
+          if [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            echo "Latest nixpkgs commit is still same failing commit, will retry later."
+            exit 0
+          fi
+
+          update_pr "$PR" "$CURRENT_COMMIT" "$NEW_COMMIT"
+          exit 0
+          ;;
+      esac
+    }
+
+    get_main_branch_commit() {
+      git fetch origin main
+
+      git show origin/main:flake.lock \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    midpoint_commit() {
+      local good="$1"
+      local bad="$2"
+
+      mkdir -p .cache
+      if [ ! -d .cache/nixpkgs.git ]; then
+        git clone --mirror https://github.com/NixOS/nixpkgs.git .cache/nixpkgs.git
+      else
+        git --git-dir=.cache/nixpkgs.git fetch origin
+      fi
+
+      local commits=()
+
+      mapfile -t commits < <(
+        git --git-dir=.cache/nixpkgs.git \
+          rev-list \
+          --reverse \
+          --ancestry-path \
+          "''${good}..''${bad}"
+      )
+
+      local count="''${#commits[@]}"
+
+      if [ "$count" -eq 0 ]; then
+        echo "$bad"
+        return
+      fi
+
+      local midpoint_index=$((count / 2))
+
+      echo "''${commits[$midpoint_index]}"
+    }
+
+    get_latest_nixpkgs_commit () {
+      git ls-remote https://github.com/NixOS/nixpkgs nixos-unstable | cut -f1
+    }
+
+    get_pr () {
+      gh pr list \
+        --head "$branch" \
+        --state open \
+        --json number \
+        --jq '.[0].number // empty'
+    }
+
+    create_pr() {
+      local test_commit
+
+      test_commit="$(get_latest_nixpkgs_commit)"
+      echo "Creating PR on nixpkgs' latest commit $test_commit"
+
+      git checkout -B "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push -u origin "$branch" --force
+
+      current_commit="$(get_main_branch_commit)"
+      gh pr create \
+        --title "Update nixpkgs" \
+        --body "$(printf "%s\n\n%s" "Automated nixpkgs update. Latest tries:" " - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit")" \
+        --label automerge-merge
+    }
+
+    append_pr_body() {
+      local pr="$1"
+      local text="$2"
+
+      local current_body
+
+      current_body="$(gh pr view "$pr" --json body --jq .body)"
+
+      gh pr edit "$pr" \
+        --body "$(printf '%s%b' "$current_body" "$text")"
+    }
+
+    update_pr() {
+      local pr="$1"
+      local current_commit="$2"
+      local test_commit="$3"
+
+      echo "Updating PR to nixpkgs' commit $test_commit"
+
+      git checkout "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push --force-with-lease origin "$branch"
+
+      local future_text
+      if [ "$goto_future" = "yes" ]; then
+        future_text="bisected in the future"
+      else
+        future_text="bisected in the past"
+      fi
+      append_pr_body "$pr" \
+        " ❌\n - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit ($future_text)"
+    }
+
+    get_pr_check_status() {
+      local pr="$1"
+
+      local status
+      status="$(gh pr checks "$pr" --json state --jq '.[].state' || true)"
+
+      if echo "$status" | grep -qE 'PENDING|QUEUED|IN_PROGRESS'; then
+        echo "pending"
+        return
+      fi
+
+      if echo "$status" | grep -qE 'FAILURE|TIMED_OUT|CANCELLED'; then
+        echo "failing"
+        return
+      fi
+
+      echo "success"
+    }
+
+    get_pr_commit() {
+      local pr="$1"
+
+      git fetch origin "$branch"
+
+      git show "origin/$branch:flake.lock" \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    main
+  '';
+}

docs/manual.md

@@ -28,6 +28,10 @@ blocks.md
 recipes.md
 ```
 
+```{=include=} chapters html:into-file=//misc.html
+misc.md
+```
+
 ```{=include=} chapters html:into-file=//demos.html
 demos.md
 ```

docs/misc.md

@@ -0,0 +1,39 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Miscellaneous {#misc}
+
+Here goes meta-discussions around the repository and other topics not related directly to the usage of SHB.
+
+## Lock file update {#misc-lock-file-update}
+
+*SHB has an unusual strategy to keep up with its flake inputs. This section explains why.*
+
+SHB only depends on `nixpkgs` as far as flake inputs goes.
+There are others but they only matter for tasks around taking care of the repository itself.
+So only the `nixpkgs` input is important for downstream consumers of SHB.
+
+To keep up to date with nixpkgs unstable,
+initially a job was updating the nixpkgs input by simply updating the lock file and creating a PR.
+Now, this job failed most of the time because the tip of unstable often breaks modules and packages, resulting in failing SHB tests.
+To fix this, we usually needed to choose a commit around 2 weeks prior to the latest unstable commit.
+This usually did not lead to successful tests but the reason was then not
+because of failing packages but because nixpkgs modules evolved and SHB needs to adapt to that.
+The net effect was SHB was lagging too far behind unstable and updating it was becoming annoying.
+
+The new strategy now is as follows. On every job run:
+
+- If there is no PR, update nixpkgs input to latest unstable commit and create a PR which auto-merges when tests are successful.
+- If there is a PR:
+  - If the checks succeeded, do nothing because the PR will auto-merge.
+  - If the checks are pending, do nothing yet.
+  - If the checks failed, bisect and update nixpkgs input to between the commit on main and the commit in the PR.
+
+The effect here is as long as the tests are failing, we'll try a commit further in the past from nixpkgs unstable
+and closer to the tip of main branch.
+
+This could be made smarter by distinguishing between types of failures.
+We will see if this is needed.
+
+One can also run the bisect manually with `nix run .#update-flake-lock-pr`.
+This will create a PR on a different branch than the one used for the automated workflow.
+It accepts also one argument `future` which instead of trying a commit in between the tip of SHB main and the PR's failing one,
+it tries a commit between the PR's failing one and the nixpkgs unstable latest commit.

docs/redirects.json

@@ -926,6 +926,9 @@
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName"
   ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.timeout": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.timeout"
+  ],
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost"
   ],
@@ -2387,6 +2390,12 @@
   "local-testing": [
     "service-implementation-guide.html#local-testing"
   ],
+  "misc": [
+    "misc.html#misc"
+  ],
+  "misc-lock-file-update": [
+    "misc.html#misc-lock-file-update"
+  ],
   "monitoring-failures": [
     "service-implementation-guide.html#monitoring-failures"
   ],

flake.nix

@@ -257,6 +257,8 @@
           '';
         };
 
+        packages.update-flake-lock-pr = pkgs.callPackage ./.github/workflows/update-flake-lock-pr.nix { };
+
         lib = (pkgs.callPackage ./lib { }) // {
           test = pkgs.callPackage ./test/common.nix { };
           contracts = pkgs.callPackage ./modules/contracts {