From 67c2705f98ce2382aa4171322be8f8427e7add01 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Tue, 29 Jul 2025 22:42:57 +0200 Subject: [PATCH] add mitmdump block --- docs/blocks.md | 8 +- docs/redirects.json | 30 ++++++ flake.nix | 4 + modules/blocks/mitmdump.nix | 136 ++++++++++++++++++++++++ modules/blocks/mitmdump/docs/default.md | 91 ++++++++++++++++ test/blocks/mitmdump.nix | 123 +++++++++++++++++++++ 6 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 modules/blocks/mitmdump.nix create mode 100644 modules/blocks/mitmdump/docs/default.md create mode 100644 test/blocks/mitmdump.nix diff --git a/docs/blocks.md b/docs/blocks.md index 6747c07..655d2d6 100644 --- a/docs/blocks.md +++ b/docs/blocks.md @@ -5,7 +5,7 @@ Blocks help you self-host apps or services. They implement a specific function l access through a subdomain. Each block is designed to be usable on its own and to fit nicely with others. -In practice, a block implements a [contract](contracts.html) that must be followed to implement a +In practice, most blocks implement a [contract](contracts.html) that must be followed to implement a specific self-hosting function. It also comes with a unit test and NixOS VM test suite to ensure any implementation follows the contract. @@ -23,7 +23,7 @@ It defines no outputs but has one major side effect: Anything that provides the inputs and expected outputs and side effects defined by the block can be used to fulfill its contract. In this example, we could use any of Nginx, Caddy, Haproxy or others. -Self Host Blocks provides at least one implementation for each block and allows you to use your own +Self Host Blocks provides at least one implementation for each contract and allows you to use your own implementation if you want to, as long as it passes the tests. You can then use blocks to improve services you already have deployed. @@ -39,6 +39,10 @@ modules/blocks/authelia/docs/default.md modules/blocks/lldap/docs/default.md ``` +```{=include=} chapters html:into-file=//blocks-mitmdump.html +modules/blocks/mitmdump/docs/default.md +``` + ```{=include=} chapters html:into-file=//blocks-sops.html modules/blocks/sops/docs/default.md ``` diff --git a/docs/redirects.json b/docs/redirects.json index d9604aa..d64bfe3 100644 --- a/docs/redirects.json +++ b/docs/redirects.json @@ -557,6 +557,36 @@ "blocks-lldap-troubleshooting": [ "blocks-lldap.html#blocks-lldap-troubleshooting" ], + "blocks-mitmdump": [ + "blocks-mitmdump.html#blocks-mitmdump" + ], + "blocks-mitmdump-example": [ + "blocks-mitmdump.html#blocks-mitmdump-example" + ], + "blocks-mitmdump-options": [ + "blocks-mitmdump.html#blocks-mitmdump-options" + ], + "blocks-mitmdump-options-shb.mitmdump.instances": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.after": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.after" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamPort": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamPort" + ], + "blocks-mitmdump-tests": [ + "blocks-mitmdump.html#blocks-mitmdump-tests" + ], + "blocks-mitmdump-usage": [ + "blocks-mitmdump.html#blocks-mitmdump-usage" + ], "blocks-monitoring": [ "blocks-monitoring.html#blocks-monitoring" ], diff --git a/flake.nix b/flake.nix index 23cf69c..d31c383 100644 --- a/flake.nix +++ b/flake.nix @@ -61,6 +61,7 @@ self.nixosModules.${system}.davfs self.nixosModules.${system}.hardcodedsecret self.nixosModules.${system}.lldap + self.nixosModules.${system}.mitmdump self.nixosModules.${system}.monitoring self.nixosModules.${system}.nginx self.nixosModules.${system}.postgresql @@ -94,6 +95,7 @@ nixosModules.davfs = modules/blocks/davfs.nix; nixosModules.hardcodedsecret = modules/blocks/hardcodedsecret.nix; nixosModules.lldap = modules/blocks/lldap.nix; + nixosModules.mitmdump = modules/blocks/mitmdump.nix; nixosModules.monitoring = modules/blocks/monitoring.nix; nixosModules.nginx = modules/blocks/nginx.nix; nixosModules.postgresql = modules/blocks/postgresql.nix; @@ -128,6 +130,7 @@ "blocks/authelia" = ./modules/blocks/authelia.nix; "blocks/lldap" = ./modules/blocks/lldap.nix; "blocks/ssl" = ./modules/blocks/ssl.nix; + "blocks/mitmdump" = ./modules/blocks/mitmdump.nix; "blocks/monitoring" = ./modules/blocks/monitoring.nix; "blocks/postgresql" = ./modules/blocks/postgresql.nix; "blocks/restic" = ./modules/blocks/restic.nix; @@ -261,6 +264,7 @@ // (vm_test "authelia" ./test/blocks/authelia.nix) // (vm_test "lldap" ./test/blocks/lldap.nix) // (vm_test "lib" ./test/blocks/lib.nix) + // (vm_test "mitmdump" ./test/blocks/mitmdump.nix) // (vm_test "postgresql" ./test/blocks/postgresql.nix) // (vm_test "restic" ./test/blocks/restic.nix) // (vm_test "ssl" ./test/blocks/ssl.nix) diff --git a/modules/blocks/mitmdump.nix b/modules/blocks/mitmdump.nix new file mode 100644 index 0000000..8df2fd1 --- /dev/null +++ b/modules/blocks/mitmdump.nix @@ -0,0 +1,136 @@ +{ config, lib, pkgs, ... }: +let + inherit (lib) mapAttrs' mkOption nameValuePair types; + inherit (types) attrsOf listOf port submodule str; + + cfg = config.shb.mitmdump; +in +{ + options.shb.mitmdump = { + instances = mkOption { + default = {}; + description = "Mitmdump instance."; + type = attrsOf (submodule ({ name, ... }: { + options = { + listenPort = mkOption { + type = port; + description = '' + Port the mitmdump instance will listen to. + + The upstream port from the client's perspective. + ''; + }; + + upstreamHost = mkOption { + type = str; + default = "http://127.0.0.1"; + description = '' + Host the mitmdump instance will connect to. + + If only an IP or domain is provided, + mitmdump will default to connect using HTTPS. + If this is not wanted, prefix the IP or domain with the 'http://' protocol. + ''; + }; + + upstreamPort = mkOption { + type = port; + description = '' + Port the mitmdump instance will connect to. + + The port the server is listening on. + ''; + }; + + after = mkOption { + type = listOf str; + default = []; + description = '' + Systemd services that must be started before this mitmdump proxy instance. + + You are guaranteed the mitmdump is listening on the `listenPort` + when its systemd service has started. + ''; + }; + }; + })); + }; + }; + + config = { + systemd.services = mapAttrs' (name: cfg: nameValuePair "mitmdump-${name}" { + environment = { + "HOME" = "/var/lib/private/mitmdump-${name}"; + }; + serviceConfig = { + Type = "notify"; + Restart = "on-failure"; + StandardOutput = "journal"; + StandardError = "journal"; + + DynamicUser = true; + WorkingDirectory = "/var/lib/mitmdump-${name}"; + StateDirectory = "mitmdump-${name}"; + + ExecStart = lib.getExe (pkgs.writers.writePython3Bin "mitmdump-${name}" + { + libraries = [ pkgs.python3Packages.systemd ]; + flakeIgnore = [ "E501" ]; + } + '' + from systemd.daemon import notify + import logging + import subprocess + import socket + import sys + import time + + + logging.basicConfig(level=logging.INFO, format='%(message)s') + + + def wait_for_port(host, port, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection((host, port), timeout=0.5): + return True + except Exception: + time.sleep(0.1) + return False + + + logging.info("Waiting for upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to be up.") + wait_for_port("${cfg.upstreamHost}", ${toString cfg.upstreamPort}, timeout=10) + logging.info("Upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to is up.") + + proc = subprocess.Popen( + [ + "${pkgs.mitmproxy}/bin/mitmdump", + "--listen-host", "127.0.0.1", + "-p", "${toString cfg.listenPort}", + "--set", "flow_detail=3", + "--set", "content_view_lines_cutoff=2000", + "--mode", "reverse:${cfg.upstreamHost}:${toString cfg.upstreamPort}", + ], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + logging.info("Waiting for mitmdump instance to start on port ${toString cfg.listenPort}.") + if wait_for_port("127.0.0.1", ${toString cfg.listenPort}, timeout=10): + logging.info("Mitmdump is started on port ${toString cfg.listenPort}.") + notify("READY=1") + else: + proc.terminate() + exit(1) + + proc.wait() + ''); + }; + requires = cfg.after; + after = cfg.after; + wantedBy = [ "multi-user.target" ]; + }) cfg.instances; + }; +} diff --git a/modules/blocks/mitmdump/docs/default.md b/modules/blocks/mitmdump/docs/default.md new file mode 100644 index 0000000..9627da0 --- /dev/null +++ b/modules/blocks/mitmdump/docs/default.md @@ -0,0 +1,91 @@ +# Mitmdump Block {#blocks-mitmdump} + +Defined in [`/modules/blocks/authelia.nix`](@REPO@/modules/blocks/authelia.nix). + +This block sets up an [Mitmdump][] service in [reverse proxy][] mode. +In other words, you can put this block between a client and a server to inspect all the network traffic. + +[Mitmdump]: https://plattner.me/mp-docs/#mitmdump +[reverse proxy]: https://plattner.me/mp-docs/concepts-modes/#reverse-proxy + +Multiple instances of mitmdump all listening on different ports +and proxying to different upstream servers can be created. + +The systemd service is made so it is started only when the mitmdump instance +has started listening on the expected port. + +## Usage {#blocks-mitmdump-usage} + +Put mitmdump in front of a HTTP server listening on port 8000 on the same machine: + +```nix +shb.mitmdump.instances."my-instance" = { + listenPort = 8001; + upstreamHost = "http://127.0.0.1"; + upstreamPort = 8000; + after = [ "server.service" ]; +}; +``` + +`upstreamHost` is the default here and can be left out. + +Put mitmdump in front of a HTTP server listening on port 8000 on another machine: + +```nix +shb.mitmdump.instances."my-instance" = { + listenPort = 8001; + upstreamHost = "http://otherhost"; + upstreamPort = 8000; + after = [ "server.service" ]; +}; +``` + +Replace `http` with `https` if the server expects an HTTPS connection. + +## Example {#blocks-mitmdump-example} + +Let's assume a server is listening on port 8000 +which responds a plain text response `test1` +and its related systemd service is named `test1.service`. +Sorry, creative naming is not my forte. + +Let's put an mitmdump instance in front of it, like so: + +```nix +shb.mitmdump.instances."test1" = { + listenPort = 8001; + upstreamPort = 8000; + after = [ "test1.service" ]; +}; +``` + +This creates an `mitmdump-test1.service` systemd service. +We can then use `journalctl -u mitmdump-test1.service` to see the output. + +If we make a `curl` request to it: `curl -v http://127.0.0.1:8001`, +we will get the following output: + +``` +mitmdump-test1[971]: 127.0.0.1:40878: GET http://127.0.0.1:8000/ HTTP/1.1 +mitmdump-test1[971]: Host: 127.0.0.1:8000 +mitmdump-test1[971]: User-Agent: curl/8.14.1 +mitmdump-test1[971]: Accept: */* +mitmdump-test1[971]: << HTTP/1.0 200 OK 5b +mitmdump-test1[971]: Server: BaseHTTP/0.6 Python/3.13.4 +mitmdump-test1[971]: Date: Thu, 31 Jul 2025 20:55:16 GMT +mitmdump-test1[971]: Content-Type: text/plain +mitmdump-test1[971]: Content-Length: 5 +mitmdump-test1[971]: test1 +``` + +## Tests {#blocks-mitmdump-tests} + +Specific integration tests are defined in [`/test/blocks/mitmdump.nix`](@REPO@/test/blocks/mitmdump.nix). + +## Options Reference {#blocks-mitmdump-options} + +```{=include=} options +id-prefix: blocks-mitmdump-options- +list-id: selfhostblocks-block-mitmdump-options +source: @OPTIONS_JSON@ +``` diff --git a/test/blocks/mitmdump.nix b/test/blocks/mitmdump.nix new file mode 100644 index 0000000..1571a32 --- /dev/null +++ b/test/blocks/mitmdump.nix @@ -0,0 +1,123 @@ +{ pkgs, lib, ... }: +let + serve = port: text: lib.getExe (pkgs.writers.writePython3Bin "serve" + { + libraries = [ pkgs.python3Packages.systemd ]; + } + (let + content = pkgs.writeText "content" text; + in '' + from http.server import BaseHTTPRequestHandler, HTTPServer + from systemd.daemon import notify + + with open("${content}", "rb") as f: + content = f.read() + + + class HardcodedHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + print("answering to GET request") + self.wfile.write(content) + + def log_message(self, format, *args): + pass # optional: suppress logging + + + if __name__ == "__main__": + notify('STATUS=Starting up...') + server_address = ('127.0.0.1', ${toString port}) + httpd = HTTPServer(server_address, HardcodedHandler) + print("Serving hardcoded page on http://127.0.0.1:${toString port}") + notify('READY=1') + httpd.serve_forever() + '') + ); +in +{ + default = pkgs.testers.runNixOSTest { + name = "mitmdump-default"; + + nodes.machine = { config, pkgs, ... }: { + imports = [ + ../../modules/blocks/mitmdump.nix + ]; + + systemd.services.test1 = { + serviceConfig.ExecStart = serve 8000 "test1"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "notify"; + StandardOutput = "journal"; + StandardError = "journal"; + }; + }; + + systemd.services.test2 = { + serviceConfig.ExecStart = serve 8002 "test2"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "notify"; + StandardOutput = "journal"; + StandardError = "journal"; + }; + }; + + shb.mitmdump.instances."test1" = { + listenPort = 8001; + upstreamPort = 8000; + after = [ "test1.service" ]; + }; + + shb.mitmdump.instances."test2" = { + listenPort = 8003; + upstreamPort = 8002; + after = [ "test2.service" ]; + }; + }; + + testScript = { nodes, ... }: '' + start_all() + + machine.wait_for_unit("test1.service") + machine.wait_for_unit("test2.service") + machine.wait_for_unit("mitmdump-test1.service") + machine.wait_for_unit("mitmdump-test2.service") + + resp = machine.succeed("curl http://127.0.0.1:8000") + print(resp) + if resp != "test1": + raise Exception("wanted 'test1'") + + resp = machine.succeed("curl -v http://127.0.0.1:8001") + print(resp) + if resp != "test1": + raise Exception("wanted 'test1'") + + resp = machine.succeed("curl http://127.0.0.1:8002") + print(resp) + if resp != "test2": + raise Exception("wanted 'test2'") + + resp = machine.succeed("curl http://127.0.0.1:8003") + print(resp) + if resp != "test2": + raise Exception("wanted 'test2'") + + dump = machine.succeed("journalctl -b -u mitmdump-test1.service") + if "HTTP/1.0 200 OK" not in dump: + raise Exception("expected to see HTTP/1.0 200 OK") + if "test1" not in dump: + raise Exception("expected to see test1") + + dump = machine.succeed("journalctl -b -u mitmdump-test2.service") + if "HTTP/1.0 200 OK" not in dump: + raise Exception("expected to see HTTP/1.0 200 OK") + if "test2" not in dump: + raise Exception("expected to see test2") + ''; + }; +}