fine grained logging with mitmdump logger addon

docs/redirects.json

@@ -560,21 +560,45 @@
   "blocks-mitmdump": [
     "blocks-mitmdump.html#blocks-mitmdump"
   ],
+  "blocks-mitmdump-addons": [
+    "blocks-mitmdump.html#blocks-mitmdump-addons"
+  ],
+  "blocks-mitmdump-addons-logger": [
+    "blocks-mitmdump.html#blocks-mitmdump-addons-logger"
+  ],
   "blocks-mitmdump-example": [
     "blocks-mitmdump.html#blocks-mitmdump-example"
   ],
   "blocks-mitmdump-options": [
     "blocks-mitmdump.html#blocks-mitmdump-options"
   ],
+  "blocks-mitmdump-options-shb.mitmdump.addons": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.addons"
+  ],
   "blocks-mitmdump-options-shb.mitmdump.instances": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances"
   ],
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.after": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.after"
   ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.enabledAddons": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.enabledAddons"
+  ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.extraArgs": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.extraArgs"
+  ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.listenHost": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenHost"
+  ],
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort"
   ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.package": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.package"
+  ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName"
+  ],
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost"
   ],
@@ -587,6 +611,15 @@
   "blocks-mitmdump-usage": [
     "blocks-mitmdump.html#blocks-mitmdump-usage"
   ],
+  "blocks-mitmdump-usage-anywhere": [
+    "blocks-mitmdump.html#blocks-mitmdump-usage-anywhere"
+  ],
+  "blocks-mitmdump-usage-https": [
+    "blocks-mitmdump.html#blocks-mitmdump-usage-https"
+  ],
+  "blocks-mitmdump-usage-logging": [
+    "blocks-mitmdump.html#blocks-mitmdump-usage-logging"
+  ],
   "blocks-monitoring": [
     "blocks-monitoring.html#blocks-monitoring"
   ],

modules/blocks/mitmdump.nix

@@ -4,18 +4,175 @@ let
   inherit (types) attrsOf listOf port submodule str;
 
   cfg = config.shb.mitmdump;
+
+  mitmdumpScript = pkgs.writers.writePython3Bin "mitmdump"
+    {
+      libraries = let
+        p = pkgs.python3Packages;
+      in [
+        p.systemd
+        p.mitmproxy
+      ];
+      flakeIgnore = [ "E501" ];
+    }
+    ''
+      from systemd.daemon import notify
+      import argparse
+      import logging
+      import os
+      import subprocess
+      import socket
+      import sys
+      import time
+
+
+      logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+
+      def wait_for_port(host, port, timeout=10):
+          deadline = time.time() + timeout
+          while time.time() < deadline:
+              try:
+                  with socket.create_connection((host, port), timeout=0.5):
+                      return True
+              except Exception:
+                  time.sleep(0.1)
+          return False
+
+
+      def flatten(xss):
+          return [x for xs in xss for x in xs]
+
+
+      parser = argparse.ArgumentParser()
+      parser.add_argument("--listen_host", default="127.0.0.1", help="Host mitmdump will listen on")
+      parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
+      parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
+      parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
+      args, rest = parser.parse_known_args()
+
+      MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
+      if MITMDUMP_BIN is None:
+          raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
+
+      logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
+      wait_for_port(args.upstream_host, args.upstream_port, timeout=10)
+      logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
+
+      proc = subprocess.Popen(
+          [
+              MITMDUMP_BIN,
+              "--listen-host", args.listen_host,
+              "-p", args.listen_port,
+              "--mode", f"reverse:{args.upstream_host}:{args.upstream_port}",
+          ] + rest,
+          stdout=sys.stdout,
+          stderr=sys.stderr,
+      )
+
+      logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
+      if wait_for_port("127.0.0.1", args.listen_port, timeout=10):
+          logging.info(f"Mitmdump is started on port {args.listen_port}.")
+          notify("READY=1")
+      else:
+          proc.terminate()
+          exit(1)
+
+      proc.wait()
+    '';
+
+  logger = toString (pkgs.writers.writeText "loggerAddon.py"
+    ''
+    import logging
+    from collections.abc import Sequence
+    from mitmproxy import ctx, http
+    import re
+
+
+    logger = logging.getLogger(__name__)
+
+
+    class RegexLogger:
+        def __init__(self):
+            self.verbose_patterns = None
+
+        def load(self, loader):
+            loader.add_option(
+                name="verbose_pattern",
+                typespec=Sequence[str],
+                default=[],
+                help="Regex patterns for verbose logging",
+            )
+
+        def response(self, flow: http.HTTPFlow):
+            if self.verbose_patterns is None:
+                self.verbose_patterns = [re.compile(p) for p in ctx.options.verbose_pattern]
+
+            matched = any(p.search(flow.request.path) for p in self.verbose_patterns)
+            if matched:
+                logger.info(format_flow(flow))
+
+
+    def format_flow(flow: http.HTTPFlow) -> str:
+        return (
+            "\n"
+            "RequestHeaders:\n"
+            f"    {format_headers(flow.request.headers.items())}\n"
+            f"RequestBody:     {flow.request.get_text()}\n"
+            f"Status:          {flow.response.data.status_code}\n"
+            "ResponseHeaders:\n"
+            f"    {format_headers(flow.response.headers.items())}\n"
+            f"ResponseBody:    {flow.response.get_text()}\n"
+        )
+
+
+    def format_headers(headers) -> str:
+        return "\n    ".join(k + ": " + v for k, v in headers)
+
+
+    addons = [RegexLogger()]
+  '');
 in
 {
   options.shb.mitmdump = {
+    addons = mkOption {
+      type = attrsOf str;
+      default = [];
+      description = ''
+        Addons available to the be added to the mitmdump instance.
+
+        To enabled them, add them to the `enabledAddons` option.
+      '';
+    };
+
     instances = mkOption {
       default = {};
       description = "Mitmdump instance.";
       type = attrsOf (submodule ({ name, ... }: {
         options = {
+          package = lib.mkPackageOption pkgs "mitmproxy" {};
+
+          serviceName = mkOption {
+            type = str;
+            description = ''
+              Name of the mitmdump system service.
+            '';
+            default = "mitmdump-${name}.service";
+            readOnly = true;
+          };
+
+          listenHost = mkOption {
+            type = str;
+            default = "127.0.0.1";
+            description = ''
+              Host the mitmdump instance will connect on.
+            '';
+          };
+
           listenPort = mkOption {
             type = port;
             description = ''
-              Port the mitmdump instance will listen to.
+              Port the mitmdump instance will listen on.
 
               The upstream port from the client's perspective.
             '';
@@ -52,15 +209,36 @@ in
               when its systemd service has started.
             '';
           };
+
+          enabledAddons = mkOption {
+            type = listOf str;
+            default = [];
+            description = ''
+              Addons to enable on this mitmdump instance.
+            '';
+            example = lib.literalExpression ''[ config.shb.mitmdump.addons.logger ]'';
+          };
+
+          extraArgs = mkOption {
+            type = listOf str;
+            default = [];
+            description = ''
+              Extra arguments to pass to the mitmdump instance.
+
+              See upstream [manual](https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) for all possible options.
+            '';
+            example = lib.literalExpression ''[ "--set" "verbose_pattern=/api" ]'';
+          };
         };
       }));
     };
   };
 
   config = {
-    systemd.services = mapAttrs' (name: cfg: nameValuePair "mitmdump-${name}" {
+    systemd.services = mapAttrs' (name: cfg': nameValuePair "mitmdump-${name}" {
       environment = {
         "HOME" = "/var/lib/private/mitmdump-${name}";
+        "MITMDUMP_BIN" = "${cfg'.package}/bin/mitmdump";
       };
       serviceConfig = {
         Type = "notify";
@@ -72,65 +250,19 @@ in
         WorkingDirectory = "/var/lib/mitmdump-${name}";
         StateDirectory = "mitmdump-${name}";
 
-        ExecStart = lib.getExe (pkgs.writers.writePython3Bin "mitmdump-${name}"
+        ExecStart = let
-          {
+          addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
-            libraries = [ pkgs.python3Packages.systemd ];
+          extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
-            flakeIgnore = [ "E501" ];
+        in
-          }
+          "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}";
-          ''
-            from systemd.daemon import notify
-            import logging
-            import subprocess
-            import socket
-            import sys
-            import time
-
-
-            logging.basicConfig(level=logging.INFO, format='%(message)s')
-            
-
-            def wait_for_port(host, port, timeout=10):
-                deadline = time.time() + timeout
-                while time.time() < deadline:
-                    try:
-                        with socket.create_connection((host, port), timeout=0.5):
-                            return True
-                    except Exception:
-                        time.sleep(0.1)
-                return False
-
-
-            logging.info("Waiting for upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to be up.")
-            wait_for_port("${cfg.upstreamHost}", ${toString cfg.upstreamPort}, timeout=10)
-            logging.info("Upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to is up.")
-            
-            proc = subprocess.Popen(
-                [
-                    "${pkgs.mitmproxy}/bin/mitmdump",
-                    "--listen-host", "127.0.0.1",
-                    "-p", "${toString cfg.listenPort}",
-                    "--set", "flow_detail=3",
-                    "--set", "content_view_lines_cutoff=2000",
-                    "--mode", "reverse:${cfg.upstreamHost}:${toString cfg.upstreamPort}",
-                ],
-                stdout=sys.stdout,
-                stderr=sys.stderr,
-            )
-
-            logging.info("Waiting for mitmdump instance to start on port ${toString cfg.listenPort}.")
-            if wait_for_port("127.0.0.1", ${toString cfg.listenPort}, timeout=10):
-                logging.info("Mitmdump is started on port ${toString cfg.listenPort}.")
-                notify("READY=1")
-            else:
-                proc.terminate()
-                exit(1)
-            
-            proc.wait()
-          '');
       };
-      requires = cfg.after;
+      requires = cfg'.after;
-      after = cfg.after;
+      after = cfg'.after;
       wantedBy = [ "multi-user.target" ];
     }) cfg.instances;
+
+    shb.mitmdump.addons = {
+      inherit logger;
+    };
   };
 }

modules/blocks/mitmdump/docs/default.md

@@ -14,6 +14,8 @@ and proxying to different upstream servers can be created.
 The systemd service is made so it is started only when the mitmdump instance
 has started listening on the expected port.
 
+Also, addons can be enabled with the `enabledAddons` option.
+
 ## Usage {#blocks-mitmdump-usage}
 
 Put mitmdump in front of a HTTP server listening on port 8000 on the same machine:
@@ -40,8 +42,89 @@ shb.mitmdump.instances."my-instance" = {
 };
 ```
 
+### Handle Upstream TLS {#blocks-mitmdump-usage-https}
+
 Replace `http` with `https` if the server expects an HTTPS connection.
 
+### Accept Connections from Anywhere {#blocks-mitmdump-usage-anywhere}
+
+By default, `mitmdump` is configured to listen only for connections from localhost.
+Add `listenHost=0.0.0.0` to make `mitmdump` accept connections from anywhere.
+
+### Extra Logging {#blocks-mitmdump-usage-logging}
+
+To print request and response bodies and more, increase the logging with:
+
+```nix
+extraArgs = [
+    "--set" "flow_detail=3"
+    "--set" "content_view_lines_cutoff=2000"
+];
+```
+
+The default `flow_details` is 1. See the [manual][] for more explanations on the option.
+
+[manual]: (https://docs.mitmproxy.org/stable/concepts/options/#flow_detail)
+
+This will change the verbosity for all requests and responses.
+If you need more fine grained logging, configure instead the [Logger Addon][].
+
+[Logger Addon]: #blocks-mitmdump-addons-logger
+
+## Addons {#blocks-mitmdump-addons}
+
+All provided addons can be found under the `shb.mitmproxy.addons` option.
+
+To enable one for an instance, add it to the `enabledAddons` option. For example:
+
+```nix
+shb.mitmdump.instances."my-instance" = {
+    enabledAddons = [ config.shb.mitmdump.addons.logger ]
+}
+```
+
+### Fine Grained Logger {#blocks-mitmdump-addons-logger}
+
+The Fine Grained Logger addon is found under `shb.mitmproxy.addons.logger`.
+Enabling this addon will add the `mitmdump` option `verbose_pattern` which takes a regex and if it matches,
+prints the request and response headers and body.
+If it does not match, it will just print the response status.
+
+For example, with the `extraArgs`:
+
+```nix
+extraArgs = [
+  "--set" "verbose_pattern=/verbose"
+];
+```
+
+A `GET` request to `/notverbose` will print something similar to:
+
+```
+mitmdump[972]: 127.0.0.1:53586: GET http://127.0.0.1:8000/notverbose HTTP/1.1
+mitmdump[972]:      << HTTP/1.0 200 OK 16b
+```
+
+While a `GET` request to `/verbose` will print something similar to:
+
+```
+mitmdump[972]: [22:42:58.840]
+mitmdump[972]: RequestHeaders:
+mitmdump[972]:     Host: 127.0.0.1:8000
+mitmdump[972]:     User-Agent: curl/8.14.1
+mitmdump[972]:     Accept: */*
+mitmdump[972]: RequestBody:
+mitmdump[972]: Status:          200
+mitmdump[972]: ResponseHeaders:
+mitmdump[972]:     Server: BaseHTTP/0.6 Python/3.13.4
+mitmdump[972]:     Date: Sun, 03 Aug 2025 22:42:58 GMT
+mitmdump[972]:     Content-Type: text/plain
+mitmdump[972]:     Content-Length: 13
+mitmdump[972]: ResponseBody:    test2/verbose
+mitmdump[972]: 127.0.0.1:53602: GET http://127.0.0.1:8000/verbose HTTP/1.1
+mitmdump[972]:      << HTTP/1.0 200 OK 13b
+```
+
 ## Example {#blocks-mitmdump-example}
 
 Let's assume a server is listening on port 8000
@@ -56,6 +139,10 @@ shb.mitmdump.instances."test1" = {
   listenPort = 8001;
   upstreamPort = 8000;
   after = [ "test1.service" ];
+  extraArgs = [
+    "--set" "flow_detail=3"
+    "--set" "content_view_lines_cutoff=2000"
+  ];
 };
 ```
 

test/blocks/mitmdump.nix

@@ -16,12 +16,13 @@ let
 
       class HardcodedHandler(BaseHTTPRequestHandler):
           def do_GET(self):
+              reponse = content + self.path.encode('utf-8')
               self.send_response(200)
               self.send_header("Content-Type", "text/plain")
-              self.send_header("Content-Length", str(len(content)))
+              self.send_header("Content-Length", str(len(reponse)))
               self.end_headers()
               print("answering to GET request")
-              self.wfile.write(content)
+              self.wfile.write(reponse)
 
           def log_message(self, format, *args):
               pass  # optional: suppress logging
@@ -76,6 +77,10 @@ in
         listenPort = 8003;
         upstreamPort = 8002;
         after = [ "test2.service" ];
+        enabledAddons = [ config.shb.mitmdump.addons.logger ];
+        extraArgs = [
+          "--set" "verbose_pattern=/verbose"
+        ];
       };
     };
 
@@ -89,35 +94,44 @@ in
 
     resp = machine.succeed("curl http://127.0.0.1:8000")
     print(resp)
-    if resp != "test1":
+    if resp != "test1/":
         raise Exception("wanted 'test1'")
 
     resp = machine.succeed("curl -v http://127.0.0.1:8001")
     print(resp)
-    if resp != "test1":
+    if resp != "test1/":
         raise Exception("wanted 'test1'")
 
     resp = machine.succeed("curl http://127.0.0.1:8002")
     print(resp)
-    if resp != "test2":
+    if resp != "test2/":
         raise Exception("wanted 'test2'")
 
-    resp = machine.succeed("curl http://127.0.0.1:8003")
+    resp = machine.succeed("curl http://127.0.0.1:8003/notverbose")
     print(resp)
-    if resp != "test2":
+    if resp != "test2/notverbose":
-        raise Exception("wanted 'test2'")
+        raise Exception("wanted 'test2/notverbose'")
+
+    resp = machine.succeed("curl http://127.0.0.1:8003/verbose")
+    print(resp)
+    if resp != "test2/verbose":
+        raise Exception("wanted 'test2/verbose'")
 
     dump = machine.succeed("journalctl -b -u mitmdump-test1.service")
+    print(dump)
     if "HTTP/1.0 200 OK" not in dump:
         raise Exception("expected to see HTTP/1.0 200 OK")
     if "test1" not in dump:
         raise Exception("expected to see test1")
 
     dump = machine.succeed("journalctl -b -u mitmdump-test2.service")
+    print(dump)
     if "HTTP/1.0 200 OK" not in dump:
         raise Exception("expected to see HTTP/1.0 200 OK")
-    if "test2" not in dump:
+    if "test2/notverbose" in dump:
-        raise Exception("expected to see test2")
+        raise Exception("expected not to see test2/notverbose")
+    if "test2/verbose" not in dump:
+        raise Exception("expected to see test2/verbose")
     '';
   };
 }