From c9ee41b1d413b460bd652845484c1b1338b8ce5c Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sat, 2 Aug 2025 22:32:49 +0200 Subject: [PATCH] fine grained logging with mitmdump logger addon --- docs/redirects.json | 33 ++++ modules/blocks/mitmdump.nix | 250 ++++++++++++++++++------ modules/blocks/mitmdump/docs/default.md | 87 +++++++++ test/blocks/mitmdump.nix | 34 +++- 4 files changed, 335 insertions(+), 69 deletions(-) diff --git a/docs/redirects.json b/docs/redirects.json index d64bfe3..54b1e15 100644 --- a/docs/redirects.json +++ b/docs/redirects.json @@ -560,21 +560,45 @@ "blocks-mitmdump": [ "blocks-mitmdump.html#blocks-mitmdump" ], + "blocks-mitmdump-addons": [ + "blocks-mitmdump.html#blocks-mitmdump-addons" + ], + "blocks-mitmdump-addons-logger": [ + "blocks-mitmdump.html#blocks-mitmdump-addons-logger" + ], "blocks-mitmdump-example": [ "blocks-mitmdump.html#blocks-mitmdump-example" ], "blocks-mitmdump-options": [ "blocks-mitmdump.html#blocks-mitmdump-options" ], + "blocks-mitmdump-options-shb.mitmdump.addons": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.addons" + ], "blocks-mitmdump-options-shb.mitmdump.instances": [ "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances" ], "blocks-mitmdump-options-shb.mitmdump.instances._name_.after": [ "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.after" ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.enabledAddons": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.enabledAddons" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.extraArgs": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.extraArgs" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.listenHost": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenHost" + ], "blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort": [ "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort" ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.package": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.package" + ], + "blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName": [ + "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName" + ], "blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [ "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost" ], @@ -587,6 +611,15 @@ "blocks-mitmdump-usage": [ "blocks-mitmdump.html#blocks-mitmdump-usage" ], + "blocks-mitmdump-usage-anywhere": [ + "blocks-mitmdump.html#blocks-mitmdump-usage-anywhere" + ], + "blocks-mitmdump-usage-https": [ + "blocks-mitmdump.html#blocks-mitmdump-usage-https" + ], + "blocks-mitmdump-usage-logging": [ + "blocks-mitmdump.html#blocks-mitmdump-usage-logging" + ], "blocks-monitoring": [ "blocks-monitoring.html#blocks-monitoring" ], diff --git a/modules/blocks/mitmdump.nix b/modules/blocks/mitmdump.nix index 8df2fd1..fea9e6a 100644 --- a/modules/blocks/mitmdump.nix +++ b/modules/blocks/mitmdump.nix @@ -4,18 +4,175 @@ let inherit (types) attrsOf listOf port submodule str; cfg = config.shb.mitmdump; + + mitmdumpScript = pkgs.writers.writePython3Bin "mitmdump" + { + libraries = let + p = pkgs.python3Packages; + in [ + p.systemd + p.mitmproxy + ]; + flakeIgnore = [ "E501" ]; + } + '' + from systemd.daemon import notify + import argparse + import logging + import os + import subprocess + import socket + import sys + import time + + + logging.basicConfig(level=logging.INFO, format='%(message)s') + + + def wait_for_port(host, port, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection((host, port), timeout=0.5): + return True + except Exception: + time.sleep(0.1) + return False + + + def flatten(xss): + return [x for xs in xss for x in xs] + + + parser = argparse.ArgumentParser() + parser.add_argument("--listen_host", default="127.0.0.1", help="Host mitmdump will listen on") + parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on") + parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost") + parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream") + args, rest = parser.parse_known_args() + + MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN") + if MITMDUMP_BIN is None: + raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary") + + logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.") + wait_for_port(args.upstream_host, args.upstream_port, timeout=10) + logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.") + + proc = subprocess.Popen( + [ + MITMDUMP_BIN, + "--listen-host", args.listen_host, + "-p", args.listen_port, + "--mode", f"reverse:{args.upstream_host}:{args.upstream_port}", + ] + rest, + stdout=sys.stdout, + stderr=sys.stderr, + ) + + logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.") + if wait_for_port("127.0.0.1", args.listen_port, timeout=10): + logging.info(f"Mitmdump is started on port {args.listen_port}.") + notify("READY=1") + else: + proc.terminate() + exit(1) + + proc.wait() + ''; + + logger = toString (pkgs.writers.writeText "loggerAddon.py" + '' + import logging + from collections.abc import Sequence + from mitmproxy import ctx, http + import re + + + logger = logging.getLogger(__name__) + + + class RegexLogger: + def __init__(self): + self.verbose_patterns = None + + def load(self, loader): + loader.add_option( + name="verbose_pattern", + typespec=Sequence[str], + default=[], + help="Regex patterns for verbose logging", + ) + + def response(self, flow: http.HTTPFlow): + if self.verbose_patterns is None: + self.verbose_patterns = [re.compile(p) for p in ctx.options.verbose_pattern] + + matched = any(p.search(flow.request.path) for p in self.verbose_patterns) + if matched: + logger.info(format_flow(flow)) + + + def format_flow(flow: http.HTTPFlow) -> str: + return ( + "\n" + "RequestHeaders:\n" + f" {format_headers(flow.request.headers.items())}\n" + f"RequestBody: {flow.request.get_text()}\n" + f"Status: {flow.response.data.status_code}\n" + "ResponseHeaders:\n" + f" {format_headers(flow.response.headers.items())}\n" + f"ResponseBody: {flow.response.get_text()}\n" + ) + + + def format_headers(headers) -> str: + return "\n ".join(k + ": " + v for k, v in headers) + + + addons = [RegexLogger()] + ''); in { options.shb.mitmdump = { + addons = mkOption { + type = attrsOf str; + default = []; + description = '' + Addons available to the be added to the mitmdump instance. + + To enabled them, add them to the `enabledAddons` option. + ''; + }; + instances = mkOption { default = {}; description = "Mitmdump instance."; type = attrsOf (submodule ({ name, ... }: { options = { + package = lib.mkPackageOption pkgs "mitmproxy" {}; + + serviceName = mkOption { + type = str; + description = '' + Name of the mitmdump system service. + ''; + default = "mitmdump-${name}.service"; + readOnly = true; + }; + + listenHost = mkOption { + type = str; + default = "127.0.0.1"; + description = '' + Host the mitmdump instance will connect on. + ''; + }; + listenPort = mkOption { type = port; description = '' - Port the mitmdump instance will listen to. + Port the mitmdump instance will listen on. The upstream port from the client's perspective. ''; @@ -52,15 +209,36 @@ in when its systemd service has started. ''; }; + + enabledAddons = mkOption { + type = listOf str; + default = []; + description = '' + Addons to enable on this mitmdump instance. + ''; + example = lib.literalExpression ''[ config.shb.mitmdump.addons.logger ]''; + }; + + extraArgs = mkOption { + type = listOf str; + default = []; + description = '' + Extra arguments to pass to the mitmdump instance. + + See upstream [manual](https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) for all possible options. + ''; + example = lib.literalExpression ''[ "--set" "verbose_pattern=/api" ]''; + }; }; })); }; }; config = { - systemd.services = mapAttrs' (name: cfg: nameValuePair "mitmdump-${name}" { + systemd.services = mapAttrs' (name: cfg': nameValuePair "mitmdump-${name}" { environment = { "HOME" = "/var/lib/private/mitmdump-${name}"; + "MITMDUMP_BIN" = "${cfg'.package}/bin/mitmdump"; }; serviceConfig = { Type = "notify"; @@ -72,65 +250,19 @@ in WorkingDirectory = "/var/lib/mitmdump-${name}"; StateDirectory = "mitmdump-${name}"; - ExecStart = lib.getExe (pkgs.writers.writePython3Bin "mitmdump-${name}" - { - libraries = [ pkgs.python3Packages.systemd ]; - flakeIgnore = [ "E501" ]; - } - '' - from systemd.daemon import notify - import logging - import subprocess - import socket - import sys - import time - - - logging.basicConfig(level=logging.INFO, format='%(message)s') - - - def wait_for_port(host, port, timeout=10): - deadline = time.time() + timeout - while time.time() < deadline: - try: - with socket.create_connection((host, port), timeout=0.5): - return True - except Exception: - time.sleep(0.1) - return False - - - logging.info("Waiting for upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to be up.") - wait_for_port("${cfg.upstreamHost}", ${toString cfg.upstreamPort}, timeout=10) - logging.info("Upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to is up.") - - proc = subprocess.Popen( - [ - "${pkgs.mitmproxy}/bin/mitmdump", - "--listen-host", "127.0.0.1", - "-p", "${toString cfg.listenPort}", - "--set", "flow_detail=3", - "--set", "content_view_lines_cutoff=2000", - "--mode", "reverse:${cfg.upstreamHost}:${toString cfg.upstreamPort}", - ], - stdout=sys.stdout, - stderr=sys.stderr, - ) - - logging.info("Waiting for mitmdump instance to start on port ${toString cfg.listenPort}.") - if wait_for_port("127.0.0.1", ${toString cfg.listenPort}, timeout=10): - logging.info("Mitmdump is started on port ${toString cfg.listenPort}.") - notify("READY=1") - else: - proc.terminate() - exit(1) - - proc.wait() - ''); + ExecStart = let + addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons; + extraArgs = lib.concatStringsSep " " cfg'.extraArgs; + in + "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}"; }; - requires = cfg.after; - after = cfg.after; + requires = cfg'.after; + after = cfg'.after; wantedBy = [ "multi-user.target" ]; }) cfg.instances; + + shb.mitmdump.addons = { + inherit logger; + }; }; } diff --git a/modules/blocks/mitmdump/docs/default.md b/modules/blocks/mitmdump/docs/default.md index 5072577..7dd2283 100644 --- a/modules/blocks/mitmdump/docs/default.md +++ b/modules/blocks/mitmdump/docs/default.md @@ -14,6 +14,8 @@ and proxying to different upstream servers can be created. The systemd service is made so it is started only when the mitmdump instance has started listening on the expected port. +Also, addons can be enabled with the `enabledAddons` option. + ## Usage {#blocks-mitmdump-usage} Put mitmdump in front of a HTTP server listening on port 8000 on the same machine: @@ -40,8 +42,89 @@ shb.mitmdump.instances."my-instance" = { }; ``` +### Handle Upstream TLS {#blocks-mitmdump-usage-https} + Replace `http` with `https` if the server expects an HTTPS connection. +### Accept Connections from Anywhere {#blocks-mitmdump-usage-anywhere} + +By default, `mitmdump` is configured to listen only for connections from localhost. +Add `listenHost=0.0.0.0` to make `mitmdump` accept connections from anywhere. + +### Extra Logging {#blocks-mitmdump-usage-logging} + +To print request and response bodies and more, increase the logging with: + +```nix +extraArgs = [ + "--set" "flow_detail=3" + "--set" "content_view_lines_cutoff=2000" +]; +``` + +The default `flow_details` is 1. See the [manual][] for more explanations on the option. + +[manual]: (https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) + +This will change the verbosity for all requests and responses. +If you need more fine grained logging, configure instead the [Logger Addon][]. + +[Logger Addon]: #blocks-mitmdump-addons-logger + +## Addons {#blocks-mitmdump-addons} + +All provided addons can be found under the `shb.mitmproxy.addons` option. + +To enable one for an instance, add it to the `enabledAddons` option. For example: + +```nix +shb.mitmdump.instances."my-instance" = { + enabledAddons = [ config.shb.mitmdump.addons.logger ] +} +``` + +### Fine Grained Logger {#blocks-mitmdump-addons-logger} + +The Fine Grained Logger addon is found under `shb.mitmproxy.addons.logger`. +Enabling this addon will add the `mitmdump` option `verbose_pattern` which takes a regex and if it matches, +prints the request and response headers and body. +If it does not match, it will just print the response status. + +For example, with the `extraArgs`: + +```nix +extraArgs = [ + "--set" "verbose_pattern=/verbose" +]; +``` + +A `GET` request to `/notverbose` will print something similar to: + +``` +mitmdump[972]: 127.0.0.1:53586: GET http://127.0.0.1:8000/notverbose HTTP/1.1 +mitmdump[972]: << HTTP/1.0 200 OK 16b +``` + +While a `GET` request to `/verbose` will print something similar to: + +``` +mitmdump[972]: [22:42:58.840] +mitmdump[972]: RequestHeaders: +mitmdump[972]: Host: 127.0.0.1:8000 +mitmdump[972]: User-Agent: curl/8.14.1 +mitmdump[972]: Accept: */* +mitmdump[972]: RequestBody: +mitmdump[972]: Status: 200 +mitmdump[972]: ResponseHeaders: +mitmdump[972]: Server: BaseHTTP/0.6 Python/3.13.4 +mitmdump[972]: Date: Sun, 03 Aug 2025 22:42:58 GMT +mitmdump[972]: Content-Type: text/plain +mitmdump[972]: Content-Length: 13 +mitmdump[972]: ResponseBody: test2/verbose +mitmdump[972]: 127.0.0.1:53602: GET http://127.0.0.1:8000/verbose HTTP/1.1 +mitmdump[972]: << HTTP/1.0 200 OK 13b +``` + ## Example {#blocks-mitmdump-example} Let's assume a server is listening on port 8000 @@ -56,6 +139,10 @@ shb.mitmdump.instances."test1" = { listenPort = 8001; upstreamPort = 8000; after = [ "test1.service" ]; + extraArgs = [ + "--set" "flow_detail=3" + "--set" "content_view_lines_cutoff=2000" + ]; }; ``` diff --git a/test/blocks/mitmdump.nix b/test/blocks/mitmdump.nix index 1571a32..03048cb 100644 --- a/test/blocks/mitmdump.nix +++ b/test/blocks/mitmdump.nix @@ -16,12 +16,13 @@ let class HardcodedHandler(BaseHTTPRequestHandler): def do_GET(self): + reponse = content + self.path.encode('utf-8') self.send_response(200) self.send_header("Content-Type", "text/plain") - self.send_header("Content-Length", str(len(content))) + self.send_header("Content-Length", str(len(reponse))) self.end_headers() print("answering to GET request") - self.wfile.write(content) + self.wfile.write(reponse) def log_message(self, format, *args): pass # optional: suppress logging @@ -76,6 +77,10 @@ in listenPort = 8003; upstreamPort = 8002; after = [ "test2.service" ]; + enabledAddons = [ config.shb.mitmdump.addons.logger ]; + extraArgs = [ + "--set" "verbose_pattern=/verbose" + ]; }; }; @@ -89,35 +94,44 @@ in resp = machine.succeed("curl http://127.0.0.1:8000") print(resp) - if resp != "test1": + if resp != "test1/": raise Exception("wanted 'test1'") resp = machine.succeed("curl -v http://127.0.0.1:8001") print(resp) - if resp != "test1": + if resp != "test1/": raise Exception("wanted 'test1'") resp = machine.succeed("curl http://127.0.0.1:8002") print(resp) - if resp != "test2": + if resp != "test2/": raise Exception("wanted 'test2'") - resp = machine.succeed("curl http://127.0.0.1:8003") + resp = machine.succeed("curl http://127.0.0.1:8003/notverbose") print(resp) - if resp != "test2": - raise Exception("wanted 'test2'") + if resp != "test2/notverbose": + raise Exception("wanted 'test2/notverbose'") + + resp = machine.succeed("curl http://127.0.0.1:8003/verbose") + print(resp) + if resp != "test2/verbose": + raise Exception("wanted 'test2/verbose'") dump = machine.succeed("journalctl -b -u mitmdump-test1.service") + print(dump) if "HTTP/1.0 200 OK" not in dump: raise Exception("expected to see HTTP/1.0 200 OK") if "test1" not in dump: raise Exception("expected to see test1") dump = machine.succeed("journalctl -b -u mitmdump-test2.service") + print(dump) if "HTTP/1.0 200 OK" not in dump: raise Exception("expected to see HTTP/1.0 200 OK") - if "test2" not in dump: - raise Exception("expected to see test2") + if "test2/notverbose" in dump: + raise Exception("expected not to see test2/notverbose") + if "test2/verbose" not in dump: + raise Exception("expected to see test2/verbose") ''; }; }