danvics/selfhostblocks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fine grained logging with mitmdump logger addon

Browse source
This commit is contained in:
ibizaman 2025-08-02 22:32:49 +02:00 committed by Pierre Penninckx
parent 9a135523d2
commit c9ee41b1d4
4 changed files with 335 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
docs/redirects.json
View file

@ -560,21 +560,45 @@
"blocks-mitmdump": [
"blocks-mitmdump.html#blocks-mitmdump"
],
"blocks-mitmdump-addons": [
"blocks-mitmdump.html#blocks-mitmdump-addons"
],
"blocks-mitmdump-addons-logger": [
"blocks-mitmdump.html#blocks-mitmdump-addons-logger"
],
"blocks-mitmdump-example": [
"blocks-mitmdump.html#blocks-mitmdump-example"
],
"blocks-mitmdump-options": [
"blocks-mitmdump.html#blocks-mitmdump-options"
],
"blocks-mitmdump-options-shb.mitmdump.addons": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.addons"
],
"blocks-mitmdump-options-shb.mitmdump.instances": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.after": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.after"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.enabledAddons": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.enabledAddons"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.extraArgs": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.extraArgs"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.listenHost": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenHost"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.package": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.package"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost"
],
@ -587,6 +611,15 @@
"blocks-mitmdump-usage": [
"blocks-mitmdump.html#blocks-mitmdump-usage"
],
"blocks-mitmdump-usage-anywhere": [
"blocks-mitmdump.html#blocks-mitmdump-usage-anywhere"
],
"blocks-mitmdump-usage-https": [
"blocks-mitmdump.html#blocks-mitmdump-usage-https"
],
"blocks-mitmdump-usage-logging": [
"blocks-mitmdump.html#blocks-mitmdump-usage-logging"
],
"blocks-monitoring": [
"blocks-monitoring.html#blocks-monitoring"
],

250
modules/blocks/mitmdump.nix
View file

@ -4,18 +4,175 @@ let
inherit (types) attrsOf listOf port submodule str;
cfg = config.shb.mitmdump;
mitmdumpScript = pkgs.writers.writePython3Bin "mitmdump"
{
libraries = let
p = pkgs.python3Packages;
in [
p.systemd
p.mitmproxy
];
flakeIgnore = [ "E501" ];
}
''
from systemd.daemon import notify
import argparse
import logging
import os
import subprocess
import socket
import sys
import time
logging.basicConfig(level=logging.INFO, format='%(message)s')
def wait_for_port(host, port, timeout=10):
deadline = time.time() + timeout
while time.time() < deadline:
try:
with socket.create_connection((host, port), timeout=0.5):
return True
except Exception:
time.sleep(0.1)
return False
def flatten(xss):
return [x for xs in xss for x in xs]
parser = argparse.ArgumentParser()
parser.add_argument("--listen_host", default="127.0.0.1", help="Host mitmdump will listen on")
parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
args, rest = parser.parse_known_args()
MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
if MITMDUMP_BIN is None:
raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
wait_for_port(args.upstream_host, args.upstream_port, timeout=10)
logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
proc = subprocess.Popen(
[
MITMDUMP_BIN,
"--listen-host", args.listen_host,
"-p", args.listen_port,
"--mode", f"reverse:{args.upstream_host}:{args.upstream_port}",
] + rest,
stdout=sys.stdout,
stderr=sys.stderr,
)
logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
if wait_for_port("127.0.0.1", args.listen_port, timeout=10):
logging.info(f"Mitmdump is started on port {args.listen_port}.")
notify("READY=1")
else:
proc.terminate()
exit(1)
proc.wait()
'';
logger = toString (pkgs.writers.writeText "loggerAddon.py"
''
import logging
from collections.abc import Sequence
from mitmproxy import ctx, http
import re
logger = logging.getLogger(__name__)
class RegexLogger:
def __init__(self):
self.verbose_patterns = None
def load(self, loader):
loader.add_option(
name="verbose_pattern",
typespec=Sequence[str],
default=[],
help="Regex patterns for verbose logging",
)
def response(self, flow: http.HTTPFlow):
if self.verbose_patterns is None:
self.verbose_patterns = [re.compile(p) for p in ctx.options.verbose_pattern]
matched = any(p.search(flow.request.path) for p in self.verbose_patterns)
if matched:
logger.info(format_flow(flow))
def format_flow(flow: http.HTTPFlow) -> str:
return (
"\n"
"RequestHeaders:\n"
f" {format_headers(flow.request.headers.items())}\n"
f"RequestBody: {flow.request.get_text()}\n"
f"Status: {flow.response.data.status_code}\n"
"ResponseHeaders:\n"
f" {format_headers(flow.response.headers.items())}\n"
f"ResponseBody: {flow.response.get_text()}\n"
)
def format_headers(headers) -> str:
return "\n ".join(k + ": " + v for k, v in headers)
addons = [RegexLogger()]
'');
in
{
options.shb.mitmdump = {
addons = mkOption {
type = attrsOf str;
default = [];
description = ''
Addons available to the be added to the mitmdump instance.
To enabled them, add them to the `enabledAddons` option.
'';
};
instances = mkOption {
default = {};
description = "Mitmdump instance.";
type = attrsOf (submodule ({ name, ... }: {
options = {
package = lib.mkPackageOption pkgs "mitmproxy" {};
serviceName = mkOption {
type = str;
description = ''
Name of the mitmdump system service.
'';
default = "mitmdump-${name}.service";
readOnly = true;
};
listenHost = mkOption {
type = str;
default = "127.0.0.1";
description = ''
Host the mitmdump instance will connect on.
'';
};
listenPort = mkOption {
type = port;
description = ''
Port the mitmdump instance will listen to.
Port the mitmdump instance will listen on.
The upstream port from the client's perspective.
'';
@ -52,15 +209,36 @@ in
when its systemd service has started.
'';
};
enabledAddons = mkOption {
type = listOf str;
default = [];
description = ''
Addons to enable on this mitmdump instance.
'';
example = lib.literalExpression ''[ config.shb.mitmdump.addons.logger ]'';
};
extraArgs = mkOption {
type = listOf str;
default = [];
description = ''
Extra arguments to pass to the mitmdump instance.
See upstream [manual](https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) for all possible options.
'';
example = lib.literalExpression ''[ "--set" "verbose_pattern=/api" ]'';
};
};
}));
};
};
config = {
systemd.services = mapAttrs' (name: cfg: nameValuePair "mitmdump-${name}" {
systemd.services = mapAttrs' (name: cfg': nameValuePair "mitmdump-${name}" {
environment = {
"HOME" = "/var/lib/private/mitmdump-${name}";
"MITMDUMP_BIN" = "${cfg'.package}/bin/mitmdump";
};
serviceConfig = {
Type = "notify";
@ -72,65 +250,19 @@ in
WorkingDirectory = "/var/lib/mitmdump-${name}";
StateDirectory = "mitmdump-${name}";
ExecStart = lib.getExe (pkgs.writers.writePython3Bin "mitmdump-${name}"
{
libraries = [ pkgs.python3Packages.systemd ];
flakeIgnore = [ "E501" ];
}
''
from systemd.daemon import notify
import logging
import subprocess
import socket
import sys
import time
logging.basicConfig(level=logging.INFO, format='%(message)s')
def wait_for_port(host, port, timeout=10):
deadline = time.time() + timeout
while time.time() < deadline:
try:
with socket.create_connection((host, port), timeout=0.5):
return True
except Exception:
time.sleep(0.1)
return False
logging.info("Waiting for upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to be up.")
wait_for_port("${cfg.upstreamHost}", ${toString cfg.upstreamPort}, timeout=10)
logging.info("Upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to is up.")
proc = subprocess.Popen(
[
"${pkgs.mitmproxy}/bin/mitmdump",
"--listen-host", "127.0.0.1",
"-p", "${toString cfg.listenPort}",
"--set", "flow_detail=3",
"--set", "content_view_lines_cutoff=2000",
"--mode", "reverse:${cfg.upstreamHost}:${toString cfg.upstreamPort}",
],
stdout=sys.stdout,
stderr=sys.stderr,
)
logging.info("Waiting for mitmdump instance to start on port ${toString cfg.listenPort}.")
if wait_for_port("127.0.0.1", ${toString cfg.listenPort}, timeout=10):
logging.info("Mitmdump is started on port ${toString cfg.listenPort}.")
notify("READY=1")
else:
proc.terminate()
exit(1)
proc.wait()
'');
ExecStart = let
addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
in
"${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}";
};
requires = cfg.after;
after = cfg.after;
requires = cfg'.after;
after = cfg'.after;
wantedBy = [ "multi-user.target" ];
}) cfg.instances;
shb.mitmdump.addons = {
inherit logger;
};
};
}

87
modules/blocks/mitmdump/docs/default.md
View file

@ -14,6 +14,8 @@ and proxying to different upstream servers can be created.
The systemd service is made so it is started only when the mitmdump instance
has started listening on the expected port.
Also, addons can be enabled with the `enabledAddons` option.
## Usage {#blocks-mitmdump-usage}
Put mitmdump in front of a HTTP server listening on port 8000 on the same machine:
@ -40,8 +42,89 @@ shb.mitmdump.instances."my-instance" = {
};
```
### Handle Upstream TLS {#blocks-mitmdump-usage-https}
Replace `http` with `https` if the server expects an HTTPS connection.
### Accept Connections from Anywhere {#blocks-mitmdump-usage-anywhere}
By default, `mitmdump` is configured to listen only for connections from localhost.
Add `listenHost=0.0.0.0` to make `mitmdump` accept connections from anywhere.
### Extra Logging {#blocks-mitmdump-usage-logging}
To print request and response bodies and more, increase the logging with:
```nix
extraArgs = [
"--set" "flow_detail=3"
"--set" "content_view_lines_cutoff=2000"
];
```
The default `flow_details` is 1. See the [manual][] for more explanations on the option.
[manual]: (https://docs.mitmproxy.org/stable/concepts/options/#flow_detail)
This will change the verbosity for all requests and responses.
If you need more fine grained logging, configure instead the [Logger Addon][].
[Logger Addon]: #blocks-mitmdump-addons-logger
## Addons {#blocks-mitmdump-addons}
All provided addons can be found under the `shb.mitmproxy.addons` option.
To enable one for an instance, add it to the `enabledAddons` option. For example:
```nix
shb.mitmdump.instances."my-instance" = {
enabledAddons = [ config.shb.mitmdump.addons.logger ]
}
```
### Fine Grained Logger {#blocks-mitmdump-addons-logger}
The Fine Grained Logger addon is found under `shb.mitmproxy.addons.logger`.
Enabling this addon will add the `mitmdump` option `verbose_pattern` which takes a regex and if it matches,
prints the request and response headers and body.
If it does not match, it will just print the response status.
For example, with the `extraArgs`:
```nix
extraArgs = [
"--set" "verbose_pattern=/verbose"
];
```
A `GET` request to `/notverbose` will print something similar to:
```
mitmdump[972]: 127.0.0.1:53586: GET http://127.0.0.1:8000/notverbose HTTP/1.1
mitmdump[972]: << HTTP/1.0 200 OK 16b
```
While a `GET` request to `/verbose` will print something similar to:
```
mitmdump[972]: [22:42:58.840]
mitmdump[972]: RequestHeaders:
mitmdump[972]: Host: 127.0.0.1:8000
mitmdump[972]: User-Agent: curl/8.14.1
mitmdump[972]: Accept: */*
mitmdump[972]: RequestBody:
mitmdump[972]: Status: 200
mitmdump[972]: ResponseHeaders:
mitmdump[972]: Server: BaseHTTP/0.6 Python/3.13.4
mitmdump[972]: Date: Sun, 03 Aug 2025 22:42:58 GMT
mitmdump[972]: Content-Type: text/plain
mitmdump[972]: Content-Length: 13
mitmdump[972]: ResponseBody: test2/verbose
mitmdump[972]: 127.0.0.1:53602: GET http://127.0.0.1:8000/verbose HTTP/1.1
mitmdump[972]: << HTTP/1.0 200 OK 13b
```
## Example {#blocks-mitmdump-example}
Let's assume a server is listening on port 8000
@ -56,6 +139,10 @@ shb.mitmdump.instances."test1" = {
listenPort = 8001;
upstreamPort = 8000;
after = [ "test1.service" ];
extraArgs = [
"--set" "flow_detail=3"
"--set" "content_view_lines_cutoff=2000"
];
};
```

34
test/blocks/mitmdump.nix
View file

@ -16,12 +16,13 @@ let
class HardcodedHandler(BaseHTTPRequestHandler):
def do_GET(self):
reponse = content + self.path.encode('utf-8')
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", str(len(content)))
self.send_header("Content-Length", str(len(reponse)))
self.end_headers()
print("answering to GET request")
self.wfile.write(content)
self.wfile.write(reponse)
def log_message(self, format, *args):
pass # optional: suppress logging
@ -76,6 +77,10 @@ in
listenPort = 8003;
upstreamPort = 8002;
after = [ "test2.service" ];
enabledAddons = [ config.shb.mitmdump.addons.logger ];
extraArgs = [
"--set" "verbose_pattern=/verbose"
];
};
};
@ -89,35 +94,44 @@ in
resp = machine.succeed("curl http://127.0.0.1:8000")
print(resp)
if resp != "test1":
if resp != "test1/":
raise Exception("wanted 'test1'")
resp = machine.succeed("curl -v http://127.0.0.1:8001")
print(resp)
if resp != "test1":
if resp != "test1/":
raise Exception("wanted 'test1'")
resp = machine.succeed("curl http://127.0.0.1:8002")
print(resp)
if resp != "test2":
if resp != "test2/":
raise Exception("wanted 'test2'")
resp = machine.succeed("curl http://127.0.0.1:8003")
resp = machine.succeed("curl http://127.0.0.1:8003/notverbose")
print(resp)
if resp != "test2":
raise Exception("wanted 'test2'")
if resp != "test2/notverbose":
raise Exception("wanted 'test2/notverbose'")
resp = machine.succeed("curl http://127.0.0.1:8003/verbose")
print(resp)
if resp != "test2/verbose":
raise Exception("wanted 'test2/verbose'")
dump = machine.succeed("journalctl -b -u mitmdump-test1.service")
print(dump)
if "HTTP/1.0 200 OK" not in dump:
raise Exception("expected to see HTTP/1.0 200 OK")
if "test1" not in dump:
raise Exception("expected to see test1")
dump = machine.succeed("journalctl -b -u mitmdump-test2.service")
print(dump)
if "HTTP/1.0 200 OK" not in dump:
raise Exception("expected to see HTTP/1.0 200 OK")
if "test2" not in dump:
raise Exception("expected to see test2")
if "test2/notverbose" in dump:
raise Exception("expected not to see test2/notverbose")
if "test2/verbose" not in dump:
raise Exception("expected to see test2/verbose")
'';
};
}