add mitmdump block
This commit is contained in:
ibizaman
Pierre Penninckx
parent
91580421be
commit
67c2705f98
6 changed files with 390 additions and 2 deletions
|
|
@ -5,7 +5,7 @@ Blocks help you self-host apps or services. They implement a specific function l
|
|||
access through a subdomain. Each block is designed to be usable on its own and to fit nicely with
|
||||
others.
|
||||
|
||||
In practice, a block implements a [contract](contracts.html) that must be followed to implement a
|
||||
In practice, most blocks implement a [contract](contracts.html) that must be followed to implement a
|
||||
specific self-hosting function. It also comes with a unit test and NixOS VM test suite to ensure any
|
||||
implementation follows the contract.
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ It defines no outputs but has one major side effect:
|
|||
Anything that provides the inputs and expected outputs and side effects defined by the block can be
|
||||
used to fulfill its contract. In this example, we could use any of Nginx, Caddy, Haproxy or others.
|
||||
|
||||
Self Host Blocks provides at least one implementation for each block and allows you to use your own
|
||||
Self Host Blocks provides at least one implementation for each contract and allows you to use your own
|
||||
implementation if you want to, as long as it passes the tests. You can then use blocks to improve
|
||||
services you already have deployed.
|
||||
|
||||
|
|
@ -39,6 +39,10 @@ modules/blocks/authelia/docs/default.md
|
|||
modules/blocks/lldap/docs/default.md
|
||||
```
|
||||
|
||||
```{=include=} chapters html:into-file=//blocks-mitmdump.html
|
||||
modules/blocks/mitmdump/docs/default.md
|
||||
```
|
||||
|
||||
```{=include=} chapters html:into-file=//blocks-sops.html
|
||||
modules/blocks/sops/docs/default.md
|
||||
```
|
||||
|
|
|
|||
|
|
@ -557,6 +557,36 @@
|
|||
"blocks-lldap-troubleshooting": [
|
||||
"blocks-lldap.html#blocks-lldap-troubleshooting"
|
||||
],
|
||||
"blocks-mitmdump": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump"
|
||||
],
|
||||
"blocks-mitmdump-example": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-example"
|
||||
],
|
||||
"blocks-mitmdump-options": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-options"
|
||||
],
|
||||
"blocks-mitmdump-options-shb.mitmdump.instances": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances"
|
||||
],
|
||||
"blocks-mitmdump-options-shb.mitmdump.instances._name_.after": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.after"
|
||||
],
|
||||
"blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.listenPort"
|
||||
],
|
||||
"blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost"
|
||||
],
|
||||
"blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamPort": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamPort"
|
||||
],
|
||||
"blocks-mitmdump-tests": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-tests"
|
||||
],
|
||||
"blocks-mitmdump-usage": [
|
||||
"blocks-mitmdump.html#blocks-mitmdump-usage"
|
||||
],
|
||||
"blocks-monitoring": [
|
||||
"blocks-monitoring.html#blocks-monitoring"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
self.nixosModules.${system}.davfs
|
||||
self.nixosModules.${system}.hardcodedsecret
|
||||
self.nixosModules.${system}.lldap
|
||||
self.nixosModules.${system}.mitmdump
|
||||
self.nixosModules.${system}.monitoring
|
||||
self.nixosModules.${system}.nginx
|
||||
self.nixosModules.${system}.postgresql
|
||||
|
|
@ -94,6 +95,7 @@
|
|||
nixosModules.davfs = modules/blocks/davfs.nix;
|
||||
nixosModules.hardcodedsecret = modules/blocks/hardcodedsecret.nix;
|
||||
nixosModules.lldap = modules/blocks/lldap.nix;
|
||||
nixosModules.mitmdump = modules/blocks/mitmdump.nix;
|
||||
nixosModules.monitoring = modules/blocks/monitoring.nix;
|
||||
nixosModules.nginx = modules/blocks/nginx.nix;
|
||||
nixosModules.postgresql = modules/blocks/postgresql.nix;
|
||||
|
|
@ -128,6 +130,7 @@
|
|||
"blocks/authelia" = ./modules/blocks/authelia.nix;
|
||||
"blocks/lldap" = ./modules/blocks/lldap.nix;
|
||||
"blocks/ssl" = ./modules/blocks/ssl.nix;
|
||||
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
|
||||
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
|
||||
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
|
||||
"blocks/restic" = ./modules/blocks/restic.nix;
|
||||
|
|
@ -261,6 +264,7 @@
|
|||
// (vm_test "authelia" ./test/blocks/authelia.nix)
|
||||
// (vm_test "lldap" ./test/blocks/lldap.nix)
|
||||
// (vm_test "lib" ./test/blocks/lib.nix)
|
||||
// (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
|
||||
// (vm_test "postgresql" ./test/blocks/postgresql.nix)
|
||||
// (vm_test "restic" ./test/blocks/restic.nix)
|
||||
// (vm_test "ssl" ./test/blocks/ssl.nix)
|
||||
|
|
|
|||
136
modules/blocks/mitmdump.nix
Normal file
136
modules/blocks/mitmdump.nix
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
inherit (lib) mapAttrs' mkOption nameValuePair types;
|
||||
inherit (types) attrsOf listOf port submodule str;
|
||||
|
||||
cfg = config.shb.mitmdump;
|
||||
in
|
||||
{
|
||||
options.shb.mitmdump = {
|
||||
instances = mkOption {
|
||||
default = {};
|
||||
description = "Mitmdump instance.";
|
||||
type = attrsOf (submodule ({ name, ... }: {
|
||||
options = {
|
||||
listenPort = mkOption {
|
||||
type = port;
|
||||
description = ''
|
||||
Port the mitmdump instance will listen to.
|
||||
|
||||
The upstream port from the client's perspective.
|
||||
'';
|
||||
};
|
||||
|
||||
upstreamHost = mkOption {
|
||||
type = str;
|
||||
default = "http://127.0.0.1";
|
||||
description = ''
|
||||
Host the mitmdump instance will connect to.
|
||||
|
||||
If only an IP or domain is provided,
|
||||
mitmdump will default to connect using HTTPS.
|
||||
If this is not wanted, prefix the IP or domain with the 'http://' protocol.
|
||||
'';
|
||||
};
|
||||
|
||||
upstreamPort = mkOption {
|
||||
type = port;
|
||||
description = ''
|
||||
Port the mitmdump instance will connect to.
|
||||
|
||||
The port the server is listening on.
|
||||
'';
|
||||
};
|
||||
|
||||
after = mkOption {
|
||||
type = listOf str;
|
||||
default = [];
|
||||
description = ''
|
||||
Systemd services that must be started before this mitmdump proxy instance.
|
||||
|
||||
You are guaranteed the mitmdump is listening on the `listenPort`
|
||||
when its systemd service has started.
|
||||
'';
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
systemd.services = mapAttrs' (name: cfg: nameValuePair "mitmdump-${name}" {
|
||||
environment = {
|
||||
"HOME" = "/var/lib/private/mitmdump-${name}";
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "notify";
|
||||
Restart = "on-failure";
|
||||
StandardOutput = "journal";
|
||||
StandardError = "journal";
|
||||
|
||||
DynamicUser = true;
|
||||
WorkingDirectory = "/var/lib/mitmdump-${name}";
|
||||
StateDirectory = "mitmdump-${name}";
|
||||
|
||||
ExecStart = lib.getExe (pkgs.writers.writePython3Bin "mitmdump-${name}"
|
||||
{
|
||||
libraries = [ pkgs.python3Packages.systemd ];
|
||||
flakeIgnore = [ "E501" ];
|
||||
}
|
||||
''
|
||||
from systemd.daemon import notify
|
||||
import logging
|
||||
import subprocess
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
|
||||
|
||||
def wait_for_port(host, port, timeout=10):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=0.5):
|
||||
return True
|
||||
except Exception:
|
||||
time.sleep(0.1)
|
||||
return False
|
||||
|
||||
|
||||
logging.info("Waiting for upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to be up.")
|
||||
wait_for_port("${cfg.upstreamHost}", ${toString cfg.upstreamPort}, timeout=10)
|
||||
logging.info("Upstream address '${cfg.upstreamHost}:${toString cfg.upstreamPort}' to is up.")
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"${pkgs.mitmproxy}/bin/mitmdump",
|
||||
"--listen-host", "127.0.0.1",
|
||||
"-p", "${toString cfg.listenPort}",
|
||||
"--set", "flow_detail=3",
|
||||
"--set", "content_view_lines_cutoff=2000",
|
||||
"--mode", "reverse:${cfg.upstreamHost}:${toString cfg.upstreamPort}",
|
||||
],
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
)
|
||||
|
||||
logging.info("Waiting for mitmdump instance to start on port ${toString cfg.listenPort}.")
|
||||
if wait_for_port("127.0.0.1", ${toString cfg.listenPort}, timeout=10):
|
||||
logging.info("Mitmdump is started on port ${toString cfg.listenPort}.")
|
||||
notify("READY=1")
|
||||
else:
|
||||
proc.terminate()
|
||||
exit(1)
|
||||
|
||||
proc.wait()
|
||||
'');
|
||||
};
|
||||
requires = cfg.after;
|
||||
after = cfg.after;
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
}) cfg.instances;
|
||||
};
|
||||
}
|
||||
91
modules/blocks/mitmdump/docs/default.md
Normal file
91
modules/blocks/mitmdump/docs/default.md
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# Mitmdump Block {#blocks-mitmdump}
|
||||
|
||||
Defined in [`/modules/blocks/authelia.nix`](@REPO@/modules/blocks/authelia.nix).
|
||||
|
||||
This block sets up an [Mitmdump][] service in [reverse proxy][] mode.
|
||||
In other words, you can put this block between a client and a server to inspect all the network traffic.
|
||||
|
||||
[Mitmdump]: https://plattner.me/mp-docs/#mitmdump
|
||||
[reverse proxy]: https://plattner.me/mp-docs/concepts-modes/#reverse-proxy
|
||||
|
||||
Multiple instances of mitmdump all listening on different ports
|
||||
and proxying to different upstream servers can be created.
|
||||
|
||||
The systemd service is made so it is started only when the mitmdump instance
|
||||
has started listening on the expected port.
|
||||
|
||||
## Usage {#blocks-mitmdump-usage}
|
||||
|
||||
Put mitmdump in front of a HTTP server listening on port 8000 on the same machine:
|
||||
|
||||
```nix
|
||||
shb.mitmdump.instances."my-instance" = {
|
||||
listenPort = 8001;
|
||||
upstreamHost = "http://127.0.0.1";
|
||||
upstreamPort = 8000;
|
||||
after = [ "server.service" ];
|
||||
};
|
||||
```
|
||||
|
||||
`upstreamHost` is the default here and can be left out.
|
||||
|
||||
Put mitmdump in front of a HTTP server listening on port 8000 on another machine:
|
||||
|
||||
```nix
|
||||
shb.mitmdump.instances."my-instance" = {
|
||||
listenPort = 8001;
|
||||
upstreamHost = "http://otherhost";
|
||||
upstreamPort = 8000;
|
||||
after = [ "server.service" ];
|
||||
};
|
||||
```
|
||||
|
||||
Replace `http` with `https` if the server expects an HTTPS connection.
|
||||
|
||||
## Example {#blocks-mitmdump-example}
|
||||
|
||||
Let's assume a server is listening on port 8000
|
||||
which responds a plain text response `test1`
|
||||
and its related systemd service is named `test1.service`.
|
||||
Sorry, creative naming is not my forte.
|
||||
|
||||
Let's put an mitmdump instance in front of it, like so:
|
||||
|
||||
```nix
|
||||
shb.mitmdump.instances."test1" = {
|
||||
listenPort = 8001;
|
||||
upstreamPort = 8000;
|
||||
after = [ "test1.service" ];
|
||||
};
|
||||
```
|
||||
|
||||
This creates an `mitmdump-test1.service` systemd service.
|
||||
We can then use `journalctl -u mitmdump-test1.service` to see the output.
|
||||
|
||||
If we make a `curl` request to it: `curl -v http://127.0.0.1:8001`,
|
||||
we will get the following output:
|
||||
|
||||
```
|
||||
mitmdump-test1[971]: 127.0.0.1:40878: GET http://127.0.0.1:8000/ HTTP/1.1
|
||||
mitmdump-test1[971]: Host: 127.0.0.1:8000
|
||||
mitmdump-test1[971]: User-Agent: curl/8.14.1
|
||||
mitmdump-test1[971]: Accept: */*
|
||||
mitmdump-test1[971]: << HTTP/1.0 200 OK 5b
|
||||
mitmdump-test1[971]: Server: BaseHTTP/0.6 Python/3.13.4
|
||||
mitmdump-test1[971]: Date: Thu, 31 Jul 2025 20:55:16 GMT
|
||||
mitmdump-test1[971]: Content-Type: text/plain
|
||||
mitmdump-test1[971]: Content-Length: 5
|
||||
mitmdump-test1[971]: test1
|
||||
```
|
||||
|
||||
## Tests {#blocks-mitmdump-tests}
|
||||
|
||||
Specific integration tests are defined in [`/test/blocks/mitmdump.nix`](@REPO@/test/blocks/mitmdump.nix).
|
||||
|
||||
## Options Reference {#blocks-mitmdump-options}
|
||||
|
||||
```{=include=} options
|
||||
id-prefix: blocks-mitmdump-options-
|
||||
list-id: selfhostblocks-block-mitmdump-options
|
||||
source: @OPTIONS_JSON@
|
||||
```
|
||||
123
test/blocks/mitmdump.nix
Normal file
123
test/blocks/mitmdump.nix
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
{ pkgs, lib, ... }:
|
||||
let
|
||||
serve = port: text: lib.getExe (pkgs.writers.writePython3Bin "serve"
|
||||
{
|
||||
libraries = [ pkgs.python3Packages.systemd ];
|
||||
}
|
||||
(let
|
||||
content = pkgs.writeText "content" text;
|
||||
in ''
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from systemd.daemon import notify
|
||||
|
||||
with open("${content}", "rb") as f:
|
||||
content = f.read()
|
||||
|
||||
|
||||
class HardcodedHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.send_header("Content-Length", str(len(content)))
|
||||
self.end_headers()
|
||||
print("answering to GET request")
|
||||
self.wfile.write(content)
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass # optional: suppress logging
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
notify('STATUS=Starting up...')
|
||||
server_address = ('127.0.0.1', ${toString port})
|
||||
httpd = HTTPServer(server_address, HardcodedHandler)
|
||||
print("Serving hardcoded page on http://127.0.0.1:${toString port}")
|
||||
notify('READY=1')
|
||||
httpd.serve_forever()
|
||||
'')
|
||||
);
|
||||
in
|
||||
{
|
||||
default = pkgs.testers.runNixOSTest {
|
||||
name = "mitmdump-default";
|
||||
|
||||
nodes.machine = { config, pkgs, ... }: {
|
||||
imports = [
|
||||
../../modules/blocks/mitmdump.nix
|
||||
];
|
||||
|
||||
systemd.services.test1 = {
|
||||
serviceConfig.ExecStart = serve 8000 "test1";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "notify";
|
||||
StandardOutput = "journal";
|
||||
StandardError = "journal";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.test2 = {
|
||||
serviceConfig.ExecStart = serve 8002 "test2";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "notify";
|
||||
StandardOutput = "journal";
|
||||
StandardError = "journal";
|
||||
};
|
||||
};
|
||||
|
||||
shb.mitmdump.instances."test1" = {
|
||||
listenPort = 8001;
|
||||
upstreamPort = 8000;
|
||||
after = [ "test1.service" ];
|
||||
};
|
||||
|
||||
shb.mitmdump.instances."test2" = {
|
||||
listenPort = 8003;
|
||||
upstreamPort = 8002;
|
||||
after = [ "test2.service" ];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = { nodes, ... }: ''
|
||||
start_all()
|
||||
|
||||
machine.wait_for_unit("test1.service")
|
||||
machine.wait_for_unit("test2.service")
|
||||
machine.wait_for_unit("mitmdump-test1.service")
|
||||
machine.wait_for_unit("mitmdump-test2.service")
|
||||
|
||||
resp = machine.succeed("curl http://127.0.0.1:8000")
|
||||
print(resp)
|
||||
if resp != "test1":
|
||||
raise Exception("wanted 'test1'")
|
||||
|
||||
resp = machine.succeed("curl -v http://127.0.0.1:8001")
|
||||
print(resp)
|
||||
if resp != "test1":
|
||||
raise Exception("wanted 'test1'")
|
||||
|
||||
resp = machine.succeed("curl http://127.0.0.1:8002")
|
||||
print(resp)
|
||||
if resp != "test2":
|
||||
raise Exception("wanted 'test2'")
|
||||
|
||||
resp = machine.succeed("curl http://127.0.0.1:8003")
|
||||
print(resp)
|
||||
if resp != "test2":
|
||||
raise Exception("wanted 'test2'")
|
||||
|
||||
dump = machine.succeed("journalctl -b -u mitmdump-test1.service")
|
||||
if "HTTP/1.0 200 OK" not in dump:
|
||||
raise Exception("expected to see HTTP/1.0 200 OK")
|
||||
if "test1" not in dump:
|
||||
raise Exception("expected to see test1")
|
||||
|
||||
dump = machine.succeed("journalctl -b -u mitmdump-test2.service")
|
||||
if "HTTP/1.0 200 OK" not in dump:
|
||||
raise Exception("expected to see HTTP/1.0 200 OK")
|
||||
if "test2" not in dump:
|
||||
raise Exception("expected to see test2")
|
||||
'';
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue