477 lines
17 KiB
Python
477 lines
17 KiB
Python
"""Ephemeral FIPS daemon lifecycle manager.
|
|
|
|
Generates an isolated ``fips.yaml``, spawns ``fipsd --config <path>`` as a
|
|
subprocess **inside its own network namespace**, polls the control socket
|
|
until the daemon is ready, and tears everything down cleanly (process +
|
|
netns + temp dir).
|
|
|
|
Why network namespaces: the FIPS daemon hardcodes a ``fd00::/8`` route on
|
|
its TUN device (see ``fips/src/upper/tun.rs``). On a host that already runs
|
|
a long-lived FIPS daemon (``fips0``), that route already exists, so a
|
|
second daemon's TUN init fails with ``File exists (os error 17)``. Running
|
|
each ephemeral daemon in its own netns gives it a fresh routing table where
|
|
``fd00::/8`` is free. The control socket is a Unix domain socket in ``/tmp``
|
|
(shared filesystem), so it remains reachable from the host.
|
|
|
|
Each ephemeral daemon gets:
|
|
- a unique temp config dir under ``/tmp/mesh-scan-<tag>-<pid>/``
|
|
- a unique control socket path inside that dir
|
|
- a unique network namespace (``fips-scan-<n>``) — or a shared one for
|
|
the self-test (so two daemons can peer over loopback)
|
|
- unique UDP/TCP/DNS ports auto-allocated from the ephemeral range
|
|
|
|
``fipsd`` needs ``CAP_NET_ADMIN`` (for TUN + netns), so the spawn goes
|
|
through ``sudo`` unless we already have the capability.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from control import ControlClient, ControlError
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
FIPS_BINARY = os.environ.get("FIPS_BINARY", "fips")
|
|
DEFAULT_NETNS_PREFIX = "fips-scan-"
|
|
STARTUP_TIMEOUT = 15.0
|
|
STOP_TIMEOUT = 5.0
|
|
|
|
|
|
def _have_net_admin() -> bool:
|
|
"""True if this process can create a TUN device / netns without sudo."""
|
|
try:
|
|
with open("/proc/self/status") as f:
|
|
for line in f:
|
|
if line.startswith("CapEff:"):
|
|
eff = int(line.split()[1], 16)
|
|
# CAP_NET_ADMIN is bit 12.
|
|
return bool(eff & (1 << 12))
|
|
except OSError:
|
|
pass
|
|
return False
|
|
|
|
|
|
def _free_udp_port() -> int:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
|
|
s.bind(("0.0.0.0", 0))
|
|
return s.getsockname()[1]
|
|
|
|
|
|
def _free_tcp_port() -> int:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
s.bind(("0.0.0.0", 0))
|
|
return s.getsockname()[1]
|
|
|
|
|
|
def _next_netns_name(index: int) -> str:
|
|
return f"{DEFAULT_NETNS_PREFIX}{index}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Network namespace management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _run(cmd: list[str], use_sudo: bool, timeout: float = 5.0) -> subprocess.CompletedProcess:
|
|
full = (["sudo", "-n", *cmd] if use_sudo else cmd)
|
|
return subprocess.run(full, capture_output=True, text=True, timeout=timeout)
|
|
|
|
|
|
def _run_quiet(cmd: list[str], use_sudo: bool, timeout: float = 5.0) -> None:
|
|
"""Run a command, log failures at debug level only."""
|
|
try:
|
|
r = _run(cmd, use_sudo, timeout)
|
|
if r.returncode != 0 and r.stderr.strip():
|
|
log.debug("cmd failed: %s -> %s", " ".join(cmd), r.stderr.strip())
|
|
except Exception as e:
|
|
log.debug("cmd exception: %s -> %s", " ".join(cmd), e)
|
|
|
|
|
|
def _host_default_iface() -> str:
|
|
"""Return the host's default-route interface name, or 'eth0' fallback."""
|
|
try:
|
|
r = subprocess.run(["ip", "route", "show", "default"],
|
|
capture_output=True, text=True, timeout=3)
|
|
for line in r.stdout.splitlines():
|
|
if "dev" in line:
|
|
return line.split("dev")[1].split()[0]
|
|
except Exception:
|
|
pass
|
|
return "eth0"
|
|
|
|
|
|
# Subnet counter for veth pairs (10.250.N.0/30 — 2 usable hosts).
|
|
_veth_counter = [0]
|
|
|
|
|
|
def _next_veth_subnet() -> tuple[str, str, str]:
|
|
"""Return (host_addr, netns_addr, subnet) for the next veth pair."""
|
|
n = _veth_counter[0]
|
|
_veth_counter[0] += 1
|
|
# 10.250.N.1 (host side) / 10.250.N.2 (netns side), /30.
|
|
return f"10.250.{n}.1/30", f"10.250.{n}.2/30", f"10.250.{n}.0/30"
|
|
|
|
|
|
def create_netns(name: str, use_sudo: bool | None = None) -> dict:
|
|
"""Create a network namespace with loopback + a veth pair for outbound NAT.
|
|
|
|
The netns gets a veth link to the host with a 10.250.N.0/30 subnet and a
|
|
default route via the host side. The host masquerades traffic from the
|
|
netns out its default interface, so the ephemeral FIPS daemon inside can
|
|
reach public seed nodes over UDP/TCP.
|
|
|
|
Returns a dict with the veth interface names and subnet, for cleanup.
|
|
"""
|
|
if use_sudo is None:
|
|
use_sudo = not _have_net_admin()
|
|
|
|
# Linux interface names are limited to 15 chars. Use a short prefix
|
|
# derived from the netns index instead of the full netns name.
|
|
n = _veth_counter[0] - 1 # _next_veth_subnet already incremented
|
|
info = {"veth_host": f"fsvh{n}", "veth_ns": f"fsvn{n}", "subnet": None}
|
|
|
|
try:
|
|
_run(["ip", "netns", "add", name], use_sudo)
|
|
except subprocess.CalledProcessError as e:
|
|
log.warning("ip netns add %s failed: %s", name, e.stderr.strip())
|
|
return info
|
|
|
|
# Loopback up so ::1 binds work (DNS responder binds ::1).
|
|
_run_quiet(["ip", "netns", "exec", name, "ip", "link", "set", "lo", "up"], use_sudo)
|
|
|
|
# Create a veth pair: host end stays on the host, ns end moves into netns.
|
|
host_addr, ns_addr, subnet = _next_veth_subnet()
|
|
info["subnet"] = subnet
|
|
_run_quiet(["ip", "link", "add", info["veth_host"], "type", "veth",
|
|
"peer", "name", info["veth_ns"]], use_sudo)
|
|
_run_quiet(["ip", "link", "set", info["veth_ns"], "netns", name], use_sudo)
|
|
_run_quiet(["ip", "addr", "add", host_addr, "dev", info["veth_host"]], use_sudo)
|
|
_run_quiet(["ip", "link", "set", info["veth_host"], "up"], use_sudo)
|
|
_run_quiet(["ip", "netns", "exec", name, "ip", "addr", "add", ns_addr,
|
|
"dev", info["veth_ns"]], use_sudo)
|
|
_run_quiet(["ip", "netns", "exec", name, "ip", "link", "set", info["veth_ns"], "up"], use_sudo)
|
|
# Default route in the netns via the host side of the veth.
|
|
_run_quiet(["ip", "netns", "exec", name, "ip", "route", "add", "default",
|
|
"via", host_addr.split("/")[0], "dev", info["veth_ns"]], use_sudo)
|
|
|
|
# Enable IP forwarding + masquerade on the host for the netns subnet.
|
|
_run_quiet(["sysctl", "-w", "net.ipv4.ip_forward=1"], use_sudo)
|
|
out_iface = _host_default_iface()
|
|
_run_quiet(["iptables", "-t", "nat", "-A", "POSTROUTING", "-s", subnet,
|
|
"-o", out_iface, "-j", "MASQUERADE"], use_sudo)
|
|
_run_quiet(["iptables", "-A", "FORWARD", "-i", info["veth_host"],
|
|
"-o", out_iface, "-j", "ACCEPT"], use_sudo)
|
|
_run_quiet(["iptables", "-A", "FORWARD", "-i", out_iface,
|
|
"-o", info["veth_host"], "-j", "ACCEPT"], use_sudo)
|
|
|
|
log.info("netns %s: veth %s<->%s subnet %s nat via %s",
|
|
name, info["veth_host"], info["veth_ns"], subnet, out_iface)
|
|
return info
|
|
|
|
|
|
def delete_netns(name: str, use_sudo: bool | None = None,
|
|
info: dict | None = None) -> None:
|
|
"""Delete a network namespace and its veth / iptables rules."""
|
|
if use_sudo is None:
|
|
use_sudo = not _have_net_admin()
|
|
# Deleting the netns also destroys the ns-end of the veth; the host-end
|
|
# is removed below. iptables rules referencing the veth are cleaned up
|
|
# best-effort.
|
|
if info:
|
|
subnet = info.get("subnet")
|
|
out_iface = _host_default_iface()
|
|
if subnet:
|
|
_run_quiet(["iptables", "-t", "nat", "-D", "POSTROUTING", "-s", subnet,
|
|
"-o", out_iface, "-j", "MASQUERADE"], use_sudo)
|
|
_run_quiet(["iptables", "-D", "FORWARD", "-i", info.get("veth_host", ""),
|
|
"-o", out_iface, "-j", "ACCEPT"], use_sudo)
|
|
_run_quiet(["iptables", "-D", "FORWARD", "-i", out_iface,
|
|
"-o", info.get("veth_host", ""), "-j", "ACCEPT"], use_sudo)
|
|
_run_quiet(["ip", "link", "del", info.get("veth_host", "")], use_sudo)
|
|
try:
|
|
_run(["ip", "netns", "del", name], use_sudo)
|
|
except Exception as e:
|
|
log.debug("ip netns del %s: %s", name, e)
|
|
|
|
|
|
def exec_in_netns(name: str, cmd: list[str], use_sudo: bool | None = None,
|
|
**kwargs) -> subprocess.CompletedProcess:
|
|
"""Run a command inside a network namespace."""
|
|
if use_sudo is None:
|
|
use_sudo = not _have_net_admin()
|
|
full = (["sudo", "-n"] if use_sudo else []) + ["ip", "netns", "exec", name] + cmd
|
|
return subprocess.run(full, **kwargs)
|
|
|
|
|
|
@dataclass
|
|
class EphemeralDaemon:
|
|
"""A running ephemeral fipsd process + its resources."""
|
|
|
|
config_dir: Path
|
|
config_path: Path
|
|
socket_path: str
|
|
netns: str
|
|
owns_netns: bool
|
|
udp_port: int
|
|
tcp_port: int
|
|
dns_port: int
|
|
process: subprocess.Popen
|
|
used_sudo: bool
|
|
netns_info: dict | None = None
|
|
client: ControlClient = field(init=False)
|
|
|
|
def __post_init__(self) -> None:
|
|
self.client = ControlClient(self.socket_path)
|
|
|
|
# -- lifecycle ---------------------------------------------------------
|
|
|
|
def wait_ready(self, timeout: float = STARTUP_TIMEOUT) -> None:
|
|
"""Poll the control socket until ``show_status`` responds."""
|
|
deadline = time.time() + timeout
|
|
last_err = ""
|
|
while time.time() < deadline:
|
|
if self.process.poll() is not None:
|
|
raise RuntimeError(f"fipsd exited early (code={self.process.returncode})")
|
|
try:
|
|
status = self.client.show_status()
|
|
if status.get("state") in ("running", "initialized", "ready"):
|
|
if status.get("tun_state") == "active":
|
|
return
|
|
# TUN not yet active; keep waiting briefly.
|
|
except ControlError as e:
|
|
last_err = str(e)
|
|
time.sleep(0.3)
|
|
raise RuntimeError(f"fipsd did not become ready in {timeout}s: {last_err}")
|
|
|
|
def stop(self) -> None:
|
|
"""Disconnect peers, SIGTERM, then SIGKILL if needed; clean up netns + dir."""
|
|
try:
|
|
self.client.disconnect_all_peers()
|
|
except Exception as e:
|
|
log.debug("disconnect-all failed: %s", e)
|
|
self._terminate()
|
|
if self.owns_netns:
|
|
delete_netns(self.netns, self.used_sudo, info=self.netns_info)
|
|
self._cleanup_dir()
|
|
|
|
def _terminate(self) -> None:
|
|
if self.process.poll() is not None:
|
|
return
|
|
try:
|
|
self.process.send_signal(signal.SIGTERM)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self.process.wait(timeout=STOP_TIMEOUT)
|
|
except subprocess.TimeoutExpired:
|
|
log.warning("fipsd in netns %s did not exit on SIGTERM, SIGKILL", self.netns)
|
|
try:
|
|
self.process.kill()
|
|
self.process.wait(timeout=2)
|
|
except Exception:
|
|
pass
|
|
|
|
def _cleanup_dir(self) -> None:
|
|
try:
|
|
shutil.rmtree(self.config_dir, ignore_errors=True)
|
|
except Exception:
|
|
pass
|
|
|
|
# -- reachability helper ----------------------------------------------
|
|
|
|
def run_in_netns(self, cmd: list[str], timeout: float = 30.0, **kwargs) -> subprocess.CompletedProcess:
|
|
"""Run a command (e.g. ping6) inside this daemon's netns.
|
|
|
|
Defaults to capturing stdout/stderr as text with a 30s hard timeout
|
|
(callers can override). The timeout prevents hung probes from
|
|
blocking the scan indefinitely.
|
|
"""
|
|
kwargs.setdefault("capture_output", True)
|
|
kwargs.setdefault("text", True)
|
|
kwargs.setdefault("timeout", timeout)
|
|
return exec_in_netns(self.netns, cmd, self.used_sudo, **kwargs)
|
|
|
|
# -- context manager ---------------------------------------------------
|
|
|
|
def __enter__(self) -> "EphemeralDaemon":
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self.stop()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Spawn
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _build_config(
|
|
socket_path: str,
|
|
udp_port: int,
|
|
tcp_port: int,
|
|
dns_port: int,
|
|
) -> str:
|
|
# Inside a netns we can use the canonical "fips0" TUN name — no collision.
|
|
# Mirrors the structure in sovereign_browser/src/net_services.c:349
|
|
# and fips/src/config/{node,transport,upper/config}.rs.
|
|
return (
|
|
"node:\n"
|
|
" identity:\n"
|
|
" persistent: false\n"
|
|
f" control:\n"
|
|
f" socket_path: \"{socket_path}\"\n"
|
|
" log_level: warn\n"
|
|
# Enable Nostr discovery so the ephemeral node learns about other
|
|
# mesh nodes from the advert relays (not just its direct peer).
|
|
# Without this, the identity cache stays at 1 (only the seed) and
|
|
# find_next_hop returns None for every non-peer target.
|
|
# policy: open lets it consider adverts for non-configured peers.
|
|
# advertise: false — the scanner node is ephemeral, don't publish.
|
|
" discovery:\n"
|
|
" nostr:\n"
|
|
" enabled: true\n"
|
|
" advertise: false\n"
|
|
" policy: open\n"
|
|
" advert_relays:\n"
|
|
" - \"wss://relay.damus.io\"\n"
|
|
" - \"wss://nos.lol\"\n"
|
|
" - \"wss://offchain.pub\"\n"
|
|
"tun:\n"
|
|
" enabled: true\n"
|
|
" name: \"fips0\"\n"
|
|
" mtu: 1280\n"
|
|
"dns:\n"
|
|
" enabled: true\n"
|
|
" bind_addr: \"::1\"\n"
|
|
f" port: {dns_port}\n"
|
|
"transports:\n"
|
|
" udp:\n"
|
|
f" bind_addr: \"0.0.0.0:{udp_port}\"\n"
|
|
" tcp:\n"
|
|
f" bind_addr: \"0.0.0.0:{tcp_port}\"\n"
|
|
"peers: []\n"
|
|
)
|
|
|
|
|
|
def spawn_daemon(tag: str = "scan", index: int = 0,
|
|
netns: str | None = None) -> EphemeralDaemon:
|
|
"""Create the config dir + netns, write fips.yaml, spawn fipsd, return handle.
|
|
|
|
If ``netns`` is given, the daemon is spawned in that (shared) namespace
|
|
and does not own it (the caller is responsible for deleting it). This is
|
|
used by the self-test to put two daemons in one netns so they can peer
|
|
over loopback.
|
|
|
|
The daemon is *not* waited on for readiness — call ``wait_ready()``.
|
|
"""
|
|
use_sudo = not _have_net_admin()
|
|
config_dir = Path(tempfile.mkdtemp(prefix=f"mesh-scan-{tag}-"))
|
|
socket_path = str(config_dir / "control.sock")
|
|
udp_port = _free_udp_port()
|
|
tcp_port = _free_tcp_port()
|
|
dns_port = _free_udp_port()
|
|
|
|
owns_netns = netns is None
|
|
netns_info = None
|
|
if owns_netns:
|
|
netns = _next_netns_name(index)
|
|
netns_info = create_netns(netns, use_sudo)
|
|
else:
|
|
# Shared netns (self-test): ensure it exists with NAT. The caller
|
|
# is responsible for creating/deleting it, but if they passed a
|
|
# name without creating it, create_netns is idempotent enough.
|
|
pass
|
|
|
|
config = _build_config(socket_path, udp_port, tcp_port, dns_port)
|
|
config_path = config_dir / "fips.yaml"
|
|
config_path.write_text(config)
|
|
os.chmod(config_path, 0o600)
|
|
|
|
# Spawn fips inside the netns.
|
|
fips_argv = [FIPS_BINARY, "--config", str(config_path)]
|
|
if use_sudo:
|
|
spawn_argv = ["sudo", "-n", "ip", "netns", "exec", netns, *fips_argv]
|
|
else:
|
|
spawn_argv = ["ip", "netns", "exec", netns, *fips_argv]
|
|
|
|
log.info("spawning fipsd: netns=%s udp=%d tcp=%d dns=%d sudo=%s",
|
|
netns, udp_port, tcp_port, dns_port, use_sudo)
|
|
proc = subprocess.Popen(
|
|
spawn_argv,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
start_new_session=True,
|
|
)
|
|
|
|
return EphemeralDaemon(
|
|
config_dir=config_dir,
|
|
config_path=config_path,
|
|
socket_path=socket_path,
|
|
netns=netns,
|
|
owns_netns=owns_netns,
|
|
netns_info=netns_info,
|
|
udp_port=udp_port,
|
|
tcp_port=tcp_port,
|
|
dns_port=dns_port,
|
|
process=proc,
|
|
used_sudo=use_sudo,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stale-namespace cleanup (--cleanup-stale)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def cleanup_stale_netns(prefix: str = DEFAULT_NETNS_PREFIX) -> int:
|
|
"""Remove any network namespaces whose name starts with ``prefix``."""
|
|
use_sudo = not _have_net_admin()
|
|
try:
|
|
out = _run(["ip", "netns", "list"], use_sudo).stdout
|
|
except Exception as e:
|
|
log.warning("could not list netns: %s", e)
|
|
return 0
|
|
count = 0
|
|
for line in out.splitlines():
|
|
name = line.split()[0] if line.strip() else ""
|
|
if name.startswith(prefix):
|
|
try:
|
|
_run(["ip", "netns", "del", name], use_sudo)
|
|
log.info("removed stale netns %s", name)
|
|
count += 1
|
|
except Exception as e:
|
|
log.warning("failed to remove stale netns %s: %s", name, e)
|
|
return count
|
|
|
|
|
|
# Back-compat alias used by mesh_scan.py's --cleanup-stale flag.
|
|
cleanup_stale_tun_devices = cleanup_stale_netns
|
|
|
|
|
|
# Extend ControlClient with a helper to disconnect all peers (used on stop).
|
|
def _disconnect_all_peers(self: ControlClient) -> None:
|
|
try:
|
|
peers = self.show_peers().get("peers", []) or []
|
|
except ControlError:
|
|
return
|
|
for p in peers:
|
|
npub = p.get("npub")
|
|
if npub:
|
|
try:
|
|
self.disconnect(npub)
|
|
except ControlError:
|
|
pass
|
|
|
|
|
|
ControlClient.disconnect_all_peers = _disconnect_all_peers # type: ignore[attr-defined]
|