mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-22 07:48:26 +00:00
Move K8s sidecar from testing/ to examples/
The K8s sidecar is a standalone example, not a test harness — it has no CI integration unlike testing/sidecar/. Move it to examples/ alongside sidecar-nostr-relay/ where it belongs. Update internal path references in Dockerfile, build.sh, and README.
This commit is contained in:
11
examples/k8s-sidecar/.dockerignore
Normal file
11
examples/k8s-sidecar/.dockerignore
Normal file
@@ -0,0 +1,11 @@
|
||||
target/
|
||||
.git/
|
||||
.github/
|
||||
testing/chaos/
|
||||
testing/static/
|
||||
testing/sidecar/
|
||||
testing/tor/
|
||||
fips
|
||||
fipsctl
|
||||
../sidecar/fips
|
||||
../sidecar/fipsctl
|
||||
63
examples/k8s-sidecar/Dockerfile
Normal file
63
examples/k8s-sidecar/Dockerfile
Normal file
@@ -0,0 +1,63 @@
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stage 1 — build
|
||||
# Compile fips and fipsctl inside a Rust toolchain image so no local Rust
|
||||
# install is needed and the image is fully reproducible.
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM rust:slim-trixie AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
pkg-config \
|
||||
libssl-dev && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy the full workspace source. .dockerignore filters target/ and other
|
||||
# large paths so the context stays small.
|
||||
COPY . .
|
||||
|
||||
# Build only the two binaries needed at runtime; skip the TUI (fipstop)
|
||||
# to avoid pulling in ratatui and crossterm. The daemon does not require
|
||||
# any default features to run.
|
||||
RUN cargo build --release --no-default-features \
|
||||
--bin fips \
|
||||
--bin fipsctl
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stage 2 — runtime
|
||||
# Minimal Debian image with only the packages required at runtime.
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM debian:trixie-slim
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
iproute2 \
|
||||
iptables \
|
||||
dnsmasq \
|
||||
procps \
|
||||
curl && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# dnsmasq: forward .fips queries to the FIPS daemon DNS resolver;
|
||||
# forward everything else to the upstream resolver passed in at runtime.
|
||||
# The upstream placeholder is replaced by entrypoint.sh with the pod's
|
||||
# original nameserver(s) from /etc/resolv.conf before dnsmasq starts.
|
||||
RUN printf '%s\n' \
|
||||
'port=53' \
|
||||
'listen-address=127.0.0.1' \
|
||||
'bind-interfaces' \
|
||||
'server=/fips/127.0.0.1#5354' \
|
||||
'no-resolv' \
|
||||
> /etc/dnsmasq.d/fips.conf
|
||||
|
||||
COPY --from=builder /build/target/release/fips /usr/local/bin/fips
|
||||
COPY --from=builder /build/target/release/fipsctl /usr/local/bin/fipsctl
|
||||
|
||||
COPY examples/k8s-sidecar/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD fipsctl show status
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
176
examples/k8s-sidecar/README.md
Normal file
176
examples/k8s-sidecar/README.md
Normal file
@@ -0,0 +1,176 @@
|
||||
# FIPS Kubernetes Sidecar
|
||||
|
||||
Run FIPS as a sidecar container in a Kubernetes Pod, injecting the FIPS mesh
|
||||
network interface into every other container in the Pod. All containers in
|
||||
the Pod share the same network namespace, so once the sidecar creates `fips0`
|
||||
it is immediately visible to the app container(s) without any further
|
||||
configuration.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Build the sidecar image
|
||||
cd examples/k8s-sidecar
|
||||
./scripts/build.sh --tag fips-k8s-sidecar:latest
|
||||
|
||||
# 2. Push to your registry (replace with your own)
|
||||
docker tag fips-k8s-sidecar:latest registry.example.com/fips-k8s-sidecar:latest
|
||||
docker push registry.example.com/fips-k8s-sidecar:latest
|
||||
|
||||
# 3. Create the identity secret
|
||||
kubectl create secret generic fips-identity \
|
||||
--from-literal=nsec=$(openssl rand -hex 32)
|
||||
|
||||
# 4. Deploy
|
||||
kubectl apply -f pod.yaml
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
In Kubernetes, all containers in a Pod share the same network namespace. The
|
||||
fips sidecar:
|
||||
|
||||
1. Generates `/etc/fips/fips.yaml` from environment variables.
|
||||
2. Rewrites `/etc/resolv.conf` to route `.fips` DNS through dnsmasq (which
|
||||
forwards `.fips` names to the FIPS daemon's built-in DNS resolver and
|
||||
everything else to the original cluster DNS).
|
||||
3. Applies `iptables` isolation rules so that the pod's physical interface
|
||||
(`eth0`) only carries FIPS transport traffic.
|
||||
4. Starts dnsmasq and then `exec`s the FIPS daemon.
|
||||
|
||||
The app container starts concurrently and immediately sees `lo`, `eth0`, and
|
||||
`fips0`. DNS for `<npub>.fips` names resolves to `fd::/8` addresses via the
|
||||
dnsmasq → FIPS daemon pipeline.
|
||||
|
||||
```text
|
||||
┌────────────────────────────────────────────────────┐
|
||||
│ Kubernetes Pod (shared network namespace) │
|
||||
│ │
|
||||
│ ┌─────────────────┐ ┌───────────────────────────┐│
|
||||
│ │ fips (sidecar) │ │ app container(s) ││
|
||||
│ │ │ │ ││
|
||||
│ │ fips daemon │ │ your workload ││
|
||||
│ │ fipsctl │ │ sees: lo, eth0, fips0 ││
|
||||
│ │ dnsmasq │ │ ││
|
||||
│ └─────────────────┘ └───────────────────────────┘│
|
||||
│ │
|
||||
│ Interfaces: │
|
||||
│ lo — loopback (unrestricted) │
|
||||
│ eth0 — pod CNI interface (iptables: FIPS only) │
|
||||
│ fips0 — FIPS TUN (fd00::/8, unrestricted) │
|
||||
└────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Network Isolation
|
||||
|
||||
`FIPS_ISOLATE` controls whether the sidecar locks down `eth0`:
|
||||
|
||||
| Value | Behaviour |
|
||||
|---|---|
|
||||
| `false` **(default)** | `fips0` is added alongside normal cluster networking. `eth0` continues to work — services, DNS, other pods, and the internet are all reachable as normal. Application traffic to FIPS mesh peers uses `fips0`. |
|
||||
| `true` | All traffic on `eth0` is dropped except FIPS transport (UDP 2121). The app container can **only** communicate via `fips0`. Use this for deployments where the workload must never bypass the mesh. |
|
||||
|
||||
> **Common gotcha**: if you deploy the sidecar and find that services or
|
||||
> other pods are suddenly unreachable, check that `FIPS_ISOLATE` is not
|
||||
> set to `true`. The mesh-only mode is intentionally strict and will break
|
||||
> normal cluster connectivity.
|
||||
|
||||
## Requirements
|
||||
|
||||
| Requirement | Notes |
|
||||
|---|---|
|
||||
| `NET_ADMIN` capability | Required for TUN creation and iptables |
|
||||
| `/dev/net/tun` | HostPath volume mount (see `pod.yaml`) |
|
||||
| Linux kernel ≥ 4.9 | Standard on all current distributions |
|
||||
| No gVisor / kata-containers | Requires real kernel TUN support |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `FIPS_NSEC` | *(required)* | Node secret key (hex or nsec1 bech32) |
|
||||
| `FIPS_PEER_NPUB` | *(empty)* | Single peer npub |
|
||||
| `FIPS_PEER_ADDR` | *(empty)* | Single peer transport address (`host:port`) |
|
||||
| `FIPS_PEER_ALIAS` | `peer` | Single peer alias |
|
||||
| `FIPS_PEER_TRANSPORT` | `udp` | Single peer transport type |
|
||||
| `FIPS_PEERS_JSON` | *(empty)* | JSON array of peer objects (overrides single-peer vars). **Note**: the included parser handles simple host:port addresses only; IPv6 addresses with brackets may not parse correctly. |
|
||||
| `FIPS_UDP_BIND` | `0.0.0.0:2121` | UDP transport bind address |
|
||||
| `FIPS_UDP_PORT` | *(from `FIPS_UDP_BIND`)* | UDP port for iptables rules |
|
||||
| `FIPS_TCP_BIND` | *(disabled)* | TCP transport bind address |
|
||||
| `FIPS_TUN_NAME` | `fips0` | TUN interface name |
|
||||
| `FIPS_TUN_MTU` | `1280` | TUN MTU |
|
||||
| `FIPS_ISOLATE` | `false` | Apply iptables isolation (blocks all eth0 traffic except FIPS transport) |
|
||||
| `FIPS_POD_IFACE` | `eth0` | Pod physical interface name |
|
||||
| `FIPS_REWRITE_DNS` | `true` | Rewrite `/etc/resolv.conf` |
|
||||
| `RUST_LOG` | `info` | FIPS log level |
|
||||
|
||||
### Multiple Peers
|
||||
|
||||
Use `FIPS_PEERS_JSON` to configure more than one peer:
|
||||
|
||||
```yaml
|
||||
- name: FIPS_PEERS_JSON
|
||||
value: |
|
||||
[
|
||||
{"npub":"npub1abc...","alias":"gw1","addr":"203.0.113.10:2121","transport":"udp"},
|
||||
{"npub":"npub1def...","alias":"gw2","addr":"198.51.100.5:2121","transport":"udp"}
|
||||
]
|
||||
```
|
||||
|
||||
Each object supports the keys: `npub` (required), `addr` (required),
|
||||
`alias`, `transport`, `priority`.
|
||||
|
||||
Alternatively, mount a hand-crafted `fips.yaml` and set `FIPS_NSEC` to
|
||||
anything (the mount takes precedence because the entrypoint writes to
|
||||
`/etc/fips/fips.yaml` which is then overridden by the volume):
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- name: fips-config
|
||||
configMap:
|
||||
name: my-fips-config
|
||||
containers:
|
||||
- name: fips
|
||||
volumeMounts:
|
||||
- name: fips-config
|
||||
mountPath: /etc/fips/fips.yaml
|
||||
subPath: fips.yaml
|
||||
```
|
||||
|
||||
### JSON Parsing Limitations
|
||||
|
||||
`FIPS_PEERS_JSON` supports parsing with a simple awk-based parser. It handles common `host:port` addresses but does not support IPv6 addresses with brackets (e.g., `[::1]:2121`) or values containing embedded commas/colons. For complex configs, install `jq` in the container and modify the entrypoint, or use a mounted `fips.yaml`.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `Dockerfile` | Builds the sidecar image |
|
||||
| `entrypoint.sh` | Generates config, rewrites DNS, applies iptables, starts fips |
|
||||
| `pod.yaml` | Example Pod manifest (single app container + fips sidecar) |
|
||||
| `scripts/build.sh` | Compiles FIPS and builds the Docker image |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**`FIPS_NSEC is required`** — The secret was not injected. Verify the
|
||||
`secretKeyRef` name and key match the secret you created with `kubectl
|
||||
create secret`.
|
||||
|
||||
**`fips0` not visible in app container** — Check that the fips sidecar
|
||||
started successfully: `kubectl logs <pod> -c fips`. The sidecar must start
|
||||
and run the daemon before `fips0` appears. Add a startup probe or
|
||||
`postStart` lifecycle hook to your app container if you need to wait.
|
||||
|
||||
**iptables errors** — The `NET_ADMIN` capability and `/dev/net/tun` volume
|
||||
mount are both required. Verify they are present in the Pod spec.
|
||||
|
||||
**`.fips` DNS not resolving** — Check that `FIPS_REWRITE_DNS=true` and that
|
||||
dnsmasq started: `kubectl exec <pod> -c fips -- pgrep dnsmasq`. Verify the
|
||||
FIPS DNS listener: `kubectl exec <pod> -c fips -- fipsctl show status`.
|
||||
|
||||
**CNI uses a different interface name** — Set `FIPS_POD_IFACE` to match your
|
||||
CNI's interface name (e.g. `ens3`, `net1` for Multus secondary interfaces).
|
||||
|
||||
**gVisor / kata-containers** — These sandboxed runtimes intercept syscalls
|
||||
and do not support `AF_PACKET` or `/dev/net/tun` in the same way as a
|
||||
standard kernel. Use a standard RuntimeClass for FIPS sidecar pods.
|
||||
309
examples/k8s-sidecar/entrypoint.sh
Executable file
309
examples/k8s-sidecar/entrypoint.sh
Executable file
@@ -0,0 +1,309 @@
|
||||
#!/bin/bash
|
||||
# FIPS Kubernetes sidecar entrypoint.
|
||||
#
|
||||
# Generates /etc/fips/fips.yaml from environment variables, rewrites the
|
||||
# pod's resolv.conf to route .fips DNS through dnsmasq, applies iptables
|
||||
# isolation rules so that the shared pod network namespace can only reach
|
||||
# the outside world via the FIPS mesh, then launches the FIPS daemon.
|
||||
#
|
||||
# Required environment variables:
|
||||
# FIPS_NSEC Node secret key (hex or nsec1 bech32)
|
||||
#
|
||||
# Optional environment variables (single peer shorthand):
|
||||
# FIPS_PEER_NPUB Peer's npub
|
||||
# FIPS_PEER_ADDR Peer's transport address (e.g. 203.0.113.10:2121)
|
||||
# FIPS_PEER_ALIAS Human-readable peer name (default: peer)
|
||||
# FIPS_PEER_TRANSPORT Transport type (default: udp)
|
||||
#
|
||||
# Multiple peers via JSON (overrides single-peer vars when set):
|
||||
# FIPS_PEERS_JSON JSON array of peer objects, e.g.:
|
||||
# '[{"npub":"npub1...","alias":"gw","addr":"1.2.3.4:2121","transport":"udp"}]'
|
||||
#
|
||||
# Transport / TUN tuning:
|
||||
# FIPS_UDP_BIND UDP bind address (default: 0.0.0.0:2121)
|
||||
# FIPS_UDP_PORT UDP transport port (derived from FIPS_UDP_BIND when unset)
|
||||
# FIPS_TCP_BIND TCP bind address (default: disabled)
|
||||
# FIPS_TUN_NAME TUN interface name (default: fips0)
|
||||
# FIPS_TUN_MTU TUN interface MTU (default: 1280, IPv6 minimum)
|
||||
#
|
||||
# Network isolation:
|
||||
# FIPS_ISOLATE Apply iptables isolation (default: false)
|
||||
# Set to "true" for mesh-only deployments where the app
|
||||
# must not communicate outside the FIPS mesh.
|
||||
# Set to "false" to add fips0 alongside normal cluster
|
||||
# networking without restricting eth0.
|
||||
# FIPS_POD_IFACE Pod network interface (default: eth0)
|
||||
# Adjust if your CNI uses a different name (e.g. ens3).
|
||||
#
|
||||
# DNS:
|
||||
# FIPS_REWRITE_DNS Rewrite /etc/resolv.conf (default: true)
|
||||
# Set to "false" if you manage DNS via another mechanism.
|
||||
#
|
||||
# Logging:
|
||||
# RUST_LOG FIPS log level (default: info)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IPv6 kernel configuration
|
||||
#
|
||||
# net.ipv6.conf.all.disable_ipv6, net.ipv6.conf.default.disable_ipv6, and
|
||||
# net.ipv6.bindv6only are set via the pod securityContext.sysctls field in
|
||||
# pod.yaml so that kubelet applies them before any container starts.
|
||||
# Nothing to do here.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FIPS_NSEC="${FIPS_NSEC:?FIPS_NSEC is required — provide your node secret key}"
|
||||
|
||||
FIPS_UDP_BIND="${FIPS_UDP_BIND:-0.0.0.0:2121}"
|
||||
FIPS_TCP_BIND="${FIPS_TCP_BIND:-}"
|
||||
FIPS_TUN_NAME="${FIPS_TUN_NAME:-fips0}"
|
||||
FIPS_TUN_MTU="${FIPS_TUN_MTU:-1280}"
|
||||
FIPS_ISOLATE="${FIPS_ISOLATE:-false}"
|
||||
FIPS_POD_IFACE="${FIPS_POD_IFACE:-eth0}"
|
||||
FIPS_REWRITE_DNS="${FIPS_REWRITE_DNS:-true}"
|
||||
|
||||
FIPS_PEER_TRANSPORT="${FIPS_PEER_TRANSPORT:-udp}"
|
||||
FIPS_PEER_ALIAS="${FIPS_PEER_ALIAS:-peer}"
|
||||
|
||||
# Derive the UDP port for iptables rules from FIPS_UDP_BIND (last colon-field).
|
||||
FIPS_UDP_PORT="${FIPS_UDP_PORT:-${FIPS_UDP_BIND##*:}}"
|
||||
|
||||
mkdir -p /etc/fips
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build peers YAML section
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
build_peers_yaml() {
|
||||
# FIPS_PEERS_JSON wins when set (supports multiple peers).
|
||||
if [ -n "${FIPS_PEERS_JSON:-}" ]; then
|
||||
# Parse the JSON array with awk — no jq required.
|
||||
# Expected element keys: npub, alias, addr, transport, priority
|
||||
#
|
||||
# Note: This parser handles simple host:port addresses only.
|
||||
# IPv6 addresses with brackets ([::1]:2121) or values containing
|
||||
# commas/colons may not parse correctly. Use single-peer env vars
|
||||
# for complex configs or install jq and replace this block.
|
||||
echo "$FIPS_PEERS_JSON" | awk '
|
||||
BEGIN { RS="}"; FS="," }
|
||||
{
|
||||
npub=""; alias="peer"; addr=""; transport="udp"; priority=100
|
||||
for (i=1; i<=NF; i++) {
|
||||
gsub(/[{}\[\] \t\n\r]/, "", $i)
|
||||
if ($i ~ /"npub"/) { split($i,a,":"); npub=a[2]; gsub(/"/,"",npub) }
|
||||
if ($i ~ /"alias"/) { split($i,a,":"); alias=a[2]; gsub(/"/,"",alias) }
|
||||
if ($i ~ /"addr"/) { n=split($i,a,":"); addr=a[2]":"a[3]; gsub(/"/,"",addr) }
|
||||
if ($i ~ /"transport"/) { split($i,a,":"); transport=a[2]; gsub(/"/,"",transport) }
|
||||
if ($i ~ /"priority"/) { split($i,a,":"); priority=a[2]; gsub(/"/,"",priority) }
|
||||
}
|
||||
if (npub != "" && addr != "") {
|
||||
printf " - npub: \"%s\"\n alias: \"%s\"\n addresses:\n - transport: %s\n addr: \"%s\"\n priority: %s\n connect_policy: auto_connect\n auto_reconnect: true\n", npub, alias, transport, addr, priority
|
||||
}
|
||||
}'
|
||||
return
|
||||
fi
|
||||
|
||||
# Single-peer shorthand via individual env vars.
|
||||
if [ -n "${FIPS_PEER_NPUB:-}" ] && [ -n "${FIPS_PEER_ADDR:-}" ]; then
|
||||
cat <<EOF
|
||||
- npub: "${FIPS_PEER_NPUB}"
|
||||
alias: "${FIPS_PEER_ALIAS}"
|
||||
addresses:
|
||||
- transport: ${FIPS_PEER_TRANSPORT}
|
||||
addr: "${FIPS_PEER_ADDR}"
|
||||
connect_policy: auto_connect
|
||||
auto_reconnect: true
|
||||
EOF
|
||||
return
|
||||
fi
|
||||
|
||||
# No peers configured — start standalone (useful for a new node joining
|
||||
# via discovery or when peers are added later via fipsctl).
|
||||
echo " []"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build optional TCP transport stanza
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
build_tcp_yaml() {
|
||||
if [ -n "$FIPS_TCP_BIND" ]; then
|
||||
cat <<EOF
|
||||
tcp:
|
||||
bind_addr: "${FIPS_TCP_BIND}"
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generate /etc/fips/fips.yaml
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PEERS_YAML="$(build_peers_yaml)"
|
||||
TCP_YAML="$(build_tcp_yaml)"
|
||||
|
||||
cat > /etc/fips/fips.yaml <<EOF
|
||||
node:
|
||||
identity:
|
||||
nsec: "${FIPS_NSEC}"
|
||||
control:
|
||||
enabled: true
|
||||
socket_path: "/run/fips/control.sock"
|
||||
|
||||
tun:
|
||||
enabled: true
|
||||
name: ${FIPS_TUN_NAME}
|
||||
mtu: ${FIPS_TUN_MTU}
|
||||
|
||||
dns:
|
||||
enabled: true
|
||||
bind_addr: "127.0.0.1"
|
||||
port: 5354
|
||||
|
||||
transports:
|
||||
udp:
|
||||
bind_addr: "${FIPS_UDP_BIND}"
|
||||
${TCP_YAML}
|
||||
peers:
|
||||
${PEERS_YAML}
|
||||
EOF
|
||||
|
||||
echo "[fips-sidecar] Generated /etc/fips/fips.yaml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rewrite /etc/resolv.conf to route .fips DNS through dnsmasq
|
||||
#
|
||||
# Strategy:
|
||||
# 1. Harvest the nameservers the kubelet injected.
|
||||
# 2. Write dnsmasq upstream rules for each harvested nameserver.
|
||||
# 3. Point /etc/resolv.conf at 127.0.0.1 (dnsmasq).
|
||||
# 4. dnsmasq forwards .fips → FIPS daemon (127.0.0.1:5354),
|
||||
# everything else → original cluster DNS (typically 169.254.20.10 or
|
||||
# the kube-dns ClusterIP injected by kubelet).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [ "$FIPS_REWRITE_DNS" = "true" ]; then
|
||||
# Collect existing upstream nameservers before we overwrite resolv.conf.
|
||||
UPSTREAM_NS=()
|
||||
while IFS= read -r line; do
|
||||
if [[ "$line" =~ ^nameserver[[:space:]]+(.+)$ ]]; then
|
||||
ns="${BASH_REMATCH[1]}"
|
||||
# Skip loopback — we're about to replace it.
|
||||
if [[ "$ns" != "127."* ]] && [[ "$ns" != "::1" ]]; then
|
||||
UPSTREAM_NS+=("$ns")
|
||||
fi
|
||||
fi
|
||||
done < /etc/resolv.conf
|
||||
|
||||
# Append upstream server lines to dnsmasq fips config.
|
||||
# (The placeholder in the image has no upstream server= lines yet.)
|
||||
for ns in "${UPSTREAM_NS[@]}"; do
|
||||
echo "server=${ns}" >> /etc/dnsmasq.d/fips.conf
|
||||
done
|
||||
|
||||
# If no upstream was found fall back to Google (better than no resolution).
|
||||
if [ "${#UPSTREAM_NS[@]}" -eq 0 ]; then
|
||||
echo "[fips-sidecar] WARNING: no upstream nameserver found in /etc/resolv.conf; falling back to 8.8.8.8"
|
||||
echo "server=8.8.8.8" >> /etc/dnsmasq.d/fips.conf
|
||||
fi
|
||||
|
||||
# Preserve the search/domain lines so pod DNS short-names still work.
|
||||
SEARCH_LINE=""
|
||||
while IFS= read -r line; do
|
||||
if [[ "$line" =~ ^(search|domain)[[:space:]] ]]; then
|
||||
SEARCH_LINE="$line"
|
||||
break
|
||||
fi
|
||||
done < /etc/resolv.conf
|
||||
|
||||
{
|
||||
echo "nameserver 127.0.0.1"
|
||||
[ -n "$SEARCH_LINE" ] && echo "$SEARCH_LINE"
|
||||
} > /etc/resolv.conf
|
||||
|
||||
echo "[fips-sidecar] DNS rewritten: upstream ${UPSTREAM_NS[*]:-8.8.8.8} → dnsmasq → fips/cluster"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# iptables isolation
|
||||
#
|
||||
# Goal: only FIPS transport traffic may leave/enter the pod via the physical
|
||||
# interface. All other pod-to-external traffic is dropped. Traffic on the
|
||||
# fips0 TUN and loopback is unrestricted. The app container sharing this
|
||||
# network namespace therefore can only communicate over the FIPS mesh.
|
||||
#
|
||||
# Skip when FIPS_ISOLATE=false (e.g. already handled by a NetworkPolicy,
|
||||
# or when the operator wants unrestricted egress alongside mesh traffic).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [ "$FIPS_ISOLATE" = "true" ]; then
|
||||
IFACE="$FIPS_POD_IFACE"
|
||||
UDP_PORT="$FIPS_UDP_PORT"
|
||||
|
||||
# IPv4: allow loopback + FIPS UDP transport; drop everything else on
|
||||
# the pod's physical interface.
|
||||
iptables -A OUTPUT -o lo -j ACCEPT
|
||||
iptables -A INPUT -i lo -j ACCEPT
|
||||
|
||||
iptables -A OUTPUT -o "$IFACE" -p udp --dport "$UDP_PORT" -j ACCEPT
|
||||
iptables -A OUTPUT -o "$IFACE" -p udp --sport "$UDP_PORT" -j ACCEPT
|
||||
iptables -A INPUT -i "$IFACE" -p udp --dport "$UDP_PORT" -j ACCEPT
|
||||
iptables -A INPUT -i "$IFACE" -p udp --sport "$UDP_PORT" -j ACCEPT
|
||||
|
||||
# Allow TCP transport port when configured.
|
||||
if [ -n "$FIPS_TCP_BIND" ]; then
|
||||
TCP_PORT="${FIPS_TCP_BIND##*:}"
|
||||
iptables -A OUTPUT -o "$IFACE" -p tcp --dport "$TCP_PORT" -j ACCEPT
|
||||
iptables -A INPUT -i "$IFACE" -p tcp --sport "$TCP_PORT" -j ACCEPT
|
||||
iptables -A OUTPUT -o "$IFACE" -p tcp --sport "$TCP_PORT" -j ACCEPT
|
||||
iptables -A INPUT -i "$IFACE" -p tcp --dport "$TCP_PORT" -j ACCEPT
|
||||
fi
|
||||
|
||||
# Allow established/related so existing connections aren't torn down mid-flight.
|
||||
iptables -A INPUT -i "$IFACE" -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
iptables -A OUTPUT -o "$IFACE" -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
|
||||
iptables -A OUTPUT -o "$IFACE" -j DROP
|
||||
iptables -A INPUT -i "$IFACE" -j DROP
|
||||
|
||||
# IPv6: allow loopback and fips TUN; block physical interface entirely.
|
||||
ip6tables -A OUTPUT -o lo -j ACCEPT
|
||||
ip6tables -A INPUT -i lo -j ACCEPT
|
||||
ip6tables -A OUTPUT -o "$FIPS_TUN_NAME" -j ACCEPT
|
||||
ip6tables -A INPUT -i "$FIPS_TUN_NAME" -j ACCEPT
|
||||
ip6tables -A OUTPUT -o "$IFACE" -j DROP
|
||||
ip6tables -A INPUT -i "$IFACE" -j DROP
|
||||
|
||||
echo "[fips-sidecar] iptables isolation applied on ${IFACE} (UDP ${UDP_PORT})"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TCP MSS clamping on fips0
|
||||
#
|
||||
# The kernel derives MSS from the TUN MTU: 1280 - 60 = 1220. But FIPS adds
|
||||
# FIPS_IPV6_OVERHEAD (77) bytes of encapsulation that the kernel doesn't know
|
||||
# Clamp the MSS in TCP SYN packets traversing fips0 to the path MTU so that
|
||||
# segments fit within the effective MTU after FIPS encapsulation.
|
||||
# This is applied regardless of FIPS_ISOLATE mode since it's needed for
|
||||
# correct TCP negotiation even when normal network access is allowed.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ip6tables -t mangle -A FORWARD -o "$FIPS_TUN_NAME" -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
|
||||
ip6tables -t mangle -A FORWARD -i "$FIPS_TUN_NAME" -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
|
||||
ip6tables -t mangle -A OUTPUT -o "$FIPS_TUN_NAME" -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
|
||||
|
||||
echo "[fips-sidecar] TCP MSS clamped to PMTU on ${FIPS_TUN_NAME}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Start dnsmasq and launch FIPS daemon
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
dnsmasq --conf-dir=/etc/dnsmasq.d
|
||||
echo "[fips-sidecar] dnsmasq started"
|
||||
|
||||
exec fips --config /etc/fips/fips.yaml
|
||||
207
examples/k8s-sidecar/pod.yaml
Normal file
207
examples/k8s-sidecar/pod.yaml
Normal file
@@ -0,0 +1,207 @@
|
||||
# FIPS Kubernetes sidecar — example Pod manifest
|
||||
#
|
||||
# WARNING: The Secret below contains a placeholder nsec. Replace it with your
|
||||
# real key before applying. Do NOT commit real keys to version control.
|
||||
# Generate a fresh key with: openssl rand -hex 32
|
||||
#
|
||||
# Apply:
|
||||
# kubectl apply -f pod.yaml
|
||||
#
|
||||
# Requirements on the node:
|
||||
# - /dev/net/tun must exist (standard on all modern Linux kernels)
|
||||
# - The container runtime must honour NET_ADMIN (true for containerd/CRI-O
|
||||
# with default settings; may require a RuntimeClass tweak for gVisor)
|
||||
#
|
||||
# Kubernetes version: 1.28+
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Secret — node identity
|
||||
# Replace the nsec value with your real key (hex or nsec1 bech32).
|
||||
# In production, manage this with a secrets operator (External Secrets,
|
||||
# Sealed Secrets, Vault Agent, etc.) rather than a plaintext manifest.
|
||||
# -----------------------------------------------------------------------------
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: fips-identity
|
||||
type: Opaque
|
||||
stringData:
|
||||
nsec: "replace-this-with-your-hex-or-nsec1-secret-key"
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: fips-sidecar-example
|
||||
labels:
|
||||
app: fips-sidecar-example
|
||||
spec:
|
||||
# Containers in the same Pod share the same network namespace automatically.
|
||||
# The fips sidecar creates fips0; the app container sees it immediately.
|
||||
#
|
||||
# Start order: initContainers run first, then all containers start
|
||||
# concurrently. If your app needs the TUN interface to be ready before it
|
||||
# starts, use a readiness probe on the sidecar and a dependency in the app
|
||||
# (or use a startup init container that waits for fips0 to appear).
|
||||
|
||||
# net.ipv6.* sysctls are namespaced per-pod in Linux. Declaring them here
|
||||
# allows the fips sidecar entrypoint to set them at runtime via sysctl(8).
|
||||
# bindv6only=0 means apps listening on 0.0.0.0 will also receive IPv6
|
||||
# connections arriving via fips0, without needing to be rewritten.
|
||||
# These are "unsafe" sysctls and require the kubelet --allowed-unsafe-sysctls
|
||||
# flag or a corresponding RuntimeClass / PodSecurityPolicy exception.
|
||||
securityContext:
|
||||
sysctls:
|
||||
- name: net.ipv6.conf.all.disable_ipv6
|
||||
value: "0"
|
||||
- name: net.ipv6.conf.default.disable_ipv6
|
||||
value: "0"
|
||||
- name: net.ipv6.bindv6only
|
||||
value: "0"
|
||||
|
||||
volumes:
|
||||
- name: tun-device
|
||||
hostPath:
|
||||
path: /dev/net/tun
|
||||
type: CharDevice
|
||||
# Optional: mount a pre-built fips.yaml instead of generating from env.
|
||||
# - name: fips-config
|
||||
# configMap:
|
||||
# name: fips-config
|
||||
|
||||
containers:
|
||||
# -----------------------------------------------------------------------
|
||||
# FIPS sidecar — owns the network namespace setup
|
||||
# -----------------------------------------------------------------------
|
||||
- name: fips
|
||||
image: your-registry.example.com/fips-sidecar:latest
|
||||
imagePullPolicy: Always
|
||||
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN # required: TUN creation and iptables
|
||||
runAsNonRoot: false # fips needs root for iptables/TUN
|
||||
runAsUser: 0
|
||||
|
||||
volumeMounts:
|
||||
- name: tun-device
|
||||
mountPath: /dev/net/tun
|
||||
|
||||
env:
|
||||
# --- Identity (required) ---
|
||||
- name: FIPS_NSEC
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: fips-identity
|
||||
key: nsec
|
||||
|
||||
# --- Single peer shorthand ---
|
||||
# Remove these and use FIPS_PEERS_JSON below for multiple peers.
|
||||
- name: FIPS_PEER_NPUB
|
||||
value: "" # e.g. npub1abc...
|
||||
- name: FIPS_PEER_ADDR
|
||||
value: "" # e.g. 203.0.113.10:2121
|
||||
- name: FIPS_PEER_ALIAS
|
||||
value: "gateway"
|
||||
- name: FIPS_PEER_TRANSPORT
|
||||
value: "udp"
|
||||
|
||||
# --- Multiple peers via JSON (overrides single-peer vars when non-empty) ---
|
||||
# - name: FIPS_PEERS_JSON
|
||||
# value: |
|
||||
# [
|
||||
# {"npub":"npub1abc...","alias":"gw1","addr":"203.0.113.10:2121","transport":"udp"},
|
||||
# {"npub":"npub1def...","alias":"gw2","addr":"198.51.100.5:2121","transport":"udp"}
|
||||
# ]
|
||||
|
||||
# --- Transport tuning ---
|
||||
- name: FIPS_UDP_BIND
|
||||
value: "0.0.0.0:2121"
|
||||
# Uncomment to enable TCP transport (useful on networks that block UDP):
|
||||
# - name: FIPS_TCP_BIND
|
||||
# value: "0.0.0.0:8443"
|
||||
|
||||
# --- TUN interface ---
|
||||
- name: FIPS_TUN_NAME
|
||||
value: "fips0"
|
||||
- name: FIPS_TUN_MTU
|
||||
value: "1280"
|
||||
|
||||
# --- Network isolation ---
|
||||
# "false" — fips0 is added alongside normal cluster networking. eth0
|
||||
# continues to work as usual (services, DNS, other pods all reachable).
|
||||
# "true" — eth0 is locked to FIPS transport only; app can only talk
|
||||
# via fips0. Use for high-security mesh-only deployments.
|
||||
- name: FIPS_ISOLATE
|
||||
value: "false"
|
||||
# Adjust if your CNI names the pod interface differently (e.g. ens3, net1).
|
||||
- name: FIPS_POD_IFACE
|
||||
value: "eth0"
|
||||
|
||||
# --- DNS ---
|
||||
# Set to "false" if you manage pod DNS externally.
|
||||
- name: FIPS_REWRITE_DNS
|
||||
value: "true"
|
||||
|
||||
# --- Logging ---
|
||||
- name: RUST_LOG
|
||||
value: "info"
|
||||
|
||||
ports:
|
||||
- name: fips-udp
|
||||
containerPort: 2121
|
||||
protocol: UDP
|
||||
# - name: fips-tcp
|
||||
# containerPort: 8443
|
||||
# protocol: TCP
|
||||
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["fipsctl", "show", "status"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["fipsctl", "show", "status"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: "50m"
|
||||
memory: "32Mi"
|
||||
limits:
|
||||
cpu: "500m"
|
||||
memory: "128Mi"
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# App container — shares the network namespace created by the fips sidecar
|
||||
# -----------------------------------------------------------------------
|
||||
- name: app
|
||||
image: your-app:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
# No special capabilities needed — app uses fips0 like a normal interface.
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
|
||||
# The app sees: lo, eth0 (isolated by iptables), fips0 (mesh).
|
||||
# All outbound traffic from the app flows through fips0.
|
||||
# DNS resolution for <npub>.fips names works via dnsmasq on 127.0.0.1.
|
||||
command: ["sleep", "infinity"] # replace with your actual workload command
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: "100m"
|
||||
memory: "64Mi"
|
||||
limits:
|
||||
cpu: "1000m"
|
||||
memory: "512Mi"
|
||||
|
||||
# Restart policy for the Pod. Use "Always" for long-running workloads.
|
||||
restartPolicy: Always
|
||||
37
examples/k8s-sidecar/scripts/build.sh
Executable file
37
examples/k8s-sidecar/scripts/build.sh
Executable file
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# Build the FIPS Kubernetes sidecar Docker image.
|
||||
# The build happens entirely inside Docker (multi-arch friendly).
|
||||
# Usage: ./scripts/build.sh [--tag TAG] [-- <extra docker buildx args>]
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
DOCKER_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
PROJECT_ROOT="$(cd "$DOCKER_DIR/../.." && pwd)"
|
||||
|
||||
IMAGE_TAG="fips-k8s-sidecar:latest"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--tag) IMAGE_TAG="$2"; shift 2 ;;
|
||||
--) shift; break ;;
|
||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$PROJECT_ROOT/Cargo.toml" ]; then
|
||||
echo "Error: Cannot find Cargo.toml at $PROJECT_ROOT" >&2
|
||||
echo "Expected layout: <project-root>/examples/k8s-sidecar/scripts/build.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building Docker image: $IMAGE_TAG"
|
||||
docker build \
|
||||
-t "$IMAGE_TAG" \
|
||||
-f "$DOCKER_DIR/Dockerfile" \
|
||||
"$@" \
|
||||
"$PROJECT_ROOT"
|
||||
|
||||
echo ""
|
||||
echo "Done. Image: $IMAGE_TAG"
|
||||
echo ""
|
||||
echo "Push to your registry, then apply the example manifest:"
|
||||
echo " kubectl apply -f examples/k8s-sidecar/pod.yaml"
|
||||
Reference in New Issue
Block a user