mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-22 15:52:20 +00:00
78377208af
The isolation loop and the failure-log dump both built container names without the run suffix, so under a suffixed run they addressed containers that do not exist. The consequence was worse than the visible failure. Two of the three checks in that loop assert a ping FAILS, and a ping into a non-existent container fails for the wrong reason - so both passed vacuously, and the security assertion the loop exists for was silently a no-op. Only the third check, which expects a ping to succeed, noticed anything was wrong. Add a guard that fails loudly when the container is absent, so a future naming slip cannot quietly turn these back into no-ops rather than surfacing as one confusing failure among two false passes. These sites were missed because the suite passes when run by hand: with no suffix set the unsuffixed names are correct, and the defect only appears under the automation that sets one. Verified this time with the suffix set, which is the condition that matters: 18 of 18, container names resolving to real containers.
313 lines
13 KiB
Bash
Executable File
313 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# Integration test for the FIPS sidecar deployment.
|
|
#
|
|
# Starts a 3-node chain (A—B—C) using standalone sidecar instances,
|
|
# verifies link establishment, multi-hop connectivity, and network
|
|
# isolation on each app container.
|
|
#
|
|
# Usage: ./test-sidecar.sh [--skip-build]
|
|
#
|
|
# Exit codes:
|
|
# 0 — all tests passed
|
|
# 1 — test failure
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
# shellcheck source=../../lib/wait-converge.sh
|
|
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
|
|
|
|
# Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c}
|
|
NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453"
|
|
NODE_A_NPUB="npub1jvren5hnege54lu3p2nzqacctmulqvkgp68yvfuj5jme5dtgnhxsdh6788"
|
|
NODE_B_NSEC="3e4e10614c0490575fa5e994524ff3f4deaac2f20db189cc9c9a79da0d90f17a"
|
|
NODE_B_NPUB="npub15h7z0ljzudqe9pgwx99cjsz2c0ennuyvkcc8zvtk3lg97xwzex9ska6g4y"
|
|
NODE_C_NSEC="15148ed0131f7da43fd13e369dfedede14fb64698f3756636b569c3a3e87438f"
|
|
NODE_C_NPUB="npub1zhezcykd0e34z4fxtranl45jaasgnlxv0kjqwlq2v56ggssn0w4qelcrvr"
|
|
|
|
NETWORK_NAME="fips-sidecar-test${FIPS_CI_NAME_SUFFIX:-}"
|
|
|
|
# Compose project names. These are explicit (they override COMPOSE_PROJECT_NAME),
|
|
# so they carry the run suffix themselves — otherwise concurrent runs would share
|
|
# one project and tear down each other's containers.
|
|
PROJ_A="sidecar-a${FIPS_CI_NAME_SUFFIX:-}"
|
|
PROJ_B="sidecar-b${FIPS_CI_NAME_SUFFIX:-}"
|
|
PROJ_C="sidecar-c${FIPS_CI_NAME_SUFFIX:-}"
|
|
# Network address range — claimed at startup by alloc_network below rather than
|
|
# hardcoded, so two concurrent runs cannot request the same range.
|
|
#
|
|
# 10.40.0.0/16 is unclaimed in-tree: the suites use 172.20-172.32 and 172.99,
|
|
# the chaos children 10.30.x, boringtun's inner network 10.99.x, and docker's
|
|
# own default pool is 172.17-31 plus 192.168.
|
|
NET_BASE="10.40"
|
|
NET_CANDIDATES=64
|
|
|
|
SUBNET=""
|
|
GATEWAY_IP=""
|
|
NODE_A_IP=""
|
|
NODE_B_IP=""
|
|
NODE_C_IP=""
|
|
|
|
CONVERGE_TIMEOUT=30
|
|
PASSED=0
|
|
FAILED=0
|
|
|
|
# Compose file paths
|
|
COMPOSE_BASE="-f $SIDECAR_DIR/docker-compose.yml"
|
|
COMPOSE_EXT="$COMPOSE_BASE -f $SIDECAR_DIR/docker-compose.external-net.yml"
|
|
|
|
# ── Helpers ────────────────────────────────────────────────────────────────
|
|
|
|
log() { echo "=== $*"; }
|
|
pass() { echo " PASS: $*"; PASSED=$((PASSED + 1)); }
|
|
fail() { echo " FAIL: $*"; FAILED=$((FAILED + 1)); }
|
|
|
|
# Claim a free /24 for this run's network.
|
|
#
|
|
# Claim-and-advance rather than deriving an offset from the run id: we attempt
|
|
# creation on a candidate range and, if docker reports the range is taken, move
|
|
# to the next. Docker's own address pool is then the arbiter, which makes an
|
|
# overlap between two concurrent runs *impossible*. A hashed offset would only
|
|
# make it unlikely, and an overlap is precisely the failure being avoided here.
|
|
#
|
|
# Deliberately does NOT discard stderr: only an address-pool conflict is worth
|
|
# advancing on. Any other failure (name already taken, daemon error) is real,
|
|
# and retrying it 64 times would bury the reason.
|
|
alloc_network() {
|
|
local i err
|
|
for (( i = 0; i < NET_CANDIDATES; i++ )); do
|
|
SUBNET="${NET_BASE}.${i}.0/24"
|
|
if err=$(docker network create \
|
|
--subnet "$SUBNET" \
|
|
--label com.corganlabs.fips-ci=1 \
|
|
--label "com.corganlabs.fips-ci.run=${FIPS_CI_RUN_ID:-manual}" \
|
|
"$NETWORK_NAME" 2>&1); then
|
|
GATEWAY_IP="${NET_BASE}.${i}.1"
|
|
NODE_A_IP="${NET_BASE}.${i}.10"
|
|
NODE_B_IP="${NET_BASE}.${i}.11"
|
|
NODE_C_IP="${NET_BASE}.${i}.12"
|
|
log "Claimed network $NETWORK_NAME on $SUBNET"
|
|
return 0
|
|
fi
|
|
case "$err" in
|
|
*"Pool overlaps"*|*"pool overlaps"*) continue ;;
|
|
*) echo " FAIL: docker network create: $err" >&2; return 1 ;;
|
|
esac
|
|
done
|
|
echo " FAIL: no free /24 in ${NET_BASE}.0.0/16 after ${NET_CANDIDATES} attempts" >&2
|
|
return 1
|
|
}
|
|
|
|
cleanup() {
|
|
log "Cleaning up..."
|
|
# Tear down C and B first, then A. All three attach to the network as
|
|
# external, so none of them removes it — we do that last, by name.
|
|
docker compose $COMPOSE_EXT -p "$PROJ_C" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker compose $COMPOSE_EXT -p "$PROJ_B" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker compose $COMPOSE_EXT -p "$PROJ_A" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Always clean up on exit
|
|
trap cleanup EXIT
|
|
|
|
# ── Build ──────────────────────────────────────────────────────────────────
|
|
|
|
if [[ "${1:-}" != "--skip-build" ]]; then
|
|
log "Building test images..."
|
|
DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)"
|
|
docker build -t fips-test:latest "$DOCKER_DIR"
|
|
docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR"
|
|
fi
|
|
|
|
# ── Start nodes ────────────────────────────────────────────────────────────
|
|
#
|
|
# Chain topology: A — B — C
|
|
# node-a: no outbound peers (accepts inbound from B)
|
|
# node-b: peers with A (middle node, transit router)
|
|
# node-c: peers with B (end node)
|
|
|
|
alloc_network || exit 1
|
|
|
|
log "Starting node-a (no peers, joins the claimed network)..."
|
|
# node-a is the root: explicitly clear FIPS_PEER_* so it does not inherit the
|
|
# external peer default from .env (which points at a real public mesh node).
|
|
# Without this, node-a auto-connects to the live mesh and the chain attaches
|
|
# under an external root, inflating tree depth and breaking isolation.
|
|
FIPS_NSEC="$NODE_A_NSEC" \
|
|
FIPS_PEER_NPUB="" \
|
|
FIPS_PEER_ADDR="" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_A_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_A" up -d
|
|
|
|
log "Starting node-b (peers with node-a, joins external network)..."
|
|
FIPS_NSEC="$NODE_B_NSEC" \
|
|
FIPS_PEER_NPUB="$NODE_A_NPUB" \
|
|
FIPS_PEER_ADDR="${NODE_A_IP}:2121" \
|
|
FIPS_PEER_ALIAS="node-a" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_B_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_B" up -d
|
|
|
|
log "Starting node-c (peers with node-b, joins external network)..."
|
|
FIPS_NSEC="$NODE_C_NSEC" \
|
|
FIPS_PEER_NPUB="$NODE_B_NPUB" \
|
|
FIPS_PEER_ADDR="${NODE_B_IP}:2121" \
|
|
FIPS_PEER_ALIAS="node-b" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_C_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_C" up -d
|
|
|
|
# ── Wait for convergence ──────────────────────────────────────────────────
|
|
|
|
log "Waiting for link establishment (up to ${CONVERGE_TIMEOUT}s)..."
|
|
|
|
converged=false
|
|
for i in $(seq 1 "$CONVERGE_TIMEOUT"); do
|
|
# node-b should have 2 links (A and C)
|
|
link_count=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
if [ "$link_count" -ge 2 ]; then
|
|
converged=true
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if [ "$converged" = true ]; then
|
|
log "Links established after ${i}s"
|
|
else
|
|
log "TIMEOUT: links did not converge in ${CONVERGE_TIMEOUT}s"
|
|
log "node-a links:"
|
|
docker exec "${PROJ_A}-fips-1" fipsctl show links 2>&1 || true
|
|
log "node-b links:"
|
|
docker exec "${PROJ_B}-fips-1" fipsctl show links 2>&1 || true
|
|
log "node-c links:"
|
|
docker exec "${PROJ_C}-fips-1" fipsctl show links 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
# Wait for end-to-end multi-hop connectivity (the same app-to-app pings
|
|
# the test asserts on) with a progress-aware deadline, instead of a
|
|
# blind fixed sleep that can fire before coordinates propagate across the
|
|
# chain. The directed pings below remain the actual assertions.
|
|
_sidecar_converged() {
|
|
PASSED=0; FAILED=0
|
|
if docker exec "${PROJ_B}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
if docker exec "${PROJ_C}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
if docker exec "${PROJ_A}-app-1" ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
}
|
|
wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true
|
|
|
|
# ── Link verification ─────────────────────────────────────────────────────
|
|
|
|
log "Verifying link counts..."
|
|
|
|
a_links=$(docker exec "${PROJ_A}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
b_links=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
c_links=$(docker exec "${PROJ_C}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
|
|
[ "$a_links" -ge 1 ] && pass "node-a has $a_links link(s)" || fail "node-a has $a_links links (expected >= 1)"
|
|
[ "$b_links" -ge 2 ] && pass "node-b has $b_links link(s)" || fail "node-b has $b_links links (expected >= 2)"
|
|
[ "$c_links" -ge 1 ] && pass "node-c has $c_links link(s)" || fail "node-c has $c_links links (expected >= 1)"
|
|
|
|
# ── Direct connectivity (adjacent nodes) ──────────────────────────────────
|
|
|
|
log "Testing direct connectivity (B app → A via fips0)..."
|
|
|
|
if docker exec "${PROJ_B}-app-1" ping6 -c2 -W5 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-b app can ping node-a via fips0"
|
|
else
|
|
fail "node-b app cannot ping node-a via fips0"
|
|
fi
|
|
|
|
# ── Multi-hop connectivity (C → A through B) ─────────────────────────────
|
|
|
|
log "Testing multi-hop connectivity (C app → A via fips0, through B)..."
|
|
|
|
if docker exec "${PROJ_C}-app-1" ping6 -c2 -W10 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-c app can ping node-a via fips0 (multi-hop through B)"
|
|
else
|
|
fail "node-c app cannot ping node-a via fips0 (multi-hop through B)"
|
|
fi
|
|
|
|
# ── Reverse direction (A → C through B) ──────────────────────────────────
|
|
|
|
log "Testing reverse multi-hop (A app → C via fips0, through B)..."
|
|
|
|
if docker exec "${PROJ_A}-app-1" ping6 -c2 -W10 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-a app can ping node-c via fips0 (multi-hop through B)"
|
|
else
|
|
fail "node-a app cannot ping node-c via fips0 (multi-hop through B)"
|
|
fi
|
|
|
|
# ── Network isolation verification ────────────────────────────────────────
|
|
#
|
|
# This is the critical security assertion: app containers must NOT be able
|
|
# to reach anything outside the FIPS mesh.
|
|
|
|
log "Verifying network isolation on app containers..."
|
|
|
|
for node in a b c; do
|
|
container="sidecar-${node}${FIPS_CI_NAME_SUFFIX:-}-app-1"
|
|
# Fail loudly if the container is missing. Without this the two isolation
|
|
# assertions below pass vacuously: they assert a ping FAILS, and a ping into
|
|
# a non-existent container fails for the wrong reason. Only the loopback
|
|
# check, which expects success, would notice — so a naming slip here would
|
|
# silently turn the security assertions into no-ops.
|
|
docker inspect "$container" >/dev/null 2>&1 \
|
|
|| { fail "$container does not exist — isolation assertions cannot be trusted"; continue; }
|
|
# Pick a peer IP that isn't this node's own address
|
|
case $node in
|
|
a) peer_ip="$NODE_B_IP" ;;
|
|
b) peer_ip="$NODE_C_IP" ;;
|
|
c) peer_ip="$NODE_A_IP" ;;
|
|
esac
|
|
log " Checking $container..."
|
|
|
|
# IPv4 gateway should be unreachable (iptables DROP on eth0)
|
|
if docker exec "$container" ping -c1 -W2 "$GATEWAY_IP" >/dev/null 2>&1; then
|
|
fail "$container can reach IPv4 gateway (isolation broken!)"
|
|
else
|
|
pass "$container cannot reach IPv4 gateway (IPv4 blocked)"
|
|
fi
|
|
|
|
# IPv4 peer should be unreachable (iptables DROP on eth0)
|
|
if docker exec "$container" ping -c1 -W2 "$peer_ip" >/dev/null 2>&1; then
|
|
fail "$container can reach peer IPv4 (isolation broken!)"
|
|
else
|
|
pass "$container cannot reach peer IPv4 (IPv4 blocked)"
|
|
fi
|
|
|
|
# Loopback should work
|
|
if docker exec "$container" ping -c1 -W2 127.0.0.1 >/dev/null 2>&1; then
|
|
pass "$container can reach loopback (expected)"
|
|
else
|
|
fail "$container cannot reach loopback"
|
|
fi
|
|
done
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────
|
|
|
|
echo ""
|
|
log "Results: $PASSED passed, $FAILED failed"
|
|
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
log "Dumping logs for failed run..."
|
|
for node in a b c; do
|
|
echo "--- sidecar-${node} logs ---"
|
|
docker logs "sidecar-${node}${FIPS_CI_NAME_SUFFIX:-}-fips-1" 2>&1 | tail -30
|
|
echo ""
|
|
done
|
|
exit 1
|
|
fi
|
|
|
|
log "All tests passed."
|