mirror of
https://github.com/jmcorgan/fips.git
synced 2026-10-30 07:37:36 +00:00
Johnathan Corgan
4cab546c61
Bloom filter, session, native API, DNS and TUN security fixes, the held over-cap filter, and the Debian, FreeBSD, Windows and OpenWrt packaging fixes. The DNS responder's child moved to src/ipv6tun/lifecycle.rs on master, so the mesh-side filter input (dns_mesh_side and the fallback warning) is applied there rather than in the node lifecycle, and the IPV6_PKTINFO helper becomes crate-visible in src/ipv6tun/dns.rs beside the socket helpers master already moved there. The identity store's session fallback reads master's proven peer key (SessionEntry::peer_key). The bloom recompute counter the root crate's tests read is a test-hooks item, since fips-core is its own crate here. The changelog entries go into master's Unreleased sections by category.
1541 lines
66 KiB
Bash
Executable File
1541 lines
66 KiB
Bash
Executable File
#!/bin/bash
|
|
# Test the fips Debian package install path across target distros.
|
|
#
|
|
# Each scenario takes the .deb from --deb, or builds (or reuses) it
|
|
# through packaging/debian/build-deb-container.sh, boots a systemd
|
|
# container with TUN access for the target distro, installs the .deb
|
|
# via `apt-get install -y --no-install-recommends ./fips_*.deb`, waits
|
|
# for fips.service + fips-dns.service to come up, and verifies that
|
|
# `dig @127.0.0.53 AAAA <npub>.fips` returns a non-empty AAAA answer
|
|
# through the resolver backend that fips-dns-setup configured. Then
|
|
# exercises fips-gateway against the same daemon to verify the
|
|
# gateway/daemon default-pairing. Finally it
|
|
# removes the package with the DNS routing file planted and fips-dns
|
|
# stopped, and checks the file is removed and systemd-resolved restarted,
|
|
# then plants the file again and purges the package from config-files
|
|
# state, with the same checks.
|
|
#
|
|
# This is the most thorough test surface — it exercises:
|
|
# - cargo deb packaging (binary stripping, dependency declaration)
|
|
# - dpkg conffile placement (/etc/fips/fips.nft) and postinst seeding
|
|
# of /etc/fips/fips.yaml
|
|
# - postinst maintainer scripts (systemd unit enablement,
|
|
# fips-dns.service running fips-dns-setup)
|
|
# - postrm remove and postrm purge (removing the DNS routing
|
|
# fips-dns-setup wrote)
|
|
# - The fips, fips-dns, and (optionally) fips-gateway systemd units
|
|
# - End-to-end .fips resolution as a real user would experience it
|
|
#
|
|
# Usage: ./test.sh [--deb PATH] [scenario ...]
|
|
# No args = run all scenarios.
|
|
# Named args = run only those (e.g., ./test.sh ubuntu26 debian12)
|
|
# --deb PATH = install that package instead of building one.
|
|
#
|
|
# Requirements: Docker able to grant SYS_ADMIN and NET_ADMIN and an
|
|
# unconfined AppArmor profile (the containers are not privileged; see
|
|
# testing/lib/systemd-container.sh), /dev/net/tun on the host (standard).
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck source=SCRIPTDIR/../lib/systemd-container.sh
|
|
source "$SCRIPT_DIR/../lib/systemd-container.sh"
|
|
# shellcheck source=SCRIPTDIR/../lib/systemd-scenario.sh
|
|
source "$SCRIPT_DIR/../lib/systemd-scenario.sh"
|
|
# shellcheck source=SCRIPTDIR/../lib/image-build.sh
|
|
source "$SCRIPT_DIR/../lib/image-build.sh"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
CACHE_DIR="$SCRIPT_DIR/.cache"
|
|
DEB_CACHE_DIR="$CACHE_DIR/deb"
|
|
|
|
# Timeouts. Each wait loop below exits as soon as its condition is met.
|
|
BOOT_TIMEOUT=30
|
|
SERVICE_TIMEOUT=20
|
|
DAEMON_TIMEOUT=15
|
|
# Bounds on a single `systemctl start`, which is not a wait loop and needs its
|
|
# own limit. See start_unit() for why an unbounded one can never return.
|
|
UNIT_START_TIMEOUT=30
|
|
# fips-gateway.service's ExecStartPre waits up to 30s for fips0 to appear, by
|
|
# design, so its start legitimately takes longer than any other.
|
|
GATEWAY_START_TIMEOUT=60
|
|
# The gateway-enable block restarts fips.service inside the container. Above
|
|
# systemd's default TimeoutStopSec of 90s, so a wedged stop trips this rather
|
|
# than this cutting a healthy stop short.
|
|
CONFIG_RESTART_TIMEOUT=120
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
SKIP=0
|
|
|
|
# Set by --deb. DEB_PREPARED keeps the copy-into-cache to a single operation
|
|
# however many scenarios run in one process.
|
|
SUPPLIED_DEB=""
|
|
DEB_PREPARED=0
|
|
# The package the scenarios install, set by build_deb() on every path that
|
|
# succeeds. Scenarios use it rather than listing the cache directory, so which
|
|
# file they install never depends on what else happens to be in there.
|
|
DEB_PATH=""
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Helpers
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
log() { echo "=== $*"; }
|
|
pass() { echo " PASS: $*"; PASS=$((PASS + 1)); }
|
|
fail() { echo " FAIL: $*"; FAIL=$((FAIL + 1)); }
|
|
skip() { echo " SKIP: $*"; SKIP=$((SKIP + 1)); }
|
|
|
|
# Remove a per-run image. A run that is killed before this leaves the image to
|
|
# ci-local.sh's teardown, which finds it by the run label it was built with.
|
|
remove_image() {
|
|
docker rmi "$1" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Fail unless the container holds this run's package, byte for byte, where the
|
|
# image put it. The image tag is scoped to the run, so another run cannot
|
|
# rebuild it; this is the check that a tag change has not undone that. Two
|
|
# runs of one commit build packages of the same name, so the name alone proves
|
|
# nothing: compare the hash.
|
|
check_holds_package() {
|
|
local name="$1" deb="$2" path="$3" want have
|
|
want=$(sha256sum "$deb" | cut -d' ' -f1)
|
|
have=$(docker exec "$name" sha256sum "$path" 2>/dev/null | cut -d' ' -f1)
|
|
if [ -z "$want" ] || [ "$want" != "$have" ]; then
|
|
fail "$name does not hold this run's package at $path (sha256 '${have}', want '${want}')"
|
|
return 1
|
|
fi
|
|
pass "$name holds this run's package (sha256 ${want:0:16})"
|
|
return 0
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Build the .deb once through packaging/debian/build-deb-container.sh,
|
|
# whose image is set in packaging/build-floor.env (cached between
|
|
# runs). Output cached at testing/deb-install/.cache/deb/.
|
|
# Rebuilt if any source/Cargo/packaging file is newer than the cached
|
|
# .deb, or if the .deb is missing.
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
build_deb() {
|
|
mkdir -p "$DEB_CACHE_DIR"
|
|
|
|
# A supplied package wins outright and bypasses the staleness check below.
|
|
# That check compares mtimes, so a cached package could otherwise beat the
|
|
# artifact the caller explicitly handed over, which is exactly the silent
|
|
# substitution this suite exists to stop making.
|
|
if [ -n "$SUPPLIED_DEB" ]; then
|
|
if [ "$DEB_PREPARED" -eq 1 ]; then
|
|
return 0
|
|
fi
|
|
if [ ! -f "$SUPPLIED_DEB" ]; then
|
|
echo " ERROR: --deb $SUPPLIED_DEB does not exist" >&2
|
|
return 1
|
|
fi
|
|
rm -f "$DEB_CACHE_DIR"/*.deb
|
|
cp "$SUPPLIED_DEB" "$DEB_CACHE_DIR/"
|
|
DEB_PATH="$DEB_CACHE_DIR/$(basename "$SUPPLIED_DEB")"
|
|
DEB_PREPARED=1
|
|
log "Installing the supplied package $(basename "$SUPPLIED_DEB")"
|
|
return 0
|
|
fi
|
|
|
|
local cached_deb
|
|
cached_deb=$(ls "$DEB_CACHE_DIR"/fips_*_amd64.deb 2>/dev/null | head -1)
|
|
|
|
if [ -n "$cached_deb" ] && [ -f "$cached_deb" ]; then
|
|
local newest_src
|
|
newest_src=$(find "$REPO_ROOT/src" "$REPO_ROOT/core" "$REPO_ROOT/Cargo.toml" \
|
|
"$REPO_ROOT/Cargo.lock" "$REPO_ROOT/packaging" \
|
|
-type f -printf '%T@\n' 2>/dev/null | sort -nr | head -1)
|
|
local cached_age
|
|
cached_age=$(stat -c '%Y' "$cached_deb" 2>/dev/null || echo 0)
|
|
if awk "BEGIN { exit !($cached_age >= $newest_src) }"; then
|
|
DEB_PATH="$cached_deb"
|
|
log "Using cached .deb at $cached_deb"
|
|
return 0
|
|
fi
|
|
log "Cached .deb is stale, rebuilding"
|
|
else
|
|
log "No cached .deb, building"
|
|
fi
|
|
|
|
# Build through the shared container script rather than a builder defined
|
|
# here. This suite used to compile its own package inside a Debian 12 image
|
|
# while the release compiled on the newest GitHub runner, so the package it
|
|
# tested had a lower glibc floor than the package users received and could
|
|
# not exhibit a defect that only the release environment produced. It stayed
|
|
# green through five releases that could not start on two of the five
|
|
# distributions in its own matrix.
|
|
#
|
|
# The cache holds one package at a time. Clearing it first is what keeps the
|
|
# reuse check above honest, since that check looks at whichever package it
|
|
# finds; and the package installed is the one the build names on the last
|
|
# line of its stdout, never one found by listing the directory.
|
|
log "Building the .deb in the pinned build container (slow on first run)"
|
|
rm -f "$DEB_CACHE_DIR"/*.deb
|
|
local build_out
|
|
if ! build_out=$(bash "$REPO_ROOT/packaging/debian/build-deb-container.sh" \
|
|
--output-dir "$DEB_CACHE_DIR"); then
|
|
echo " ERROR: container build failed" >&2
|
|
return 1
|
|
fi
|
|
|
|
cached_deb=$(printf '%s\n' "$build_out" | tail -n 1)
|
|
if [ -z "$cached_deb" ] || [ ! -f "$cached_deb" ]; then
|
|
echo " ERROR: the container build did not report a package path: '$cached_deb'" >&2
|
|
return 1
|
|
fi
|
|
DEB_PATH="$cached_deb"
|
|
log "Cached at $cached_deb ($(stat -c %s "$cached_deb") bytes)"
|
|
return 0
|
|
}
|
|
|
|
# The packages a runtime image installs on top of the distro base image.
|
|
# Ubuntu 22.04 bundles systemd-resolved into systemd; other distros require it
|
|
# as a separate package.
|
|
runtime_packages() {
|
|
local base_image="$1"
|
|
if [ "$base_image" = "ubuntu:22.04" ]; then
|
|
echo "systemd iproute2 dbus dnsutils procps"
|
|
else
|
|
echo "systemd systemd-resolved iproute2 dbus dnsutils procps"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Patch a minimal gateway config into the container's fips.yaml, since the
|
|
# shipped one has the gateway disabled, and restart fips.service to load it.
|
|
# The caller checks that the daemon came back.
|
|
apply_gateway_config() {
|
|
local name="$1"
|
|
timeout "$CONFIG_RESTART_TIMEOUT" docker exec "$name" bash -c '
|
|
systemctl unmask fips-gateway.service 2>/dev/null
|
|
cp /etc/fips/fips.yaml /etc/fips/fips.yaml.orig
|
|
cat >> /etc/fips/fips.yaml <<EOF
|
|
gateway:
|
|
enabled: true
|
|
pool: "fd01::/112"
|
|
lan_interface: "eth0"
|
|
EOF
|
|
systemctl restart fips.service
|
|
' >/dev/null 2>&1
|
|
return
|
|
}
|
|
|
|
# The installed gateway, on the default config, serves .fips on its default
|
|
# listen address. Each check needs the gateway running: a gateway that exited
|
|
# fails the first one rather than letting the others pass on nothing.
|
|
# Args: <name> <npub>, the daemon's npub to resolve through the gateway.
|
|
check_gateway_default_listener() {
|
|
local name="$1" npub="$2"
|
|
local journal="" _i
|
|
for _i in $(seq 1 5); do
|
|
journal=$(docker exec "$name" journalctl -u fips-gateway.service --no-pager 2>/dev/null) || journal=""
|
|
printf '%s\n' "$journal" | grep -q "fips-gateway running" && break
|
|
sleep 1
|
|
done
|
|
if ! printf '%s\n' "$journal" | grep -q "fips-gateway running"; then
|
|
fail "the installed fips-gateway did not reach 'fips-gateway running', so its default listener was not observed"
|
|
echo " --- fips-gateway journal ---"
|
|
printf '%s\n' "$journal" | tail -15
|
|
return
|
|
fi
|
|
|
|
local sockets
|
|
sockets=$(docker exec "$name" ss -Hulnp 'sport = :5365' 2>/dev/null) || sockets=""
|
|
if printf '%s\n' "$sockets" | grep -F '[::1]:5365' | grep -q 'fips-gateway'; then
|
|
pass "fips-gateway listens on its default [::1]:5365"
|
|
else
|
|
fail "no fips-gateway socket on [::1]:5365: '$sockets'"
|
|
fi
|
|
|
|
local answer
|
|
answer=$(docker exec "$name" dig +short +tries=1 +time=3 @::1 -p 5365 AAAA "${npub}.fips" 2>&1)
|
|
if printf '%s\n' "$answer" | grep -qE '^fd01::[0-9a-f]{1,4}$'; then
|
|
pass "the gateway answers ${npub}.fips on [::1]:5365 from its fd01::/112 pool"
|
|
else
|
|
fail "the gateway did not answer ${npub}.fips on [::1]:5365 from its pool: '$answer'"
|
|
fi
|
|
}
|
|
|
|
# Print systemd-resolved's unit status and the tail of its journal, for a
|
|
# failure report. The status is printed without its own journal lines so that
|
|
# its Active line, which carries the unit's Result, is not cut off.
|
|
show_resolved() {
|
|
local name="$1"
|
|
echo " --- systemd-resolved status ---"
|
|
cexec "$name" systemctl status --no-pager --lines=0 systemd-resolved 2>&1
|
|
echo " --- systemd-resolved journal ---"
|
|
cexec "$name" journalctl -u systemd-resolved --no-pager -n 20 2>&1
|
|
return 0
|
|
}
|
|
|
|
# Restart systemd-resolved and wait for it to be active. Returns 1, having
|
|
# recorded a failure with <label> in front, if it does not come back.
|
|
#
|
|
# The restart lands seconds after several others: fips-dns-teardown and
|
|
# fips-dns-setup each restart resolved when the gateway block restarts the
|
|
# daemon, the teardown does again when the remove check stops fips-dns, and
|
|
# postrm does on the remove. By default systemd refuses a sixth start within one
|
|
# 10 s start-limit window (StartLimitBurst=5). The window is fixed, not sliding:
|
|
# it opens at the first start after the previous window has expired, so whether
|
|
# this restart is refused depends on where the window opened, not only on how
|
|
# fast the run is. A refusal left resolved failed and the `resolvectl status`
|
|
# after it waiting 25 s on D-Bus activation. Clearing the unit's failed state
|
|
# and start counter first makes this restart independent of the ones before it.
|
|
restart_resolved() {
|
|
local name="$1" label="$2" out
|
|
cexec "$name" systemctl reset-failed systemd-resolved >/dev/null 2>&1
|
|
if ! out=$(cexec "$name" systemctl restart systemd-resolved 2>&1); then
|
|
fail "$label: restarting systemd-resolved failed before the $label: $out"
|
|
show_resolved "$name"
|
|
return 1
|
|
fi
|
|
if ! wait_for_service_active "$name" systemd-resolved; then
|
|
fail "$label: systemd-resolved not active ${SERVICE_TIMEOUT}s after its restart before the $label"
|
|
show_resolved "$name"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Put the saved DNS routing file back at <file> and restart systemd-resolved,
|
|
# then check the state in which removal leaves the file behind: the file in
|
|
# place, fips-dns.service not active, and the resolver routing .fips to
|
|
# [::1]:5354. Every failure is recorded with <label> in front.
|
|
#
|
|
# Args: <name> <file> <label>. Returns 1 if any precondition failed.
|
|
plant_routing() {
|
|
local name="$1" file="$2" label="$3"
|
|
cexec "$name" mkdir -p "$(dirname "$file")"
|
|
cexec "$name" cp /root/fips-dns.saved "$file"
|
|
|
|
local ok=1 status
|
|
if ! cexec "$name" sh -c "test -s '$file' && cmp -s '$file' /root/fips-dns.saved"; then
|
|
fail "$label: $file not restored before the $label"
|
|
ok=0
|
|
fi
|
|
if cexec "$name" systemctl is-active --quiet fips-dns.service; then
|
|
fail "$label: fips-dns.service still active before the $label"
|
|
ok=0
|
|
fi
|
|
# A resolver that did not come back would only time out the resolvectl
|
|
# check, so that check is not run on top of the failure already recorded.
|
|
restart_resolved "$name" "$label" || return 1
|
|
# Captured rather than piped into grep -q: under pipefail, grep closing the
|
|
# pipe early can fail the pipeline on a match.
|
|
if ! status=$(cexec "$name" resolvectl status 2>&1); then
|
|
fail "$label: resolvectl status failed before the $label"
|
|
echo "$status" | tail -10
|
|
ok=0
|
|
elif ! grep -q ':5354' <<<"$status"; then
|
|
fail "$label: resolvectl status does not show $file in effect before the $label"
|
|
echo "$status" | tail -25
|
|
ok=0
|
|
fi
|
|
[ "$ok" = 1 ]
|
|
}
|
|
|
|
# Check that the apt run just made removed <file> and restarted
|
|
# systemd-resolved, and that the resolver no longer routes to [::1]:5354.
|
|
#
|
|
# Args: <name> <file> <label> <InvocationID of systemd-resolved before the run>
|
|
expect_cleared() {
|
|
local name="$1" file="$2" label="$3" before="$4"
|
|
# test exits 1 for a missing file; any other failure is docker exec's.
|
|
local rc=0 after status
|
|
cexec "$name" test -e "$file" || rc=$?
|
|
case "$rc" in
|
|
1) pass "$label removed $file" ;;
|
|
0) fail "$label left $file behind" ;;
|
|
*) fail "$label: could not check for $file (exit $rc)" ;;
|
|
esac
|
|
after=$(cexec "$name" systemctl show -p InvocationID --value systemd-resolved)
|
|
if [ -n "$before" ] && [ -n "$after" ] && [ "$before" != "$after" ]; then
|
|
pass "$label restarted systemd-resolved"
|
|
else
|
|
fail "$label did not restart systemd-resolved (InvocationID '$before' -> '$after')"
|
|
fi
|
|
if ! status=$(cexec "$name" resolvectl status 2>&1); then
|
|
fail "$label: resolvectl status failed after the $label"
|
|
echo "$status" | tail -10
|
|
elif grep -q ':5354' <<<"$status"; then
|
|
fail "$label: resolvectl status still routes to port 5354"
|
|
echo "$status" | tail -25
|
|
else
|
|
pass "$label: resolvectl status no longer routes to port 5354"
|
|
fi
|
|
return
|
|
}
|
|
|
|
# Remove the package with the DNS routing file planted and fips-dns stopped, and
|
|
# check that postrm removes the file and restarts systemd-resolved.
|
|
#
|
|
# Stopping fips-dns runs fips-dns-teardown, which removes the file; putting it
|
|
# back gives the state in which removal leaves it behind: a live delegation and
|
|
# an inactive fips-dns, so prerm's stop runs no teardown. Only postrm is left
|
|
# to clean up, and a file it misses keeps the resolver sending .fips to
|
|
# [::1]:5354 after nothing listens there.
|
|
#
|
|
# Args: <name> <expected_backend>, the backend the scenario expects
|
|
# fips-dns-setup to pick (dns-delegate or global-drop-in). Returns 0 once
|
|
# apt-get remove has succeeded, whatever the checks after it found, so the
|
|
# purge check can follow; 1, having recorded why, if anything before that
|
|
# failed.
|
|
check_remove_clears_dns() {
|
|
local name="$1" backend="$2" file
|
|
case "$backend" in
|
|
dns-delegate) file=/etc/systemd/dns-delegate.d/fips.dns-delegate ;;
|
|
global-drop-in) file=/etc/systemd/resolved.conf.d/fips.conf ;;
|
|
*)
|
|
fail "remove: no DNS routing file known for backend '$backend'"
|
|
return 1
|
|
;;
|
|
esac
|
|
|
|
# The gateway-enable restart of fips.service is passed on to fips-dns
|
|
# (Requires=fips.service), whose setup waits for fips0 before it writes the
|
|
# file, and nothing since has waited for it. After=fips.service stops the
|
|
# old instance before the daemon, so active here means the new setup ran.
|
|
if ! wait_for_service_active "$name" fips-dns.service; then
|
|
fail "remove: fips-dns.service not active again after the gateway-enable restart"
|
|
echo " --- fips-dns.service journal ---"
|
|
docker exec "$name" journalctl -u fips-dns.service --no-pager 2>&1 | tail -20
|
|
return 1
|
|
fi
|
|
if ! cexec "$name" test -f "$file"; then
|
|
fail "remove: $file not written by fips-dns-setup before the remove"
|
|
return 1
|
|
fi
|
|
# Saved inside the container: cexec runs docker exec without -i, so a
|
|
# copy piped back from the host would arrive empty.
|
|
if ! cexec "$name" cp "$file" /root/fips-dns.saved; then
|
|
fail "remove: could not save $file"
|
|
return 1
|
|
fi
|
|
|
|
cexec "$name" systemctl stop fips-dns.service >/dev/null 2>&1
|
|
plant_routing "$name" "$file" remove || return 1
|
|
|
|
# The runtime image's tag is shared by every run on the host, so show which
|
|
# postrm this run is testing.
|
|
local branches
|
|
branches=$(cexec "$name" grep -E '^[[:space:]]*[a-z|-]+\)[[:space:]]*$' \
|
|
/var/lib/dpkg/info/fips.postrm 2>&1 | tr -s ' \n' ' ')
|
|
echo " installed postrm branches: $branches"
|
|
|
|
local before
|
|
before=$(cexec "$name" systemctl show -p InvocationID --value systemd-resolved)
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" remove -y fips
|
|
echo " remove took ${APT_SECS}s"
|
|
if [ "$APT_RC" -ne 0 ]; then
|
|
fail "remove: apt-get remove exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
return 1
|
|
fi
|
|
|
|
expect_cleared "$name" "$file" remove "$before"
|
|
return 0
|
|
}
|
|
|
|
# Purge the removed package with the DNS routing file planted again, and check
|
|
# that postrm removes the file and restarts systemd-resolved.
|
|
#
|
|
# The package is in config-files (rc) state after the remove, so dpkg runs only
|
|
# postrm purge. Purging an installed package would run postrm remove first,
|
|
# which clears the file and leaves this check unable to fail on the purge
|
|
# branch.
|
|
#
|
|
# Args: <name> <expected_backend>, as for check_remove_clears_dns, which must
|
|
# have run first: it saves the file this plants.
|
|
check_purge_clears_dns() {
|
|
local name="$1" backend="$2" file state
|
|
case "$backend" in
|
|
dns-delegate) file=/etc/systemd/dns-delegate.d/fips.dns-delegate ;;
|
|
global-drop-in) file=/etc/systemd/resolved.conf.d/fips.conf ;;
|
|
*)
|
|
fail "purge: no DNS routing file known for backend '$backend'"
|
|
return
|
|
;;
|
|
esac
|
|
|
|
# shellcheck disable=SC2016 # a dpkg-query format field, not a shell expansion
|
|
state=$(cexec "$name" dpkg-query -W -f='${db:Status-Abbrev}' fips 2>&1)
|
|
state="${state%"${state##*[![:space:]]}"}"
|
|
if [ "$state" != "rc" ]; then
|
|
fail "purge: fips is in state '$state' after the remove, not config-files (rc)"
|
|
return
|
|
fi
|
|
|
|
plant_routing "$name" "$file" purge || return
|
|
|
|
local before
|
|
before=$(cexec "$name" systemctl show -p InvocationID --value systemd-resolved)
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" purge -y fips
|
|
echo " purge took ${APT_SECS}s"
|
|
if [ "$APT_RC" -ne 0 ]; then
|
|
fail "purge: apt-get purge exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
return
|
|
fi
|
|
|
|
expect_cleared "$name" "$file" purge "$before"
|
|
return
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Scenario runner
|
|
#
|
|
# Args: <distro_label> <docker_base_image>
|
|
# distro_label: short tag for container/image names (e.g. "debian12")
|
|
# docker_base_image: e.g. "debian:12", "ubuntu:26.04"
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
# The image bakes in this run's package, so its tag carries the run's suffix
|
|
# like the container name: under one shared tag, a concurrent run that builds
|
|
# between this run's build and its boot makes this run install the other
|
|
# run's package. The image goes when the scenario ends, however it ends.
|
|
_run_deb_install_scenario() {
|
|
local distro_label="$1"
|
|
local image="fips-deb-test:${distro_label}${FIPS_CI_NAME_SUFFIX:-}"
|
|
_deb_install_scenario "$distro_label" "$2" "$image"
|
|
remove_image "$image"
|
|
return 0
|
|
}
|
|
|
|
# Args: <distro_label> <docker_base_image> <image tag>
|
|
_deb_install_scenario() {
|
|
local distro_label="$1"
|
|
local base_image="$2"
|
|
local image="$3"
|
|
|
|
local name="fips-deb-test-${distro_label}${FIPS_CI_NAME_SUFFIX:-}"
|
|
log ".deb install: ${base_image}"
|
|
|
|
build_deb || { fail ".deb build failed"; return; }
|
|
|
|
local cached_deb="$DEB_PATH"
|
|
if [ -z "$cached_deb" ] || [ ! -f "$cached_deb" ]; then
|
|
fail "no .deb available at $DEB_CACHE_DIR"
|
|
return
|
|
fi
|
|
local deb_basename
|
|
deb_basename=$(basename "$cached_deb")
|
|
|
|
local apt_packages
|
|
apt_packages=$(runtime_packages "$base_image")
|
|
|
|
log "Building ${base_image} runtime image"
|
|
cp "$cached_deb" "$CACHE_DIR/deb-for-image"
|
|
# Place the .deb under /opt — systemd remounts /tmp during boot
|
|
# (PrivateTmp / tmpfs) which would wipe a .deb COPY'd to /tmp.
|
|
build_image "$image" "$(cat <<DOCKERFILE
|
|
FROM ${base_image}
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
${apt_packages} && \\
|
|
apt-get clean && rm -rf /var/lib/apt/lists/* && \\
|
|
systemctl enable systemd-resolved && \\
|
|
mkdir -p /opt/fips-deb
|
|
COPY testing/deb-install/.cache/deb-for-image /opt/fips-deb/${deb_basename}
|
|
CMD ["/lib/systemd/systemd"]
|
|
DOCKERFILE
|
|
)" "${CI_LABELS[@]}" || { fail "runtime build failed"; rm -f "$CACHE_DIR/deb-for-image"; return; }
|
|
rm -f "$CACHE_DIR/deb-for-image"
|
|
|
|
start_systemd_container_with_tun "$name" "$image"
|
|
wait_for_systemd "$name" || {
|
|
fail "systemd did not boot in $name; the install checks below would read an unstarted system"
|
|
cleanup_container "$name"
|
|
return
|
|
}
|
|
check_holds_package "$name" "$cached_deb" "/opt/fips-deb/${deb_basename}" || {
|
|
cleanup_container "$name"
|
|
return
|
|
}
|
|
|
|
# Install the .deb. apt handles dependencies (libc6, systemd,
|
|
# libdbus-1-3) and runs the maintainer scripts (postinst →
|
|
# systemctl enable fips.service; fips-dns.service starts and
|
|
# runs fips-dns-setup).
|
|
log "Installing .deb (apt install /opt/fips-deb/${deb_basename})"
|
|
# `|| true` here used to discard apt's exit status, and an empty capture (a
|
|
# failed `docker exec`) matches neither error pattern below, so a install
|
|
# that never ran reached `pass "apt install completed"`. Keep the capture on
|
|
# failure so the diagnostics below can print it, but remember the status.
|
|
local install_output install_rc=0
|
|
install_output=$(docker exec "$name" bash -c "
|
|
apt-get update >/dev/null 2>&1
|
|
cd /opt/fips-deb && apt-get install -y --no-install-recommends ./${deb_basename} 2>&1
|
|
") || install_rc=$?
|
|
if [ "$install_rc" -ne 0 ]; then
|
|
fail "apt install exited $install_rc"
|
|
echo "$install_output" | tail -20
|
|
cleanup_container "$name"
|
|
return
|
|
fi
|
|
if echo "$install_output" | grep -qE "^E:|errors? were encountered"; then
|
|
fail "apt install reported errors"
|
|
echo "$install_output" | tail -20
|
|
cleanup_container "$name"
|
|
return
|
|
else
|
|
pass "apt install completed"
|
|
fi
|
|
|
|
# Verify shipped files landed where expected.
|
|
if docker exec "$name" test -x /usr/bin/fips; then
|
|
pass "/usr/bin/fips installed"
|
|
else
|
|
fail "/usr/bin/fips missing"
|
|
fi
|
|
if docker exec "$name" test -x /usr/bin/fips-gateway; then
|
|
pass "/usr/bin/fips-gateway installed"
|
|
else
|
|
fail "/usr/bin/fips-gateway missing"
|
|
fi
|
|
if docker exec "$name" test -f /etc/fips/fips.yaml; then
|
|
pass "/etc/fips/fips.yaml seeded by postinst"
|
|
else
|
|
fail "/etc/fips/fips.yaml missing"
|
|
fi
|
|
|
|
# Verify fips.service is enabled (postinst enables but does not
|
|
# start on fresh install — standard Debian convention).
|
|
if docker exec "$name" systemctl is-enabled --quiet fips.service; then
|
|
pass "fips.service enabled by postinst"
|
|
else
|
|
fail "fips.service not enabled after install"
|
|
fi
|
|
if docker exec "$name" systemctl is-enabled --quiet fips-dns.service; then
|
|
pass "fips-dns.service enabled by postinst"
|
|
else
|
|
fail "fips-dns.service not enabled after install"
|
|
fi
|
|
|
|
# ── nftables firewall baseline (v0.3.0) ──────────────────────────
|
|
# The fips-firewall.service unit ships installed but DISABLED by
|
|
# default; operators opt in explicitly. The fips.nft ruleset is a
|
|
# dpkg conffile, and /etc/fips/fips.d/ is a drop-in directory the
|
|
# ruleset includes via a glob. None of these are exercised by the
|
|
# service-start path below — verify them as static install state.
|
|
if docker exec "$name" test -f /lib/systemd/system/fips-firewall.service; then
|
|
pass "fips-firewall.service unit installed at /lib/systemd/system/"
|
|
else
|
|
fail "fips-firewall.service unit missing from /lib/systemd/system/"
|
|
fi
|
|
# is-enabled prints 'disabled' (and exits non-zero) for an
|
|
# installed-but-not-enabled unit; capture stdout, don't gate on rc.
|
|
fw_state=$(docker exec "$name" systemctl is-enabled fips-firewall.service 2>/dev/null || true)
|
|
if [ "$fw_state" = "disabled" ]; then
|
|
pass "fips-firewall.service disabled by default (opt-in)"
|
|
else
|
|
fail "fips-firewall.service unexpected state: '$fw_state' (expected 'disabled')"
|
|
fi
|
|
if docker exec "$name" test -f /etc/fips/fips.nft && \
|
|
docker exec "$name" dpkg-query -W -f='${Conffiles}\n' fips 2>/dev/null \
|
|
| grep -q '/etc/fips/fips.nft'; then
|
|
pass "/etc/fips/fips.nft installed and registered as dpkg conffile"
|
|
else
|
|
fail "/etc/fips/fips.nft missing or not a registered conffile"
|
|
fi
|
|
if docker exec "$name" test -d /etc/fips/fips.d; then
|
|
fwd_mode=$(docker exec "$name" stat -c '%a %U:%G' /etc/fips/fips.d 2>/dev/null)
|
|
if [ "$fwd_mode" = "755 root:root" ]; then
|
|
pass "/etc/fips/fips.d/ drop-in dir present (755 root:root)"
|
|
else
|
|
fail "/etc/fips/fips.d/ wrong mode/owner: '$fwd_mode' (expected '755 root:root')"
|
|
fi
|
|
else
|
|
fail "/etc/fips/fips.d/ drop-in directory missing"
|
|
fi
|
|
if docker exec "$name" grep -qF 'include "/etc/fips/fips.d/*.nft"' /etc/fips/fips.nft; then
|
|
pass "fips.nft includes drop-in glob /etc/fips/fips.d/*.nft"
|
|
else
|
|
fail "fips.nft missing drop-in include for /etc/fips/fips.d/*.nft"
|
|
fi
|
|
|
|
# Start the services as a simulated boot. (On a real system,
|
|
# they'd come up on next reboot.)
|
|
start_unit "$name" fips.service || true
|
|
start_unit_queued "$name" fips-dns.service || true
|
|
|
|
if wait_for_service_active "$name" fips.service; then
|
|
pass "fips.service active after explicit start"
|
|
else
|
|
fail "fips.service did not become active in ${SERVICE_TIMEOUT}s"
|
|
echo " --- fips.service status ---"
|
|
docker exec "$name" systemctl status fips.service --no-pager 2>&1 | tail -20
|
|
echo " --- fips.service journal ---"
|
|
docker exec "$name" journalctl -u fips.service --no-pager 2>&1 | tail -20
|
|
cleanup_container "$name"
|
|
return
|
|
fi
|
|
|
|
# Wait for the DNS responder to bind on the daemon's [::1]:5354.
|
|
local ready=0
|
|
for _i in $(seq 1 "$DAEMON_TIMEOUT"); do
|
|
if docker exec "$name" ss -uln 2>/dev/null | grep -q ':5354'; then
|
|
ready=1
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ "$ready" = "1" ]; then
|
|
pass "fips DNS listener up on port 5354"
|
|
else
|
|
fail "fips DNS listener did not appear within ${DAEMON_TIMEOUT}s"
|
|
cleanup_container "$name"
|
|
return
|
|
fi
|
|
|
|
# Wait for fips-dns.service to finish. Its start job is queued rather than
|
|
# waited on above, so this is what makes /run/fips/dns-backend safe to read:
|
|
# fips-dns-setup waits up to 30s for fips0 and restarts systemd-resolved
|
|
# before it writes that file, so reading it on a timer races the setup.
|
|
# RemainAfterExit=yes makes is-active correct for this oneshot.
|
|
if wait_for_service_active "$name" fips-dns.service; then
|
|
pass "fips-dns.service completed"
|
|
else
|
|
fail "fips-dns.service did not complete in ${SERVICE_TIMEOUT}s"
|
|
echo " --- fips-dns.service journal ---"
|
|
docker exec "$name" journalctl -u fips-dns.service --no-pager 2>&1 | tail -20
|
|
cleanup_container "$name"
|
|
return
|
|
fi
|
|
|
|
local backend
|
|
backend=$(docker exec "$name" cat /run/fips/dns-backend 2>/dev/null || echo "(missing)")
|
|
local ver
|
|
ver=$(container_systemd_version "$name")
|
|
local expected_backend
|
|
if [ -n "$ver" ] && [ "$ver" -ge 258 ]; then
|
|
expected_backend="dns-delegate"
|
|
else
|
|
expected_backend="global-drop-in"
|
|
fi
|
|
if [ "$backend" = "$expected_backend" ]; then
|
|
pass "fips-dns.service picked $expected_backend backend (systemd $ver)"
|
|
else
|
|
fail "expected $expected_backend (systemd $ver), got: $backend"
|
|
echo " --- fips-dns.service journal ---"
|
|
docker exec "$name" journalctl -u fips-dns.service --no-pager 2>&1 | tail -20
|
|
fi
|
|
|
|
# Get the daemon's npub via fipsctl. Works for both ephemeral
|
|
# and persistent identity (no need to override /etc/fips/fips.yaml).
|
|
local npub
|
|
npub=$(docker exec "$name" fipsctl show status 2>/dev/null \
|
|
| grep -oE 'npub1[a-z0-9]+' | head -1)
|
|
if [ -z "$npub" ]; then
|
|
fail "could not read npub from fipsctl show status"
|
|
cleanup_container "$name"
|
|
return
|
|
fi
|
|
echo " daemon npub: $npub"
|
|
|
|
# The actual end-to-end test: a stock dpkg-installed deployment
|
|
# must successfully resolve a .fips query through the system
|
|
# resolver. This covers the full path: maintainer scripts ran,
|
|
# service started, DNS responder bound, resolver backend
|
|
# configured, query routed and answered.
|
|
sleep 1
|
|
local stub_output
|
|
stub_output=$(docker exec "$name" dig +tries=1 +time=3 @127.0.0.53 AAAA "${npub}.fips" 2>&1)
|
|
if echo "$stub_output" | grep -qE '^[a-zA-Z0-9].*\sAAAA\s+[0-9a-f:]+'; then
|
|
pass "end-to-end dig @127.0.0.53 returns AAAA on stock .deb install"
|
|
else
|
|
fail "end-to-end dig @127.0.0.53 did not return AAAA"
|
|
echo " --- dig output ---"
|
|
echo "$stub_output" | tail -15
|
|
echo " --- resolved status ---"
|
|
docker exec "$name" resolvectl status 2>&1 | tail -25 || true
|
|
echo " --- fips journal ---"
|
|
docker exec "$name" journalctl -u fips.service --no-pager 2>&1 | tail -10
|
|
fi
|
|
|
|
# Verify fips-gateway can run against the installed daemon. Tests
|
|
# the gateway/daemon default-pairing on a real .deb install (no
|
|
# custom config). Requires enabling the unit (it's not in the
|
|
# default preset) and ipv6 forwarding (gateway checks before
|
|
# the DNS upstream check), which the container is started with;
|
|
# see start_systemd_container_with_tun.
|
|
apply_gateway_config "$name"
|
|
|
|
sleep 3
|
|
if wait_for_service_active "$name" fips.service 5; then
|
|
:
|
|
else
|
|
fail "fips.service did not stay up after gateway-enable restart"
|
|
fi
|
|
|
|
# systemd removes and recreates RuntimeDirectory=fips across this restart
|
|
# as root:root 0750. The daemon must restore the fips group on every start,
|
|
# not only when it created the directory itself.
|
|
local runtime_access
|
|
runtime_access=$(docker exec "$name" stat -c '%a %U:%G' /run/fips 2>/dev/null || true)
|
|
if [ "$runtime_access" = "750 root:fips" ]; then
|
|
pass "/run/fips ownership restored after service restart"
|
|
else
|
|
fail "/run/fips wrong after service restart: '$runtime_access' (expected '750 root:fips')"
|
|
fi
|
|
|
|
if docker exec "$name" bash -c '
|
|
useradd --system --no-create-home --user-group --groups fips fips-test-client
|
|
runuser -u fips-test-client -- fipsctl show status >/dev/null
|
|
'; then
|
|
pass "non-root fips group member reaches control socket after restart"
|
|
else
|
|
fail "non-root fips group member cannot reach control socket after restart"
|
|
fi
|
|
|
|
start_unit "$name" fips-gateway.service "$GATEWAY_START_TIMEOUT" >/dev/null 2>&1 || true
|
|
sleep 3
|
|
if docker exec "$name" journalctl -u fips-gateway.service --no-pager 2>/dev/null \
|
|
| grep -q "DNS upstream is reachable"; then
|
|
pass "fips-gateway reaches DNS upstream at [::1]:5354 via .deb install"
|
|
else
|
|
fail "fips-gateway DNS upstream check failed against installed daemon"
|
|
echo " --- fips-gateway journal ---"
|
|
docker exec "$name" journalctl -u fips-gateway.service --no-pager 2>&1 | tail -15
|
|
fi
|
|
|
|
check_gateway_default_listener "$name" "$npub"
|
|
|
|
if check_remove_clears_dns "$name" "$expected_backend"; then
|
|
check_purge_clears_dns "$name" "$expected_backend"
|
|
fi
|
|
|
|
cleanup_container "$name"
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Upgrade scenario
|
|
#
|
|
# Upgrades an installed package to a newer one and checks what the
|
|
# maintainer scripts do to the running services on the way. The newer
|
|
# package is made from the one under test inside the container: unpacked,
|
|
# given a higher Version and repacked, so the upgrade runs this tree's prerm
|
|
# and postinst without a second build.
|
|
#
|
|
# Its runtime image holds no package. The install scenario's image does, but
|
|
# under a tag every run shares and a file name every build of one version
|
|
# shares, so another run could retag it in the minutes between the two
|
|
# scenarios and this one would upgrade from that run's package. Instead the
|
|
# package this run built is copied into each container, and its checksum is
|
|
# compared there before anything is installed.
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
# Every apt run here is bounded: an upgrade that blocks in postinst is one of
|
|
# the defects this scenario exists to catch, and an unbounded one would hang
|
|
# the suite instead of failing it.
|
|
# The package's own worst case on a healthy daemon is its three bounded starts,
|
|
# 60s + 60s + 90s, so an upgrade bound above that reports the package's
|
|
# diagnosis rather than this one.
|
|
UPGRADE_APT_TIMEOUT=300
|
|
# The dead-daemon reinstall skips the units behind the daemon, so its worst
|
|
# case is one 60s bound.
|
|
DEAD_DAEMON_APT_TIMEOUT=150
|
|
# postinst waits up to 60s for a unit that does not start. When the daemon
|
|
# cannot start, apt has to return a failure well inside this.
|
|
DEAD_DAEMON_LIMIT=120
|
|
# Bound on a single short command inside a container.
|
|
EXEC_TIMEOUT=60
|
|
|
|
# Run apt-get in /opt/fips-deb inside the container, bounded by the given
|
|
# number of seconds, keeping the existing configuration files. Sets APT_RC,
|
|
# APT_SECS and APT_OUT rather than returning a status, because every caller
|
|
# needs all three.
|
|
run_apt() {
|
|
local name="$1" limit="$2"
|
|
shift 2
|
|
local start=$SECONDS
|
|
APT_RC=0
|
|
APT_OUT=$(timeout "$limit" docker exec -w /opt/fips-deb "$name" \
|
|
apt-get -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold \
|
|
"$@" 2>&1) || APT_RC=$?
|
|
APT_SECS=$((SECONDS - start))
|
|
return 0
|
|
}
|
|
|
|
# Boot an upgrade container, copy this run's package into it and install it.
|
|
# Returns 1, having recorded why, when any step fails.
|
|
upgrade_boot() {
|
|
local name="$1" image="$2" deb="$3"
|
|
if ! start_systemd_container_with_tun "$name" "$image"; then
|
|
fail "$name: container did not start"
|
|
return 1
|
|
fi
|
|
if ! wait_for_systemd "$name"; then
|
|
fail "systemd did not boot in $name"
|
|
return 1
|
|
fi
|
|
# /opt rather than /tmp: systemd mounts a fresh /tmp during boot.
|
|
if ! timeout "$EXEC_TIMEOUT" docker cp "$DEB_PATH" "$name:/opt/fips-deb/$deb"; then
|
|
fail "$name: could not copy $deb into the container"
|
|
return 1
|
|
fi
|
|
local want have
|
|
want=$(sha256sum "$DEB_PATH" | cut -d' ' -f1)
|
|
have=$(cexec "$name" sha256sum "/opt/fips-deb/$deb" 2>/dev/null | cut -d' ' -f1)
|
|
if [ -z "$want" ] || [ "$want" != "$have" ]; then
|
|
fail "$name: the package in the container is not the one under test ('$have', want '$want')"
|
|
return 1
|
|
fi
|
|
local start=$SECONDS rc=0 out
|
|
out=$(timeout "$UPGRADE_APT_TIMEOUT" docker exec -w /opt/fips-deb "$name" bash -c "
|
|
apt-get update >/dev/null 2>&1
|
|
apt-get install -y --no-install-recommends ./${deb} 2>&1
|
|
") || rc=$?
|
|
echo " install took $((SECONDS - start))s"
|
|
if [ "$rc" -ne 0 ]; then
|
|
fail "$name: installing $deb exited $rc"
|
|
echo "$out" | tail -20
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Make /opt/fips-deb/next.deb from the package under test, inside the
|
|
# container so the host needs no dpkg tooling. Its Version is the original's
|
|
# with "+upgrade1" appended, which must compare higher, and its fips.nft gains
|
|
# a named counter inside the fips table, so a check can tell whether the
|
|
# ruleset loaded after the upgrade is the new one. Any step failing is a
|
|
# failure of the scenario, never a skip.
|
|
make_next_package() {
|
|
local name="$1" deb="$2" out rc=0
|
|
# shellcheck disable=SC2016 # the script expands inside the container
|
|
out=$(timeout "$EXEC_TIMEOUT" docker exec -w /opt/fips-deb -e DEB="$deb" "$name" \
|
|
bash -euo pipefail -c '
|
|
rm -rf /root/next
|
|
dpkg-deb -R "./$DEB" /root/next
|
|
old=$(dpkg-deb -f "./$DEB" Version)
|
|
new="${old}+upgrade1"
|
|
sed -i "s/^Version: .*/Version: ${new}/" /root/next/DEBIAN/control
|
|
dpkg --compare-versions "$new" gt "$old"
|
|
nft_file=/root/next/etc/fips/fips.nft
|
|
grep -q "^table inet fips {\$" "$nft_file"
|
|
sed -i "/^table inet fips {\$/a\\ counter fips_upgrade_probe { packets 0 bytes 0 }" "$nft_file"
|
|
grep -q "counter fips_upgrade_probe" "$nft_file"
|
|
nft -c -f "$nft_file"
|
|
if grep -q " etc/fips/fips.nft\$" /root/next/DEBIAN/md5sums 2>/dev/null; then
|
|
sum=$(md5sum "$nft_file" | cut -d" " -f1)
|
|
sed -i "s|^[0-9a-f]* etc/fips/fips.nft\$|${sum} etc/fips/fips.nft|" /root/next/DEBIAN/md5sums
|
|
grep -q "^${sum} etc/fips/fips.nft\$" /root/next/DEBIAN/md5sums
|
|
fi
|
|
dpkg-deb -b /root/next /opt/fips-deb/next.deb >/dev/null
|
|
echo "made next.deb at Version $new"
|
|
' 2>&1) || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
fail "$name: could not make the newer package (exit $rc)"
|
|
echo "$out" | tail -20
|
|
return 1
|
|
fi
|
|
echo " $out"
|
|
return 0
|
|
}
|
|
|
|
# Start fips.service and fips-dns.service the way the install scenario does
|
|
# and require both to be active.
|
|
start_daemon_units() {
|
|
local name="$1"
|
|
start_unit "$name" fips.service >/dev/null || true
|
|
start_unit_queued "$name" fips-dns.service >/dev/null || true
|
|
if wait_for_service_active "$name" fips.service &&
|
|
wait_for_service_active "$name" fips-dns.service; then
|
|
return 0
|
|
fi
|
|
cexec "$name" systemctl status --no-pager fips.service fips-dns.service 2>&1 | tail -20
|
|
return 1
|
|
}
|
|
|
|
# Pass or fail on whether a unit is active.
|
|
check_active() {
|
|
local name="$1" unit="$2" what="$3"
|
|
if cexec "$name" systemctl is-active --quiet "$unit"; then
|
|
pass "$what: $unit active"
|
|
else
|
|
fail "$what: $unit not active"
|
|
cexec "$name" systemctl status --no-pager "$unit" 2>&1 | tail -15
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Pass or fail on whether a unit the host never enabled is still neither
|
|
# running nor enabled.
|
|
check_left_off() {
|
|
local name="$1" unit="$2" what="$3" state
|
|
state=$(cexec "$name" systemctl is-enabled "$unit" 2>/dev/null || true)
|
|
if ! cexec "$name" systemctl is-active --quiet "$unit" && [ "$state" = "disabled" ]; then
|
|
pass "$what: $unit inactive and disabled"
|
|
else
|
|
fail "$what: $unit is $(cexec "$name" systemctl is-active "$unit" 2>/dev/null) and '$state' (want inactive and disabled)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Start `nft monitor tables` in the background, writing to
|
|
# /root/nft-monitor.log, and prove it is recording by adding and deleting a
|
|
# table of its own. An empty log from a monitor that never ran would otherwise
|
|
# read as a ruleset that was never removed. The probe table's name does not
|
|
# begin with "fips", so it cannot match a check on the fips table.
|
|
start_nft_monitor() {
|
|
local name="$1"
|
|
if ! timeout "$EXEC_TIMEOUT" docker exec -d "$name" \
|
|
sh -c 'exec nft monitor tables > /root/nft-monitor.log 2>&1'; then
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
cexec "$name" sh -c 'nft add table inet monprobe && nft delete table inet monprobe' || return 1
|
|
local _i
|
|
for _i in 1 2 3 4 5; do
|
|
if cexec "$name" grep -Eq '^delete table inet monprobe( |$)' /root/nft-monitor.log; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
cexec "$name" cat /root/nft-monitor.log 2>&1 | tail -10
|
|
return 1
|
|
}
|
|
|
|
# Apply the gateway config and start fips-gateway.service, bounded, requiring
|
|
# it to be active. Leaves it enabled or not as the caller already set it.
|
|
start_gateway() {
|
|
local name="$1"
|
|
apply_gateway_config "$name"
|
|
if ! wait_for_service_active "$name" fips.service 10; then
|
|
return 1
|
|
fi
|
|
start_unit "$name" fips-gateway.service "$GATEWAY_START_TIMEOUT" >/dev/null 2>&1 || true
|
|
if wait_for_service_active "$name" fips-gateway.service 10; then
|
|
return 0
|
|
fi
|
|
cexec "$name" systemctl status --no-pager fips-gateway.service 2>&1 | tail -15
|
|
return 1
|
|
}
|
|
|
|
# Print a unit's MainPID; 0 when it has no main process.
|
|
main_pid() {
|
|
local name="$1" unit="$2"
|
|
cexec "$name" systemctl show -p MainPID --value "$unit" 2>/dev/null || echo 0
|
|
return 0
|
|
}
|
|
|
|
# Pass or fail on whether a unit runs a new process of the installed binary
|
|
# after the upgrade: active, a MainPID other than the one before, and an
|
|
# executable that is the installed file rather than one the upgrade replaced,
|
|
# which the kernel reports with a " (deleted)" suffix.
|
|
check_new_binary() {
|
|
local name="$1" unit="$2" binary="$3" before="$4" what="$5" pid exe
|
|
pid=$(main_pid "$name" "$unit")
|
|
exe=$(cexec "$name" readlink "/proc/$pid/exe" 2>/dev/null || true)
|
|
if cexec "$name" systemctl is-active --quiet "$unit" && [ "$pid" != 0 ] &&
|
|
[ "$pid" != "$before" ] && [ "$exe" = "$binary" ]; then
|
|
pass "$what: $unit runs the upgraded $binary"
|
|
else
|
|
fail "$what: $unit is $(cexec "$name" systemctl is-active "$unit" 2>/dev/null), MainPID $before -> $pid, exe '$exe' (want active, a new process, $binary)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Host that opted in to the firewall and enabled the gateway: the upgrade must
|
|
# apply the new ruleset in place, with no moment at which the fips table is
|
|
# absent, and bring the gateway back on the new binary. Purging the package
|
|
# must then leave no enablement behind for the gateway.
|
|
_upgrade_opted_in() {
|
|
local name="$1" image="$2" deb="$3"
|
|
log "upgrade on a host that opted in ($name)"
|
|
upgrade_boot "$name" "$image" "$deb" || { cleanup_container "$name"; return 0; }
|
|
make_next_package "$name" "$deb" || { cleanup_container "$name"; return 0; }
|
|
if ! cexec "$name" systemctl enable --now fips-firewall.service >/dev/null 2>&1 ||
|
|
! start_daemon_units "$name"; then
|
|
fail "opted in: the firewall, fips and fips-dns did not all start before the upgrade"
|
|
cexec "$name" systemctl status --no-pager fips-firewall.service 2>&1 | tail -15
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
cexec "$name" systemctl enable fips-gateway.service >/dev/null 2>&1
|
|
if ! start_gateway "$name"; then
|
|
fail "opted in: fips-gateway did not start before the upgrade"
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
if ! start_nft_monitor "$name"; then
|
|
fail "opted in: nft monitor is not observing table changes"
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
local fips_pid gw_pid
|
|
fips_pid=$(main_pid "$name" fips.service)
|
|
gw_pid=$(main_pid "$name" fips-gateway.service)
|
|
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install -y ./next.deb
|
|
echo " upgrade took ${APT_SECS}s"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "opted in: upgrade exits 0"
|
|
else
|
|
fail "opted in: upgrade exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_active "$name" fips.service "opted in, after upgrade"
|
|
check_active "$name" fips-dns.service "opted in, after upgrade"
|
|
check_active "$name" fips-firewall.service "opted in, after upgrade"
|
|
if cexec "$name" nft list counter inet fips fips_upgrade_probe >/dev/null 2>&1; then
|
|
pass "opted in: the upgraded ruleset is loaded"
|
|
else
|
|
fail "opted in: the upgraded ruleset is not loaded (no fips_upgrade_probe counter)"
|
|
fi
|
|
if cexec "$name" grep -Eq '^delete table inet fips( |$)' /root/nft-monitor.log; then
|
|
fail "opted in: the fips table was deleted during the upgrade"
|
|
cexec "$name" cat /root/nft-monitor.log 2>&1 | tail -10
|
|
else
|
|
pass "opted in: the fips table was never deleted during the upgrade"
|
|
fi
|
|
check_new_binary "$name" fips.service /usr/bin/fips "$fips_pid" "opted in, after upgrade"
|
|
check_new_binary "$name" fips-gateway.service /usr/bin/fips-gateway "$gw_pid" "opted in, after upgrade"
|
|
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" purge -y fips
|
|
echo " purge took ${APT_SECS}s"
|
|
if [ "$APT_RC" -ne 0 ]; then
|
|
fail "opted in: purge exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
local link=/etc/systemd/system/multi-user.target.wants/fips-gateway.service state
|
|
state=$(cexec "$name" systemctl is-enabled fips-gateway.service 2>/dev/null || true)
|
|
if ! cexec "$name" test -e "$link" && ! cexec "$name" test -L "$link" &&
|
|
[ "$state" != "enabled" ]; then
|
|
pass "opted in, after purge: no fips-gateway enablement left behind"
|
|
else
|
|
fail "opted in, after purge: fips-gateway still enabled ('$state', $(cexec "$name" ls -l "$link" 2>&1))"
|
|
fi
|
|
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
# Host that never opted in to the firewall and ran the gateway without enabling
|
|
# it: the upgrade must leave neither running nor enabled. Then the package is
|
|
# reinstalled three times: with the daemon masked, when apt must succeed and
|
|
# start nothing; with an enabled gateway that cannot start, when apt must
|
|
# succeed, say so, and leave the daemon running; and with a daemon that cannot
|
|
# start, when apt must fail, promptly, naming the unit, rather than wait for
|
|
# ever on a unit that requires a daemon which never comes up.
|
|
_upgrade_not_opted_in() {
|
|
local name="$1" image="$2" deb="$3"
|
|
log "upgrade on a host that never opted in ($name)"
|
|
upgrade_boot "$name" "$image" "$deb" || { cleanup_container "$name"; return 0; }
|
|
make_next_package "$name" "$deb" || { cleanup_container "$name"; return 0; }
|
|
if ! start_daemon_units "$name" || ! start_gateway "$name"; then
|
|
fail "not opted in: fips, fips-dns and fips-gateway did not start before the upgrade"
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install -y ./next.deb
|
|
echo " upgrade took ${APT_SECS}s"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "not opted in: upgrade exits 0"
|
|
else
|
|
fail "not opted in: upgrade exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_active "$name" fips.service "not opted in, after upgrade"
|
|
check_active "$name" fips-dns.service "not opted in, after upgrade"
|
|
check_left_off "$name" fips-firewall.service "not opted in, after upgrade"
|
|
check_left_off "$name" fips-gateway.service "not opted in, after upgrade"
|
|
if cexec "$name" nft list table inet fips >/dev/null 2>&1; then
|
|
fail "not opted in, after upgrade: the fips firewall table is loaded"
|
|
else
|
|
pass "not opted in, after upgrade: no fips firewall table"
|
|
fi
|
|
|
|
# A host that masked the daemon on purpose: the upgrade must skip it with a
|
|
# message, not fail. The package before this change printed nothing for a
|
|
# masked unit, so the message is what tells the two apart.
|
|
cexec "$name" bash -c 'systemctl stop fips-dns.service fips.service; systemctl mask fips.service' \
|
|
>/dev/null 2>&1
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install --reinstall -y ./next.deb
|
|
echo " reinstall with the daemon masked took ${APT_SECS}s (exit $APT_RC)"
|
|
if [ "$APT_RC" -eq 0 ] && grep -q "fips.service is masked" <<<"$APT_OUT" &&
|
|
! cexec "$name" systemctl is-active --quiet fips.service &&
|
|
! cexec "$name" systemctl is-active --quiet fips-dns.service; then
|
|
pass "masked daemon: apt succeeds, says the unit was skipped and starts nothing"
|
|
else
|
|
fail "masked daemon: apt exited $APT_RC (want 0, a message that fips.service is masked, and fips and fips-dns inactive)"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
cexec "$name" bash -c 'systemctl unmask fips.service; systemctl daemon-reload' >/dev/null 2>&1
|
|
if ! start_daemon_units "$name"; then
|
|
fail "dead daemon: fips and fips-dns did not start again after unmasking"
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
|
|
# An enabled gateway that fails on start, behind a daemon that is healthy:
|
|
# the gateway is an opt-in addition, so apt must report it and succeed.
|
|
# Restart=no so it reaches failed at once rather than looping.
|
|
cexec "$name" bash -c '
|
|
mkdir -p /etc/systemd/system/fips-gateway.service.d
|
|
printf "[Service]\nExecStart=\nExecStart=/bin/false\nRestart=no\n" \
|
|
> /etc/systemd/system/fips-gateway.service.d/broken.conf
|
|
systemctl daemon-reload
|
|
systemctl enable fips-gateway.service
|
|
' >/dev/null 2>&1
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install --reinstall -y ./next.deb
|
|
echo " reinstall with a broken gateway took ${APT_SECS}s (exit $APT_RC)"
|
|
if [ "$APT_RC" -eq 0 ] &&
|
|
grep -q "fips-gateway.service did not come back" <<<"$APT_OUT" &&
|
|
cexec "$name" systemctl is-active --quiet fips.service; then
|
|
pass "broken gateway: apt succeeds, reports the gateway and leaves the daemon running"
|
|
else
|
|
fail "broken gateway: apt exited $APT_RC (want 0, a message that fips-gateway.service did not come back, and fips active)"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
cexec "$name" bash -c '
|
|
systemctl disable fips-gateway.service
|
|
rm -rf /etc/systemd/system/fips-gateway.service.d
|
|
systemctl daemon-reload
|
|
systemctl reset-failed fips-gateway.service
|
|
' >/dev/null 2>&1
|
|
|
|
# A daemon that fails on every start: exit 1, so Restart=on-failure loops,
|
|
# and the start job of fips-dns, which requires it, is never dispatched.
|
|
cexec "$name" bash -c '
|
|
mkdir -p /etc/systemd/system/fips.service.d
|
|
printf "[Service]\nExecStart=\nExecStart=/bin/false\n" \
|
|
> /etc/systemd/system/fips.service.d/broken.conf
|
|
systemctl daemon-reload
|
|
'
|
|
run_apt "$name" "$DEAD_DAEMON_APT_TIMEOUT" install --reinstall -y ./next.deb
|
|
echo " reinstall with a dead daemon took ${APT_SECS}s (exit $APT_RC)"
|
|
if [ "$APT_RC" -ne 0 ] && [ "$APT_RC" -ne 124 ] &&
|
|
[ "$APT_SECS" -lt "$DEAD_DAEMON_LIMIT" ] &&
|
|
grep -q "fips.service did not become active" <<<"$APT_OUT"; then
|
|
pass "dead daemon: apt fails in ${APT_SECS}s and names fips.service"
|
|
else
|
|
fail "dead daemon: apt exited $APT_RC after ${APT_SECS}s (want a failure under ${DEAD_DAEMON_LIMIT}s naming fips.service; 124 is the harness bound)"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
# Print the unit's enablement state, or "(unreadable)" when systemctl could
|
|
# not be asked.
|
|
unit_enabled_state() {
|
|
local name="$1" unit="$2" state
|
|
state=$(cexec "$name" systemctl is-enabled "$unit" 2>/dev/null) || true
|
|
echo "${state:-(unreadable)}"
|
|
return 0
|
|
}
|
|
|
|
# Pass or fail on a unit's enablement state.
|
|
check_enabled_state() {
|
|
local name="$1" unit="$2" want="$3" what="$4" have
|
|
have=$(unit_enabled_state "$name" "$unit")
|
|
if [ "$have" = "$want" ]; then
|
|
pass "$what: $unit is $want"
|
|
else
|
|
fail "$what: $unit is '$have' (want $want)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Pass or fail on dpkg's status for the fips package.
|
|
check_pkg_status() {
|
|
local name="$1" want="$2" what="$3" have
|
|
have=$(cexec "$name" dpkg-query -W -f='${db:Status-Status}' fips 2>/dev/null) || true
|
|
if [ "$have" = "$want" ]; then
|
|
pass "$what: package is $want"
|
|
else
|
|
fail "$what: package is '${have:-(unreadable)}' (want $want)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# An operator who disabled fips-dns must find it still disabled after the
|
|
# package is configured again. Then, on a host whose daemon config renames the
|
|
# TUN (tun.name: mesh0), so fips0 never exists, an upgrade must configure
|
|
# cleanly with fips-dns running, because fips-dns-setup takes the name from the
|
|
# config and the resolved backends never wait for the interface. The disable
|
|
# check runs first, on the default name, so that it does not depend on the
|
|
# upgrade's outcome.
|
|
_upgrade_renamed_tun() {
|
|
local name="$1" image="$2" deb="$3"
|
|
log "configure with fips-dns disabled, then upgrade with tun.name mesh0 ($name)"
|
|
upgrade_boot "$name" "$image" "$deb" || { cleanup_container "$name"; return 0; }
|
|
make_next_package "$name" "$deb" || { cleanup_container "$name"; return 0; }
|
|
|
|
cexec "$name" systemctl disable fips-dns.service >/dev/null 2>&1 || true
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install --reinstall -y "./$deb"
|
|
echo " reinstall with fips-dns disabled took ${APT_SECS}s (exit $APT_RC)"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "fips-dns disabled: reinstall exits 0"
|
|
else
|
|
fail "fips-dns disabled: reinstall exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_enabled_state "$name" fips-dns.service disabled "fips-dns disabled, after reinstall"
|
|
|
|
cexec "$name" systemctl enable fips-dns.service >/dev/null 2>&1 || true
|
|
if ! cexec "$name" sed -i 's/^ name: fips0$/ name: mesh0/' /etc/fips/fips.yaml ||
|
|
! cexec "$name" grep -q '^ name: mesh0$' /etc/fips/fips.yaml; then
|
|
fail "renamed tun: could not set tun.name to mesh0 in /etc/fips/fips.yaml"
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
cexec "$name" systemctl restart fips.service >/dev/null 2>&1 || true
|
|
# fips.service reads active as soon as it forks, before the daemon has
|
|
# created its TUN, so wait for the interface itself.
|
|
local _i up=0
|
|
for _i in $(seq 1 "$DAEMON_TIMEOUT"); do
|
|
if cexec "$name" ip link show mesh0 >/dev/null 2>&1; then
|
|
up=1
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if ! wait_for_service_active "$name" fips.service || [ "$up" -ne 1 ]; then
|
|
fail "renamed tun: the daemon did not come up on mesh0 before the upgrade"
|
|
cexec "$name" systemctl status --no-pager fips.service 2>&1 | tail -15
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install -y ./next.deb
|
|
echo " upgrade took ${APT_SECS}s"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "renamed tun: upgrade exits 0"
|
|
else
|
|
fail "renamed tun: upgrade exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_pkg_status "$name" installed "renamed tun, after upgrade"
|
|
check_active "$name" fips.service "renamed tun, after upgrade"
|
|
check_active "$name" fips-dns.service "renamed tun, after upgrade"
|
|
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
# Removing the package (not purging it) disables its units, and installing it
|
|
# again from that config-files state must put the operator's enablement back:
|
|
# fips-dns enabled if it was enabled at the remove, disabled if the operator
|
|
# had disabled it. Once restored, a later configure must again respect a
|
|
# disable, so nothing left by the remove outlives the reinstall. A remove by
|
|
# an older package, which recorded nothing, is treated as enabled at remove,
|
|
# as that package's postinst always enabled fips-dns.
|
|
_reinstall_after_remove() {
|
|
local name="$1" image="$2" deb="$3"
|
|
log "remove and reinstall ($name)"
|
|
upgrade_boot "$name" "$image" "$deb" || { cleanup_container "$name"; return 0; }
|
|
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" remove -y fips
|
|
if [ "$APT_RC" -ne 0 ]; then
|
|
fail "reinstall after remove: removing the package exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
check_pkg_status "$name" config-files "enabled at remove, after remove"
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install -y "./$deb"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "enabled at remove: reinstall exits 0"
|
|
else
|
|
fail "enabled at remove: reinstall exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_enabled_state "$name" fips.service enabled "enabled at remove, after reinstall"
|
|
check_enabled_state "$name" fips-dns.service enabled "enabled at remove, after reinstall"
|
|
|
|
cexec "$name" systemctl disable fips-dns.service >/dev/null 2>&1 || true
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install --reinstall -y "./$deb"
|
|
echo " configure again with fips-dns disabled (exit $APT_RC)"
|
|
check_enabled_state "$name" fips-dns.service disabled "disabled after the reinstall, configured again"
|
|
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" remove -y fips
|
|
if [ "$APT_RC" -ne 0 ]; then
|
|
fail "disabled at remove: removing the package exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install -y "./$deb"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "disabled at remove: reinstall exits 0"
|
|
else
|
|
fail "disabled at remove: reinstall exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_enabled_state "$name" fips.service enabled "disabled at remove, after reinstall"
|
|
check_enabled_state "$name" fips-dns.service disabled "disabled at remove, after reinstall"
|
|
|
|
# A package from before the remove kept any record disabled fips-dns
|
|
# and left nothing under /var/lib/fips, and its postinst enabled
|
|
# fips-dns on every configure. Deleting what this package's remove
|
|
# left reproduces that state, and the install must still enable it.
|
|
cexec "$name" systemctl enable fips-dns.service >/dev/null 2>&1 || true
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" remove -y fips
|
|
if [ "$APT_RC" -ne 0 ]; then
|
|
fail "nothing recorded at remove: removing the package exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
cleanup_container "$name"
|
|
return 0
|
|
fi
|
|
# The unit file is gone with the package, so look for the link itself.
|
|
local link
|
|
link=$(cexec "$name" sh -c 'if [ -L /etc/systemd/system/multi-user.target.wants/fips-dns.service ]; then echo linked; else echo unlinked; fi' 2>/dev/null) || true
|
|
if [ "$link" = unlinked ]; then
|
|
pass "nothing recorded at remove: the remove disabled fips-dns.service"
|
|
else
|
|
fail "nothing recorded at remove: fips-dns.service link after the remove is '${link:-(unreadable)}' (want unlinked)"
|
|
fi
|
|
if cexec "$name" sh -c 'ls /var/lib/fips/fips-dns.* >/dev/null 2>&1 && rm -f /var/lib/fips/fips-dns.* && ! ls /var/lib/fips/fips-dns.* >/dev/null 2>&1'; then
|
|
pass "nothing recorded at remove: deleted the record the remove left"
|
|
else
|
|
fail "nothing recorded at remove: the remove left no record to delete, or it could not be deleted"
|
|
fi
|
|
run_apt "$name" "$UPGRADE_APT_TIMEOUT" install -y "./$deb"
|
|
if [ "$APT_RC" -eq 0 ]; then
|
|
pass "nothing recorded at remove: reinstall exits 0"
|
|
else
|
|
fail "nothing recorded at remove: reinstall exited $APT_RC"
|
|
echo "$APT_OUT" | tail -20
|
|
fi
|
|
check_enabled_state "$name" fips.service enabled "nothing recorded at remove, after reinstall"
|
|
check_enabled_state "$name" fips-dns.service enabled "nothing recorded at remove, after reinstall"
|
|
|
|
cleanup_container "$name"
|
|
return 0
|
|
}
|
|
|
|
_run_deb_upgrade_scenario() {
|
|
local distro_label="$1"
|
|
local base_image="$2"
|
|
# Scoped to the run like the container names, although it holds no
|
|
# package, so concurrent runs never rebuild an image under each other.
|
|
local image="fips-deb-upgrade:${distro_label}${FIPS_CI_NAME_SUFFIX:-}"
|
|
log ".deb upgrade: ${base_image}"
|
|
|
|
if [ -z "$DEB_PATH" ] || [ ! -f "$DEB_PATH" ]; then
|
|
fail "no package to upgrade from"
|
|
return
|
|
fi
|
|
local deb
|
|
deb=$(basename "$DEB_PATH")
|
|
|
|
log "Building $image (runtime packages and nftables, no fips package)"
|
|
build_image "$image" "$(cat <<DOCKERFILE
|
|
FROM ${base_image}
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
$(runtime_packages "$base_image") nftables && \\
|
|
apt-get clean && rm -rf /var/lib/apt/lists/* && \\
|
|
systemctl enable systemd-resolved && \\
|
|
mkdir -p /opt/fips-deb
|
|
CMD ["/lib/systemd/systemd"]
|
|
DOCKERFILE
|
|
)" "${CI_LABELS[@]}" || {
|
|
fail "upgrade image build failed"
|
|
return
|
|
}
|
|
|
|
# Named under the install scenario's prefix, so a CI step that collects
|
|
# that scenario's container logs on failure collects these too.
|
|
_upgrade_opted_in "fips-deb-test-${distro_label}-upg-a${FIPS_CI_NAME_SUFFIX:-}" "$image" "$deb"
|
|
_upgrade_not_opted_in "fips-deb-test-${distro_label}-upg-b${FIPS_CI_NAME_SUFFIX:-}" "$image" "$deb"
|
|
_upgrade_renamed_tun "fips-deb-test-${distro_label}-upg-c${FIPS_CI_NAME_SUFFIX:-}" "$image" "$deb"
|
|
_reinstall_after_remove "fips-deb-test-${distro_label}-upg-d${FIPS_CI_NAME_SUFFIX:-}" "$image" "$deb"
|
|
remove_image "$image"
|
|
return 0
|
|
}
|
|
|
|
# Per-distro wrappers
|
|
# debian12 also runs the upgrade scenario. One distro keeps the suite's cost
|
|
# down; this one because a oneshot start behind a daemon in its restart loop
|
|
# waits for ever on its systemd (252), while on Ubuntu 22.04's (249) the start
|
|
# returns with an error, so only here does the upgrade scenario see the hang.
|
|
test_debian12() {
|
|
_run_deb_install_scenario debian12 debian:12
|
|
_run_deb_upgrade_scenario debian12 debian:12
|
|
}
|
|
test_debian13() { _run_deb_install_scenario debian13 debian:trixie; }
|
|
test_ubuntu22() { _run_deb_install_scenario ubuntu22 ubuntu:22.04; }
|
|
test_ubuntu24() { _run_deb_install_scenario ubuntu24 ubuntu:24.04; }
|
|
test_ubuntu26() { _run_deb_install_scenario ubuntu26 ubuntu:26.04; }
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Main
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
ALL_SCENARIOS="debian12 debian13 ubuntu22 ubuntu24 ubuntu26"
|
|
|
|
# `--deb PATH` installs a package the caller already built, which is how one
|
|
# build serves all five distributions and how GitHub CI and a local run come to
|
|
# do the same work: both build once through the container script and hand the
|
|
# result here. Keep ALL_SCENARIOS above on its own line at column zero;
|
|
# check-ci-parity.sh reads it to compare this matrix against the GitHub one.
|
|
_args=()
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--deb)
|
|
SUPPLIED_DEB="${2:?--deb requires a path}"
|
|
shift 2
|
|
;;
|
|
-h|--help)
|
|
echo "usage: test.sh [--deb PATH] [scenario ...]"
|
|
echo "scenarios: $ALL_SCENARIOS"
|
|
exit 0
|
|
;;
|
|
-*)
|
|
echo "Unknown option: $1" >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
_args+=("$1")
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
set -- ${_args[@]+"${_args[@]}"}
|
|
|
|
if [ $# -eq 0 ]; then
|
|
scenarios="$ALL_SCENARIOS"
|
|
else
|
|
scenarios="$*"
|
|
fi
|
|
|
|
for scenario in $scenarios; do
|
|
case "$scenario" in
|
|
debian12) test_debian12 ;;
|
|
debian13) test_debian13 ;;
|
|
ubuntu22) test_ubuntu22 ;;
|
|
ubuntu24) test_ubuntu24 ;;
|
|
ubuntu26) test_ubuntu26 ;;
|
|
*)
|
|
echo "Unknown scenario: $scenario"
|
|
echo "Available: $ALL_SCENARIOS"
|
|
exit 1
|
|
;;
|
|
esac
|
|
echo
|
|
done
|
|
|
|
echo "═══════════════════════════════════════"
|
|
echo "Results: $PASS passed, $FAIL failed, $SKIP skipped"
|
|
echo "═══════════════════════════════════════"
|
|
|
|
# A skip must not read as a pass. Nothing calls skip() today, so SKIP is
|
|
# always 0 and this changes no current outcome -- which is exactly why it is
|
|
# worth adding now: the helper and the counter already existed and were
|
|
# reported, so a later skip path would have printed "N skipped" next to a
|
|
# zero exit and looked like coverage. Gate on it before that happens.
|
|
if [ "$SKIP" -ne 0 ]; then
|
|
echo "FAIL: $SKIP check(s) skipped; a skipped check is not a passed one." >&2
|
|
echo " Either make the check run here, or remove it and record the" >&2
|
|
echo " gap deliberately rather than skipping it at runtime." >&2
|
|
fi
|
|
|
|
[ "$FAIL" -eq 0 ] && [ "$SKIP" -eq 0 ]
|