mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-22 07:48:26 +00:00
e4a854f6b0
Two runs on one host destroyed each other's containers, producing mid-test "No such container" failures that look like real defects. The automated builder runs a full local CI on the same box every few minutes, so the machine is contended almost always and this has red-ed both a hand run and an automated gate. Two independent causes are fixed here. Container names were hardcoded, and docker names are global rather than scoped by compose project, so two runs collided on the same name; every name now takes an optional suffix that the harness sets from the run id. And the cleanup sweep matched a label shared by every run, so one run's teardown force-removed another's containers; resources now also carry a per-run label and the sweep can be narrowed to it. A third hazard turned up that was not in the original report: the sidecar suite passes explicit compose project names, which override the run-scoped project and put it outside the shared prefix entirely. Its project names, network, and derived container references are now scoped too. Both are default-off. With the suffix unset, names render exactly as they do today and a bare compose invocation is unchanged, which is what keeps the hosted CI and the documentation correct. A cleanup run with no run id still reaps everything, which is what a manual "clear the box" wants. The literal-name sweep was not sufficient: six scripts build container names dynamically from node labels, and two suites create their own containers outside compose. Those are handled at their construction sites. Verified: syntax check on all modified scripts; compose validation on every modified file with the suffix both set and unset; and a synthetic two-run reproduction that shows the old cleanup destroying a bystander run and the new one leaving it alone. Known gap: subnets are still hardcoded, so two concurrent full runs will still collide on address-pool overlap. That fix reaches into topology configs, chaos scenarios, diagrams and production source, so it is left for its own change rather than half-done here.
125 lines
5.0 KiB
Bash
Executable File
125 lines
5.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# Reap FIPS CI docker resources: containers, networks, volumes, images.
|
|
#
|
|
# Force-removes everything created by ci-local.sh that is still around —
|
|
# whether a run finished cleanly, was preempted (SIGTERM/SIGKILL), OOM-killed,
|
|
# or crashed. Two complementary selectors make this robust no matter how a
|
|
# prior run died:
|
|
#
|
|
# 1. The CI label com.corganlabs.fips-ci=1 (attached to every direct
|
|
# `docker run`/network/volume ci-local drives). Every run additionally
|
|
# stamps com.corganlabs.fips-ci.run=<run-id> on the same resources.
|
|
# 2. The compose project-name prefix fipsci_ (every compose project ci-local
|
|
# starts is named fipsci_<run-id>_<suite>, so its containers/networks/
|
|
# volumes all carry com.docker.compose.project=fipsci_... and are named
|
|
# with that prefix).
|
|
#
|
|
# The generic CI label is shared by every run on the host, so an unscoped label
|
|
# sweep would tear down a CONCURRENT run's resources. Pass --run-id to narrow
|
|
# the label sweep to one run; a run's own teardown always does. Without it the
|
|
# label sweep stays broad, which is what a manual "reap everything" wants.
|
|
#
|
|
# Usage:
|
|
# ci-cleanup.sh Reap ALL fips-ci resources (any run)
|
|
# ci-cleanup.sh --project-prefix P Restrict the compose-project sweep to
|
|
# names starting with P (scopes the reap
|
|
# to a single run)
|
|
# ci-cleanup.sh --run-id ID Restrict the label sweep to the run
|
|
# labelled ID (leaves other runs alone)
|
|
# ci-cleanup.sh --label L Override the CI label (default above)
|
|
# ci-cleanup.sh --images "a b,c" Also `docker rmi -f` these image tags
|
|
# (space- or comma-separated)
|
|
#
|
|
# Safe to run when there is nothing to reap, and safe to run repeatedly.
|
|
# Also reachable as `ci-local.sh --reap`.
|
|
set -uo pipefail
|
|
|
|
LABEL="com.corganlabs.fips-ci=1"
|
|
RUN_LABEL_KEY="com.corganlabs.fips-ci.run"
|
|
PROJECT_PREFIX="fipsci_" # broad default: every CI run
|
|
RUN_ID="" # broad default: every CI run
|
|
IMAGES=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--label) LABEL="$2"; shift 2 ;;
|
|
--project-prefix) PROJECT_PREFIX="$2"; shift 2 ;;
|
|
--run-id) RUN_ID="$2"; shift 2 ;;
|
|
--images) IMAGES="$2"; shift 2 ;;
|
|
-h|--help) sed -n '2,/^set /{ /^set /d; s/^# \?//; p }' "$0"; exit 0 ;;
|
|
*) echo "Unknown option: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
# No docker, nothing to reap.
|
|
exit 0
|
|
fi
|
|
if ! docker info >/dev/null 2>&1; then
|
|
# Daemon unreachable; treat as nothing to reap rather than wedging a caller.
|
|
exit 0
|
|
fi
|
|
|
|
# Each docker mutation is wrapped in `timeout` so a stuck daemon/resource can
|
|
# never wedge a caller (ci-local's signal trap relies on this being bounded).
|
|
TMO=30
|
|
|
|
# Selector for the label sweep. With --run-id it matches only the named run, so
|
|
# a concurrent run's resources are left alone; without it, every CI run.
|
|
if [[ -n "$RUN_ID" ]]; then
|
|
SWEEP_LABEL="${RUN_LABEL_KEY}=${RUN_ID}"
|
|
else
|
|
SWEEP_LABEL="$LABEL"
|
|
fi
|
|
|
|
# Distinct compose project names (read off container labels) that start with
|
|
# the configured prefix.
|
|
ci_projects() {
|
|
docker ps -a --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \
|
|
| grep -E "^${PROJECT_PREFIX}" | sort -u
|
|
}
|
|
|
|
reap_containers() {
|
|
# By CI label.
|
|
docker ps -aq --filter "label=${SWEEP_LABEL}" 2>/dev/null \
|
|
| xargs -r timeout "$TMO" docker rm -f >/dev/null 2>&1 || true
|
|
# By compose project (carried even when container_name is explicit).
|
|
local p
|
|
for p in $(ci_projects); do
|
|
docker ps -aq --filter "label=com.docker.compose.project=${p}" 2>/dev/null \
|
|
| xargs -r timeout "$TMO" docker rm -f >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
reap_networks() {
|
|
docker network ls -q --filter "label=${SWEEP_LABEL}" 2>/dev/null \
|
|
| xargs -r timeout "$TMO" docker network rm >/dev/null 2>&1 || true
|
|
# Compose networks are named <project>_<net> → match by name prefix so
|
|
# orphaned networks (whose containers are already gone) are still caught.
|
|
docker network ls --format '{{.Name}}' 2>/dev/null | grep -E "^${PROJECT_PREFIX}" \
|
|
| xargs -r timeout "$TMO" docker network rm >/dev/null 2>&1 || true
|
|
}
|
|
|
|
reap_volumes() {
|
|
docker volume ls -q --filter "label=${SWEEP_LABEL}" 2>/dev/null \
|
|
| xargs -r timeout "$TMO" docker volume rm >/dev/null 2>&1 || true
|
|
docker volume ls --format '{{.Name}}' 2>/dev/null | grep -E "^${PROJECT_PREFIX}" \
|
|
| xargs -r timeout "$TMO" docker volume rm >/dev/null 2>&1 || true
|
|
}
|
|
|
|
reap_images() {
|
|
[[ -z "$IMAGES" ]] && return 0
|
|
local imgs
|
|
read -ra imgs <<< "${IMAGES//,/ }"
|
|
[[ ${#imgs[@]} -eq 0 ]] && return 0
|
|
timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Order matters: containers reference networks/volumes, so drop them first.
|
|
reap_containers
|
|
reap_networks
|
|
reap_volumes
|
|
reap_images
|
|
|
|
exit 0
|