testing: convert remaining fixed-sleep settles to progress-aware polling

Replace the fixed post-first-rekey and post-second-rekey settle sleeps in
the rekey integration test, and the fixed-deadline baseline wait in the
interop test, with the deterministic wait_until_connected progress-aware
polling helper already used for the initial baseline convergence.

Each converted site fails fast with structured diagnostics when the mesh
is stuck and extends its deadline only while pairwise reachability is
still climbing, removing the wall-clock settle windows that produced
intermittent connectivity failures after a rekey.

testing/interop/interop-test.sh

@@ -162,6 +162,13 @@ REKEY_SETTLE=12                 # FSP-cutover settle budget (Phase 6)
 # strict assertion sweep runs. A genuinely stuck pair still fails — the
 # poll times out and the recording sweep captures it.
 POST_REKEY_TIMEOUT=45
+# Progress-aware stall budget for the convergence detector
+# (wait_for_full_baseline → wait_until_connected). If no additional pair
+# becomes reachable for this long while more than the near-converged
+# slack of pairs is still down, the detector gives up early instead of
+# burning the whole convergence/post-rekey window; any progress resets
+# the clock, so a slow-but-converging mesh under netem keeps polling.
+RECONVERGE_STALL=15
 LOG_POLL_INTERVAL=2
 
 # Data-plane continuity stream (control-differential). Streams run a
@@ -434,25 +441,27 @@ ping_all_pairs() {
     done
 }
 
+# Convergence detector probe: one full all-pairs ping sweep in the
+# "convergence" context (which ping_all_pairs deliberately does NOT
+# record as a failure), setting PASSED/FAILED for wait_until_connected.
+_baseline_probe() {
+    ping_all_pairs quiet 1 "convergence"
+}
+
 # Poll until every directed pair pings clean, or until timeout. This is a
 # convergence DETECTOR — used at establishment (Phase 1) and after each
 # rekey (Phases 3/5). It pings with the "convergence" context, which
 # ping_all_pairs deliberately does not record as a failure; the caller
 # runs a separate strict assertion sweep afterwards.
+#
+# Delegates to the shared progress-aware wait_until_connected so the
+# deadline extends while more pairs are still coming up and gives up fast
+# on a genuine stall, instead of the prior fixed-deadline poll that could
+# false-time-out under heavy CI contention even while still converging.
+# Returns 0 once every pair is reachable, 1 on stall/timeout.
 wait_for_full_baseline() {
     local timeout="$1"
-    local start=$SECONDS
-    local best_passed=0 best_failed="$NUM_DIRECTED"
-    while (( SECONDS - start < timeout )); do
-        ping_all_pairs quiet 1 "convergence"
-        if [ "$PASSED" -gt "$best_passed" ]; then
-            best_passed="$PASSED"; best_failed="$FAILED"
-        fi
-        [ "$FAILED" -eq 0 ] && return 0
-        sleep 1
-    done
-    PASSED="$best_passed"; FAILED="$best_failed"
-    return 1
+    wait_until_connected _baseline_probe "$timeout" "$RECONVERGE_STALL"
 }
 
 phase_result() {
@@ -759,8 +768,15 @@ echo ""
 
 # ── Phase 4: second rekey cycle ──────────────────────────────────────
 
-echo "Phase 4: Second rekey cycle (waiting ${SECOND_REKEY_WAIT}s)"
-sleep "$SECOND_REKEY_WAIT"
+echo "Phase 4: Second rekey cycle (waiting up to ${SECOND_REKEY_WAIT}s for the next cutover)"
+# Poll for the next FMP cutover beyond what Phases 2/3 already saw, using
+# the same pre/post cutover-count delta convention as the control window
+# (Phase 1b), instead of a blind sleep. Bounded by SECOND_REKEY_WAIT so a
+# stalled rekey falls through to the strict Phase 5/6 assertions.
+fmp_cutovers_before="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
+wait_for_log_pattern_count \
+    "Rekey cutover complete \(initiator\), K-bit flipped" \
+    "$((fmp_cutovers_before + 1))" "$SECOND_REKEY_WAIT" || true
 echo ""
 
 echo "Phase 5: Post-second-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)"

testing/static/scripts/rekey-test.sh

@@ -158,9 +158,20 @@ REKEY_SETTLE=12        # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off
 # fully converged. Keep this bounded to preserve a meaningful scheduling check
 # while still allowing for log visibility at the timeout edge.
 FIRST_REKEY_TIMEOUT=$((REKEY_AFTER_SECS + 15))
-SECOND_REKEY_WAIT=40   # wait for second cycle
+SECOND_REKEY_WAIT=40   # upper bound for observing the second rekey cutover
 LOG_EVENT_POLL_INTERVAL=1
 
+# Post-rekey data-plane reconvergence is polled, not fixed-slept.
+# wait_until_connected drives the same all-pairs ping sweep the strict
+# per-phase assert depends on: it fails fast when reachability stops
+# improving (no new reachable pair for RECONVERGE_STALL seconds while
+# more than the near-converged slack of pairs is still down) and extends
+# its deadline up to RECONVERGE_TIMEOUT only while pairs keep coming up,
+# so a slow-but-converging post-rekey mesh under CI load passes instead
+# of false-failing on a fixed settle window.
+RECONVERGE_TIMEOUT=45
+RECONVERGE_STALL=10
+
 TIMEOUT=5
 CONVERGENCE_PING_TIMEOUT=1
 # Strict-ping retry policy for the per-phase ping_all asserts. Under 1%
@@ -392,20 +403,38 @@ assert_min_count "Rekey cutover complete (initiator), K-bit flipped" 1 \
 phase_result "FMP rekey events"
 echo ""
 
-# Verify connectivity after first rekey (strict — no failures allowed)
-echo "Phase 3: Post-rekey connectivity (settling ${REKEY_SETTLE}s)"
-sleep "$REKEY_SETTLE"
+# Verify connectivity after first rekey (strict — no failures allowed).
+# Wait for the data plane to reconverge with a progress-aware deadline
+# (the same all-pairs ping sweep the strict assert below depends on)
+# instead of a fixed settle: fail fast on a genuinely stuck pair, extend
+# only while reachability is still climbing. The strict ping_all is the
+# actual assertion, run only after the reconverge poll.
+echo "Phase 3: Post-rekey connectivity (reconverging up to ${RECONVERGE_TIMEOUT}s)"
+wait_until_connected _baseline_ping "$RECONVERGE_TIMEOUT" "$RECONVERGE_STALL" || true
 ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS"
 phase_result "Post-first-rekey (all 20 pairs)"
 echo ""
 
 # ── Phase 4: Wait for second rekey cycle ──────────────────────────────
-echo "Phase 4: Second rekey cycle (waiting ${SECOND_REKEY_WAIT}s)"
-sleep "$SECOND_REKEY_WAIT"
+# Poll for the next FMP rekey cutover instead of blind-sleeping: capture
+# the cutover count reached so far, then wait until at least one more
+# cutover lands — that increment is the second rekey cycle firing.
+# Bounded by SECOND_REKEY_WAIT so a stalled rekey still falls through to
+# the strict Phase 5/6 assertions rather than hanging.
+echo "Phase 4: Second rekey cycle (waiting up to ${SECOND_REKEY_WAIT}s for the next cutover)"
+fmp_cutovers_before=$(count_log_pattern "Rekey cutover complete (initiator), K-bit flipped")
+wait_for_log_pattern_count \
+    "Rekey cutover complete (initiator), K-bit flipped" \
+    "$((fmp_cutovers_before + 1))" "$SECOND_REKEY_WAIT" || true
 
-# Verify connectivity after second rekey (back-to-back)
-echo "Phase 5: Post-second-rekey connectivity (settling ${REKEY_SETTLE}s)"
-sleep "$REKEY_SETTLE"
+# Verify connectivity after second rekey (back-to-back). This is the
+# site of the recurring post-second-rekey straggler-pair flake: wait for
+# the data plane to reconverge with a progress-aware deadline (the same
+# all-pairs ping sweep the strict assert below depends on) rather than a
+# fixed settle window — fail fast if reachability stops improving, extend
+# only while pairs are still coming up.
+echo "Phase 5: Post-second-rekey connectivity (reconverging up to ${RECONVERGE_TIMEOUT}s)"
+wait_until_connected _baseline_ping "$RECONVERGE_TIMEOUT" "$RECONVERGE_STALL" || true
 ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS"
 phase_result "Post-second-rekey (all 20 pairs)"
 echo ""