#!/usr/bin/env bash # Test: warpgate survives 60 s of network jitter without crashing # # Runs a stress loop that writes a small file every 2 seconds while # toggling the network up/down every 5 seconds. After the loop, restores # the network and waits for the write-back queue to drain. Verifies that # warpgate is still running and that all written files eventually reach the # NAS. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "$SCRIPT_DIR/../harness/helpers.sh" source "$SCRIPT_DIR/../harness/mock-nas.sh" require_root require_long_tests # ~60 s runtime setup_test_env trap teardown_test_env EXIT # --- Arrange --- start_mock_nas gen_config write_back=2s start_warpgate wait_for_mount wait_for_rc_api # --- Act: stress loop --- DURATION=60 start_time=$SECONDS file_index=0 # Background writer: drop a file every 2 s ( while [[ $(( SECONDS - start_time )) -lt $DURATION ]]; do fname="jitter-${file_index}.txt" echo "jitter-data-${file_index}" > "$TEST_MOUNT/$fname" 2>/dev/null || true file_index=$(( file_index + 1 )) sleep 2 done # Signal completion echo "$file_index" > "$TEST_DIR/writer_count" ) & writer_pid=$! _BG_PIDS+=("$writer_pid") # Background toggler: flip network every 5 s ( net_up=true while [[ $(( SECONDS - start_time )) -lt $DURATION ]]; do sleep 5 if $net_up; then inject_network_down 2>/dev/null || true net_up=false else inject_network_up 2>/dev/null || true net_up=true fi done ) & toggler_pid=$! _BG_PIDS+=("$toggler_pid") # Wait for both loops to finish wait "$writer_pid" 2>/dev/null || true wait "$toggler_pid" 2>/dev/null || true # --- Restore & drain --- inject_network_up sleep 2 # Read the count of files written total_files=0 if [[ -f "$TEST_DIR/writer_count" ]]; then total_files=$(cat "$TEST_DIR/writer_count") fi echo " files written during jitter loop: $total_files" # Wait for all dirty files to be written back wait_for_dirty_zero 120 # --- Assert --- # 1. Warpgate must still be running (no crash) if ! is_warpgate_running; then echo "FAIL: warpgate crashed during network jitter stress test" >&2 warpgate_log >&2 exit 1 fi # 2. Every file created by the writer must have arrived on the NAS missing=0 for (( i = 0; i < total_files; i++ )); do fname="jitter-${i}.txt" if ! nas_file_exists "$fname"; then echo " MISSING on NAS: $fname" >&2 missing=$(( missing + 1 )) fi done if [[ "$missing" -gt 0 ]]; then echo "FAIL: $missing of $total_files files did not reach the NAS" >&2 exit 1 fi echo " all $total_files files verified on NAS" echo "PASS: $(basename "$0" .sh)"