#!/usr/bin/env bash # Warpgate Integration Test Harness — shared helpers # Provides setup/teardown, assertions, fault injection, and utility functions. # # Usage: source this file from each test script. # source "$SCRIPT_DIR/../harness/helpers.sh" set -euo pipefail # --------------------------------------------------------------------------- # Environment & paths # --------------------------------------------------------------------------- WARPGATE_BIN="${WARPGATE_BIN:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/target/release/warpgate}" WARPGATE_TEST_DIR="${WARPGATE_TEST_DIR:-/tmp/warpgate-test}" WARPGATE_TEST_LONG="${WARPGATE_TEST_LONG:-0}" WARPGATE_TEST_BTRFS="${WARPGATE_TEST_BTRFS:-}" # Populated by setup_test_env TEST_DIR="" TEST_CONFIG="" TEST_MOUNT="" NAS_ROOT="" CACHE_DIR="" WARPGATE_PID="" MOCK_NAS_NS="nas-sim" MOCK_NAS_IP="10.99.0.2" HOST_IP="10.99.0.1" MOCK_NAS_SSHD_PID="" TEST_SSH_KEY="" TEST_SSH_PUBKEY="" # TAP helpers _TEST_NUM=0 _TEST_FAILURES=0 # Track all background PIDs for cleanup _BG_PIDS=() # --------------------------------------------------------------------------- # TAP output # --------------------------------------------------------------------------- tap_ok() { _TEST_NUM=$((_TEST_NUM + 1)) echo "ok $_TEST_NUM - $1" } tap_not_ok() { _TEST_NUM=$((_TEST_NUM + 1)) _TEST_FAILURES=$((_TEST_FAILURES + 1)) echo "not ok $_TEST_NUM - $1" if [[ -n "${2:-}" ]]; then echo " # $2" fi } tap_skip() { _TEST_NUM=$((_TEST_NUM + 1)) echo "ok $_TEST_NUM - SKIP $1" } tap_plan() { echo "1..$1" } tap_exit() { exit "$_TEST_FAILURES" } # --------------------------------------------------------------------------- # Setup / Teardown # --------------------------------------------------------------------------- setup_test_env() { TEST_DIR=$(mktemp -d "${WARPGATE_TEST_DIR}/test-XXXXXX") NAS_ROOT="$TEST_DIR/nas-root" CACHE_DIR="$TEST_DIR/cache" TEST_MOUNT="$TEST_DIR/mnt" TEST_CONFIG="$TEST_DIR/config.toml" TEST_SSH_KEY="$TEST_DIR/test_key" TEST_SSH_PUBKEY="$TEST_DIR/test_key.pub" mkdir -p "$NAS_ROOT" "$CACHE_DIR" "$TEST_MOUNT" "$TEST_DIR/run" # Generate SSH key pair for mock NAS auth ssh-keygen -t ed25519 -f "$TEST_SSH_KEY" -N "" -q export TEST_DIR TEST_CONFIG TEST_MOUNT NAS_ROOT CACHE_DIR export TEST_SSH_KEY TEST_SSH_PUBKEY } teardown_test_env() { local exit_code=$? # Kill warpgate if running if [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null; then kill -TERM "$WARPGATE_PID" 2>/dev/null || true wait "$WARPGATE_PID" 2>/dev/null || true fi # Kill any tracked background PIDs for pid in "${_BG_PIDS[@]}"; do if kill -0 "$pid" 2>/dev/null; then kill -9 "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true fi done # Stop mock NAS stop_mock_nas 2>/dev/null || true # Clear network injection clear_network_injection 2>/dev/null || true # Unmount if still mounted if mountpoint -q "$TEST_MOUNT" 2>/dev/null; then fusermount3 -uz "$TEST_MOUNT" 2>/dev/null || fusermount -uz "$TEST_MOUNT" 2>/dev/null || true fi # Clean up test directory if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" ]]; then rm -rf "$TEST_DIR" fi return $exit_code } # --------------------------------------------------------------------------- # Config generation (delegates to config-gen.sh) # --------------------------------------------------------------------------- HARNESS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" gen_config() { source "$HARNESS_DIR/config-gen.sh" _gen_config "$@" } # --------------------------------------------------------------------------- # Warpgate process management # --------------------------------------------------------------------------- start_warpgate() { local log_file="${TEST_DIR}/warpgate.log" "$WARPGATE_BIN" run -c "$TEST_CONFIG" > "$log_file" 2>&1 & WARPGATE_PID=$! _BG_PIDS+=("$WARPGATE_PID") export WARPGATE_PID } start_warpgate_with_args() { local log_file="${TEST_DIR}/warpgate.log" local cmd="$1" shift "$WARPGATE_BIN" "$cmd" -c "$TEST_CONFIG" "$@" > "$log_file" 2>&1 & WARPGATE_PID=$! _BG_PIDS+=("$WARPGATE_PID") export WARPGATE_PID } stop_warpgate() { if [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null; then kill -TERM "$WARPGATE_PID" wait_for_exit "$WARPGATE_PID" 30 fi } warpgate_log() { cat "${TEST_DIR}/warpgate.log" 2>/dev/null || true } # Run a warpgate subcommand (not `run`) and capture output run_warpgate_cmd() { local cmd="$1" shift "$WARPGATE_BIN" "$cmd" -c "$TEST_CONFIG" "$@" 2>&1 } # --------------------------------------------------------------------------- # Wait helpers # --------------------------------------------------------------------------- wait_for_mount() { local timeout="${1:-30}" local deadline=$((SECONDS + timeout)) while [[ $SECONDS -lt $deadline ]]; do if mountpoint -q "$TEST_MOUNT" 2>/dev/null; then return 0 fi sleep 0.5 done echo "TIMEOUT: mount not ready after ${timeout}s" >&2 return 1 } wait_for_rc_api() { local timeout="${1:-10}" local deadline=$((SECONDS + timeout)) while [[ $SECONDS -lt $deadline ]]; do if curl -sf "http://127.0.0.1:5572/core/stats" -d '{}' > /dev/null 2>&1; then return 0 fi sleep 0.5 done echo "TIMEOUT: RC API not ready after ${timeout}s" >&2 return 1 } wait_for_file() { local path="$1" local timeout="${2:-10}" local deadline=$((SECONDS + timeout)) while [[ $SECONDS -lt $deadline ]]; do if [[ -f "$path" ]]; then return 0 fi sleep 0.5 done echo "TIMEOUT: file $path not found after ${timeout}s" >&2 return 1 } wait_for_exit() { local pid="$1" local timeout="${2:-30}" local deadline=$((SECONDS + timeout)) while [[ $SECONDS -lt $deadline ]]; do if ! kill -0 "$pid" 2>/dev/null; then return 0 fi sleep 0.5 done echo "TIMEOUT: PID $pid did not exit after ${timeout}s" >&2 return 1 } wait_for_dirty_zero() { local timeout="${1:-60}" local deadline=$((SECONDS + timeout)) while [[ $SECONDS -lt $deadline ]]; do local dirty dirty=$(get_dirty_count 2>/dev/null) || true if [[ "$dirty" == "0" ]]; then return 0 fi sleep 2 done echo "TIMEOUT: dirty count did not reach 0 after ${timeout}s" >&2 return 1 } wait_for_log_line() { local pattern="$1" local timeout="${2:-30}" local log_file="${TEST_DIR}/warpgate.log" local deadline=$((SECONDS + timeout)) while [[ $SECONDS -lt $deadline ]]; do if grep -q "$pattern" "$log_file" 2>/dev/null; then return 0 fi sleep 0.5 done echo "TIMEOUT: log pattern '$pattern' not found after ${timeout}s" >&2 return 1 } # --------------------------------------------------------------------------- # RC API helpers # --------------------------------------------------------------------------- rc_api() { local endpoint="$1" local json="${2:-{}}" curl -sf "http://127.0.0.1:5572/$endpoint" -d "$json" 2>/dev/null } get_dirty_count() { local stats stats=$(rc_api "vfs/stats") local in_progress queued in_progress=$(echo "$stats" | jq -r '.diskCache.uploadsInProgress // 0') queued=$(echo "$stats" | jq -r '.diskCache.uploadsQueued // 0') echo $((in_progress + queued)) } # --------------------------------------------------------------------------- # Assertions # --------------------------------------------------------------------------- assert_file_content() { local path="$1" local expected="$2" if [[ ! -f "$path" ]]; then echo "FAIL: file does not exist: $path" >&2 return 1 fi if [[ -f "$expected" ]]; then # Compare against another file if ! diff -q "$path" "$expected" > /dev/null 2>&1; then echo "FAIL: file content mismatch: $path vs $expected" >&2 return 1 fi else # Compare against a string local actual actual=$(cat "$path") if [[ "$actual" != "$expected" ]]; then echo "FAIL: file content mismatch in $path" >&2 echo " expected: $expected" >&2 echo " actual: $actual" >&2 return 1 fi fi return 0 } assert_file_exists() { local path="$1" if [[ ! -f "$path" ]]; then echo "FAIL: file does not exist: $path" >&2 return 1 fi return 0 } assert_dir_exists() { local path="$1" if [[ ! -d "$path" ]]; then echo "FAIL: directory does not exist: $path" >&2 return 1 fi return 0 } assert_cached() { local relative_path="$1" local cache_file="$CACHE_DIR/vfs/nas/$relative_path" if [[ ! -f "$cache_file" ]]; then echo "FAIL: not cached: $relative_path (expected at $cache_file)" >&2 return 1 fi return 0 } assert_not_cached() { local relative_path="$1" local cache_file="$CACHE_DIR/vfs/nas/$relative_path" if [[ -f "$cache_file" ]]; then echo "FAIL: unexpectedly cached: $relative_path" >&2 return 1 fi return 0 } assert_dirty_count() { local expected="$1" local actual actual=$(get_dirty_count) if [[ "$actual" != "$expected" ]]; then echo "FAIL: dirty count mismatch: expected=$expected actual=$actual" >&2 return 1 fi return 0 } assert_exit_code() { local pid="$1" local expected="$2" local actual=0 wait "$pid" 2>/dev/null || actual=$? if [[ "$actual" != "$expected" ]]; then echo "FAIL: exit code mismatch for PID $pid: expected=$expected actual=$actual" >&2 return 1 fi return 0 } assert_mounted() { if ! mountpoint -q "$TEST_MOUNT" 2>/dev/null; then echo "FAIL: $TEST_MOUNT is not mounted" >&2 return 1 fi return 0 } assert_not_mounted() { if mountpoint -q "$TEST_MOUNT" 2>/dev/null; then echo "FAIL: $TEST_MOUNT is still mounted" >&2 return 1 fi return 0 } assert_no_orphan_rclone() { local count count=$(pgrep -c -f "rclone.*$TEST_MOUNT" 2>/dev/null || echo 0) if [[ "$count" -gt 0 ]]; then echo "FAIL: orphan rclone processes found for $TEST_MOUNT" >&2 return 1 fi return 0 } assert_output_contains() { local output="$1" local pattern="$2" if ! echo "$output" | grep -q "$pattern"; then echo "FAIL: output does not contain '$pattern'" >&2 echo " output: $output" >&2 return 1 fi return 0 } assert_output_not_contains() { local output="$1" local pattern="$2" if echo "$output" | grep -q "$pattern"; then echo "FAIL: output unexpectedly contains '$pattern'" >&2 return 1 fi return 0 } assert_log_contains() { local pattern="$1" if ! grep -q "$pattern" "$TEST_DIR/warpgate.log" 2>/dev/null; then echo "FAIL: log does not contain '$pattern'" >&2 return 1 fi return 0 } assert_log_not_contains() { local pattern="$1" if grep -q "$pattern" "$TEST_DIR/warpgate.log" 2>/dev/null; then echo "FAIL: log unexpectedly contains '$pattern'" >&2 return 1 fi return 0 } assert_log_order() { # Verify that pattern1 appears before pattern2 in the log local pattern1="$1" local pattern2="$2" local log="$TEST_DIR/warpgate.log" local line1 line2 line1=$(grep -n "$pattern1" "$log" 2>/dev/null | head -1 | cut -d: -f1) line2=$(grep -n "$pattern2" "$log" 2>/dev/null | head -1 | cut -d: -f1) if [[ -z "$line1" ]]; then echo "FAIL: pattern '$pattern1' not found in log" >&2 return 1 fi if [[ -z "$line2" ]]; then echo "FAIL: pattern '$pattern2' not found in log" >&2 return 1 fi if [[ "$line1" -ge "$line2" ]]; then echo "FAIL: '$pattern1' (line $line1) does not appear before '$pattern2' (line $line2)" >&2 return 1 fi return 0 } # --------------------------------------------------------------------------- # Network fault injection (requires root + network namespace) # --------------------------------------------------------------------------- inject_network_down() { ip netns exec "$MOCK_NAS_NS" ip link set veth-nas down 2>/dev/null || \ ip link set veth-wg down 2>/dev/null || true } inject_network_up() { ip netns exec "$MOCK_NAS_NS" ip link set veth-nas up 2>/dev/null || true ip link set veth-wg up 2>/dev/null || true } inject_latency() { local ms="$1" # Remove existing qdisc first tc qdisc del dev veth-wg root 2>/dev/null || true tc qdisc add dev veth-wg root netem delay "${ms}ms" } inject_packet_loss() { local pct="$1" tc qdisc del dev veth-wg root 2>/dev/null || true tc qdisc add dev veth-wg root netem loss "${pct}%" } clear_network_injection() { tc qdisc del dev veth-wg root 2>/dev/null || true } # --------------------------------------------------------------------------- # Test file creation # --------------------------------------------------------------------------- create_test_file() { local path="$1" local size_kb="${2:-1}" local full_path if [[ "$path" == /* ]]; then full_path="$path" else full_path="$NAS_ROOT/$path" fi mkdir -p "$(dirname "$full_path")" dd if=/dev/urandom of="$full_path" bs=1K count="$size_kb" 2>/dev/null } create_test_file_content() { local path="$1" local content="$2" local full_path if [[ "$path" == /* ]]; then full_path="$path" else full_path="$NAS_ROOT/$path" fi mkdir -p "$(dirname "$full_path")" echo -n "$content" > "$full_path" } # --------------------------------------------------------------------------- # Power loss simulation # --------------------------------------------------------------------------- simulate_power_loss() { # Kill all warpgate-related processes with SIGKILL if [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null; then # Kill the entire process group kill -9 -"$WARPGATE_PID" 2>/dev/null || kill -9 "$WARPGATE_PID" 2>/dev/null || true fi # Also kill any orphaned rclone/smbd processes for this test pkill -9 -f "rclone.*$TEST_MOUNT" 2>/dev/null || true pkill -9 -f "smbd.*$TEST_DIR" 2>/dev/null || true # Sync filesystem sync # Wait briefly for processes to die sleep 1 WARPGATE_PID="" } # --------------------------------------------------------------------------- # Small cache disk (for cache-full tests) # --------------------------------------------------------------------------- setup_small_cache_disk() { local size_mb="${1:-10}" local img="$TEST_DIR/cache-disk.img" local loop_dev fallocate -l "${size_mb}M" "$img" loop_dev=$(losetup --find --show "$img") mkfs.ext4 -q "$loop_dev" mount "$loop_dev" "$CACHE_DIR" echo "$loop_dev" > "$TEST_DIR/cache-loop-dev" } teardown_small_cache_disk() { if [[ -f "$TEST_DIR/cache-loop-dev" ]]; then local loop_dev loop_dev=$(cat "$TEST_DIR/cache-loop-dev") umount "$CACHE_DIR" 2>/dev/null || true losetup -d "$loop_dev" 2>/dev/null || true rm -f "$TEST_DIR/cache-disk.img" "$TEST_DIR/cache-loop-dev" fi } # --------------------------------------------------------------------------- # Utility: check if running as root # --------------------------------------------------------------------------- require_root() { if [[ $EUID -ne 0 ]]; then echo "SKIP: test requires root" >&2 exit 0 fi } require_command() { local cmd="$1" if ! command -v "$cmd" > /dev/null 2>&1; then echo "SKIP: required command not found: $cmd" >&2 exit 0 fi } require_long_tests() { if [[ "$WARPGATE_TEST_LONG" != "1" ]]; then echo "SKIP: slow test (set WARPGATE_TEST_LONG=1)" >&2 exit 0 fi } # --------------------------------------------------------------------------- # Process detection # --------------------------------------------------------------------------- is_warpgate_running() { [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null } count_smbd_processes() { pgrep -c -f "smbd.*--configfile" 2>/dev/null || echo 0 }