grabbit
46e592c3a4
Single-crate project doesn't need a subdirectory. Moves Cargo.toml, src/, templates/ to root for standard Rust project layout. Updates .gitignore and test harness binary paths accordingly. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
643 lines
16 KiB
Bash
Executable File
643 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Warpgate Integration Test Harness — shared helpers
|
|
# Provides setup/teardown, assertions, fault injection, and utility functions.
|
|
#
|
|
# Usage: source this file from each test script.
|
|
# source "$SCRIPT_DIR/../harness/helpers.sh"
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Environment & paths
|
|
# ---------------------------------------------------------------------------
|
|
|
|
WARPGATE_BIN="${WARPGATE_BIN:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/target/release/warpgate}"
|
|
WARPGATE_TEST_DIR="${WARPGATE_TEST_DIR:-/tmp/warpgate-test}"
|
|
WARPGATE_TEST_LONG="${WARPGATE_TEST_LONG:-0}"
|
|
WARPGATE_TEST_BTRFS="${WARPGATE_TEST_BTRFS:-}"
|
|
|
|
# Populated by setup_test_env
|
|
TEST_DIR=""
|
|
TEST_CONFIG=""
|
|
TEST_MOUNT=""
|
|
NAS_ROOT=""
|
|
CACHE_DIR=""
|
|
WARPGATE_PID=""
|
|
MOCK_NAS_NS="nas-sim"
|
|
MOCK_NAS_IP="10.99.0.2"
|
|
HOST_IP="10.99.0.1"
|
|
MOCK_NAS_SSHD_PID=""
|
|
TEST_SSH_KEY=""
|
|
TEST_SSH_PUBKEY=""
|
|
|
|
# TAP helpers
|
|
_TEST_NUM=0
|
|
_TEST_FAILURES=0
|
|
|
|
# Track all background PIDs for cleanup
|
|
_BG_PIDS=()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TAP output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
tap_ok() {
|
|
_TEST_NUM=$((_TEST_NUM + 1))
|
|
echo "ok $_TEST_NUM - $1"
|
|
}
|
|
|
|
tap_not_ok() {
|
|
_TEST_NUM=$((_TEST_NUM + 1))
|
|
_TEST_FAILURES=$((_TEST_FAILURES + 1))
|
|
echo "not ok $_TEST_NUM - $1"
|
|
if [[ -n "${2:-}" ]]; then
|
|
echo " # $2"
|
|
fi
|
|
}
|
|
|
|
tap_skip() {
|
|
_TEST_NUM=$((_TEST_NUM + 1))
|
|
echo "ok $_TEST_NUM - SKIP $1"
|
|
}
|
|
|
|
tap_plan() {
|
|
echo "1..$1"
|
|
}
|
|
|
|
tap_exit() {
|
|
exit "$_TEST_FAILURES"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Setup / Teardown
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_test_env() {
|
|
TEST_DIR=$(mktemp -d "${WARPGATE_TEST_DIR}/test-XXXXXX")
|
|
NAS_ROOT="$TEST_DIR/nas-root"
|
|
CACHE_DIR="$TEST_DIR/cache"
|
|
TEST_MOUNT="$TEST_DIR/mnt"
|
|
TEST_CONFIG="$TEST_DIR/config.toml"
|
|
TEST_SSH_KEY="$TEST_DIR/test_key"
|
|
TEST_SSH_PUBKEY="$TEST_DIR/test_key.pub"
|
|
|
|
mkdir -p "$NAS_ROOT" "$CACHE_DIR" "$TEST_MOUNT" "$TEST_DIR/run"
|
|
|
|
# Generate SSH key pair for mock NAS auth
|
|
ssh-keygen -t ed25519 -f "$TEST_SSH_KEY" -N "" -q
|
|
|
|
export TEST_DIR TEST_CONFIG TEST_MOUNT NAS_ROOT CACHE_DIR
|
|
export TEST_SSH_KEY TEST_SSH_PUBKEY
|
|
}
|
|
|
|
teardown_test_env() {
|
|
local exit_code=$?
|
|
|
|
# Kill warpgate if running
|
|
if [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null; then
|
|
kill -TERM "$WARPGATE_PID" 2>/dev/null || true
|
|
wait "$WARPGATE_PID" 2>/dev/null || true
|
|
fi
|
|
|
|
# Kill any tracked background PIDs
|
|
for pid in "${_BG_PIDS[@]}"; do
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
fi
|
|
done
|
|
|
|
# Stop mock NAS
|
|
stop_mock_nas 2>/dev/null || true
|
|
|
|
# Clear network injection
|
|
clear_network_injection 2>/dev/null || true
|
|
|
|
# Unmount if still mounted
|
|
if mountpoint -q "$TEST_MOUNT" 2>/dev/null; then
|
|
fusermount3 -uz "$TEST_MOUNT" 2>/dev/null || fusermount -uz "$TEST_MOUNT" 2>/dev/null || true
|
|
fi
|
|
|
|
# Clean up test directory
|
|
if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" ]]; then
|
|
rm -rf "$TEST_DIR"
|
|
fi
|
|
|
|
return $exit_code
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config generation (delegates to config-gen.sh)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
HARNESS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
gen_config() {
|
|
source "$HARNESS_DIR/config-gen.sh"
|
|
_gen_config "$@"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Warpgate process management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
start_warpgate() {
|
|
local log_file="${TEST_DIR}/warpgate.log"
|
|
|
|
"$WARPGATE_BIN" run -c "$TEST_CONFIG" > "$log_file" 2>&1 &
|
|
WARPGATE_PID=$!
|
|
_BG_PIDS+=("$WARPGATE_PID")
|
|
|
|
export WARPGATE_PID
|
|
}
|
|
|
|
start_warpgate_with_args() {
|
|
local log_file="${TEST_DIR}/warpgate.log"
|
|
local cmd="$1"
|
|
shift
|
|
|
|
"$WARPGATE_BIN" "$cmd" -c "$TEST_CONFIG" "$@" > "$log_file" 2>&1 &
|
|
WARPGATE_PID=$!
|
|
_BG_PIDS+=("$WARPGATE_PID")
|
|
|
|
export WARPGATE_PID
|
|
}
|
|
|
|
stop_warpgate() {
|
|
if [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null; then
|
|
kill -TERM "$WARPGATE_PID"
|
|
wait_for_exit "$WARPGATE_PID" 30
|
|
fi
|
|
}
|
|
|
|
warpgate_log() {
|
|
cat "${TEST_DIR}/warpgate.log" 2>/dev/null || true
|
|
}
|
|
|
|
# Run a warpgate subcommand (not `run`) and capture output
|
|
run_warpgate_cmd() {
|
|
local cmd="$1"
|
|
shift
|
|
"$WARPGATE_BIN" "$cmd" -c "$TEST_CONFIG" "$@" 2>&1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wait helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
wait_for_mount() {
|
|
local timeout="${1:-30}"
|
|
local deadline=$((SECONDS + timeout))
|
|
|
|
while [[ $SECONDS -lt $deadline ]]; do
|
|
if mountpoint -q "$TEST_MOUNT" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "TIMEOUT: mount not ready after ${timeout}s" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_for_rc_api() {
|
|
local timeout="${1:-10}"
|
|
local deadline=$((SECONDS + timeout))
|
|
|
|
while [[ $SECONDS -lt $deadline ]]; do
|
|
if curl -sf "http://127.0.0.1:5572/core/stats" -d '{}' > /dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "TIMEOUT: RC API not ready after ${timeout}s" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_for_file() {
|
|
local path="$1"
|
|
local timeout="${2:-10}"
|
|
local deadline=$((SECONDS + timeout))
|
|
|
|
while [[ $SECONDS -lt $deadline ]]; do
|
|
if [[ -f "$path" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "TIMEOUT: file $path not found after ${timeout}s" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_for_exit() {
|
|
local pid="$1"
|
|
local timeout="${2:-30}"
|
|
local deadline=$((SECONDS + timeout))
|
|
|
|
while [[ $SECONDS -lt $deadline ]]; do
|
|
if ! kill -0 "$pid" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "TIMEOUT: PID $pid did not exit after ${timeout}s" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_for_dirty_zero() {
|
|
local timeout="${1:-60}"
|
|
local deadline=$((SECONDS + timeout))
|
|
|
|
while [[ $SECONDS -lt $deadline ]]; do
|
|
local dirty
|
|
dirty=$(get_dirty_count 2>/dev/null) || true
|
|
if [[ "$dirty" == "0" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
echo "TIMEOUT: dirty count did not reach 0 after ${timeout}s" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_for_log_line() {
|
|
local pattern="$1"
|
|
local timeout="${2:-30}"
|
|
local log_file="${TEST_DIR}/warpgate.log"
|
|
local deadline=$((SECONDS + timeout))
|
|
|
|
while [[ $SECONDS -lt $deadline ]]; do
|
|
if grep -q "$pattern" "$log_file" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "TIMEOUT: log pattern '$pattern' not found after ${timeout}s" >&2
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# RC API helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
rc_api() {
|
|
local endpoint="$1"
|
|
local json="${2:-{}}"
|
|
curl -sf "http://127.0.0.1:5572/$endpoint" -d "$json" 2>/dev/null
|
|
}
|
|
|
|
get_dirty_count() {
|
|
local stats
|
|
stats=$(rc_api "vfs/stats")
|
|
local in_progress queued
|
|
in_progress=$(echo "$stats" | jq -r '.diskCache.uploadsInProgress // 0')
|
|
queued=$(echo "$stats" | jq -r '.diskCache.uploadsQueued // 0')
|
|
echo $((in_progress + queued))
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Assertions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
assert_file_content() {
|
|
local path="$1"
|
|
local expected="$2"
|
|
|
|
if [[ ! -f "$path" ]]; then
|
|
echo "FAIL: file does not exist: $path" >&2
|
|
return 1
|
|
fi
|
|
|
|
if [[ -f "$expected" ]]; then
|
|
# Compare against another file
|
|
if ! diff -q "$path" "$expected" > /dev/null 2>&1; then
|
|
echo "FAIL: file content mismatch: $path vs $expected" >&2
|
|
return 1
|
|
fi
|
|
else
|
|
# Compare against a string
|
|
local actual
|
|
actual=$(cat "$path")
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
echo "FAIL: file content mismatch in $path" >&2
|
|
echo " expected: $expected" >&2
|
|
echo " actual: $actual" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_file_exists() {
|
|
local path="$1"
|
|
if [[ ! -f "$path" ]]; then
|
|
echo "FAIL: file does not exist: $path" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_dir_exists() {
|
|
local path="$1"
|
|
if [[ ! -d "$path" ]]; then
|
|
echo "FAIL: directory does not exist: $path" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_cached() {
|
|
local relative_path="$1"
|
|
local cache_file="$CACHE_DIR/vfs/nas/$relative_path"
|
|
|
|
if [[ ! -f "$cache_file" ]]; then
|
|
echo "FAIL: not cached: $relative_path (expected at $cache_file)" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_not_cached() {
|
|
local relative_path="$1"
|
|
local cache_file="$CACHE_DIR/vfs/nas/$relative_path"
|
|
|
|
if [[ -f "$cache_file" ]]; then
|
|
echo "FAIL: unexpectedly cached: $relative_path" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_dirty_count() {
|
|
local expected="$1"
|
|
local actual
|
|
actual=$(get_dirty_count)
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
echo "FAIL: dirty count mismatch: expected=$expected actual=$actual" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_exit_code() {
|
|
local pid="$1"
|
|
local expected="$2"
|
|
|
|
local actual=0
|
|
wait "$pid" 2>/dev/null || actual=$?
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
echo "FAIL: exit code mismatch for PID $pid: expected=$expected actual=$actual" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_mounted() {
|
|
if ! mountpoint -q "$TEST_MOUNT" 2>/dev/null; then
|
|
echo "FAIL: $TEST_MOUNT is not mounted" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_not_mounted() {
|
|
if mountpoint -q "$TEST_MOUNT" 2>/dev/null; then
|
|
echo "FAIL: $TEST_MOUNT is still mounted" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_no_orphan_rclone() {
|
|
local count
|
|
count=$(pgrep -c -f "rclone.*$TEST_MOUNT" 2>/dev/null || echo 0)
|
|
if [[ "$count" -gt 0 ]]; then
|
|
echo "FAIL: orphan rclone processes found for $TEST_MOUNT" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_output_contains() {
|
|
local output="$1"
|
|
local pattern="$2"
|
|
if ! echo "$output" | grep -q "$pattern"; then
|
|
echo "FAIL: output does not contain '$pattern'" >&2
|
|
echo " output: $output" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_output_not_contains() {
|
|
local output="$1"
|
|
local pattern="$2"
|
|
if echo "$output" | grep -q "$pattern"; then
|
|
echo "FAIL: output unexpectedly contains '$pattern'" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_log_contains() {
|
|
local pattern="$1"
|
|
if ! grep -q "$pattern" "$TEST_DIR/warpgate.log" 2>/dev/null; then
|
|
echo "FAIL: log does not contain '$pattern'" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_log_not_contains() {
|
|
local pattern="$1"
|
|
if grep -q "$pattern" "$TEST_DIR/warpgate.log" 2>/dev/null; then
|
|
echo "FAIL: log unexpectedly contains '$pattern'" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
assert_log_order() {
|
|
# Verify that pattern1 appears before pattern2 in the log
|
|
local pattern1="$1"
|
|
local pattern2="$2"
|
|
local log="$TEST_DIR/warpgate.log"
|
|
|
|
local line1 line2
|
|
line1=$(grep -n "$pattern1" "$log" 2>/dev/null | head -1 | cut -d: -f1)
|
|
line2=$(grep -n "$pattern2" "$log" 2>/dev/null | head -1 | cut -d: -f1)
|
|
|
|
if [[ -z "$line1" ]]; then
|
|
echo "FAIL: pattern '$pattern1' not found in log" >&2
|
|
return 1
|
|
fi
|
|
if [[ -z "$line2" ]]; then
|
|
echo "FAIL: pattern '$pattern2' not found in log" >&2
|
|
return 1
|
|
fi
|
|
if [[ "$line1" -ge "$line2" ]]; then
|
|
echo "FAIL: '$pattern1' (line $line1) does not appear before '$pattern2' (line $line2)" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Network fault injection (requires root + network namespace)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
inject_network_down() {
|
|
ip netns exec "$MOCK_NAS_NS" ip link set veth-nas down 2>/dev/null || \
|
|
ip link set veth-wg down 2>/dev/null || true
|
|
}
|
|
|
|
inject_network_up() {
|
|
ip netns exec "$MOCK_NAS_NS" ip link set veth-nas up 2>/dev/null || true
|
|
ip link set veth-wg up 2>/dev/null || true
|
|
}
|
|
|
|
inject_latency() {
|
|
local ms="$1"
|
|
# Remove existing qdisc first
|
|
tc qdisc del dev veth-wg root 2>/dev/null || true
|
|
tc qdisc add dev veth-wg root netem delay "${ms}ms"
|
|
}
|
|
|
|
inject_packet_loss() {
|
|
local pct="$1"
|
|
tc qdisc del dev veth-wg root 2>/dev/null || true
|
|
tc qdisc add dev veth-wg root netem loss "${pct}%"
|
|
}
|
|
|
|
clear_network_injection() {
|
|
tc qdisc del dev veth-wg root 2>/dev/null || true
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test file creation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
create_test_file() {
|
|
local path="$1"
|
|
local size_kb="${2:-1}"
|
|
|
|
local full_path
|
|
if [[ "$path" == /* ]]; then
|
|
full_path="$path"
|
|
else
|
|
full_path="$NAS_ROOT/$path"
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$full_path")"
|
|
dd if=/dev/urandom of="$full_path" bs=1K count="$size_kb" 2>/dev/null
|
|
}
|
|
|
|
create_test_file_content() {
|
|
local path="$1"
|
|
local content="$2"
|
|
|
|
local full_path
|
|
if [[ "$path" == /* ]]; then
|
|
full_path="$path"
|
|
else
|
|
full_path="$NAS_ROOT/$path"
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$full_path")"
|
|
echo -n "$content" > "$full_path"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Power loss simulation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
simulate_power_loss() {
|
|
# Kill all warpgate-related processes with SIGKILL
|
|
if [[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null; then
|
|
# Kill the entire process group
|
|
kill -9 -"$WARPGATE_PID" 2>/dev/null || kill -9 "$WARPGATE_PID" 2>/dev/null || true
|
|
fi
|
|
|
|
# Also kill any orphaned rclone/smbd processes for this test
|
|
pkill -9 -f "rclone.*$TEST_MOUNT" 2>/dev/null || true
|
|
pkill -9 -f "smbd.*$TEST_DIR" 2>/dev/null || true
|
|
|
|
# Sync filesystem
|
|
sync
|
|
|
|
# Wait briefly for processes to die
|
|
sleep 1
|
|
|
|
WARPGATE_PID=""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Small cache disk (for cache-full tests)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_small_cache_disk() {
|
|
local size_mb="${1:-10}"
|
|
local img="$TEST_DIR/cache-disk.img"
|
|
local loop_dev
|
|
|
|
fallocate -l "${size_mb}M" "$img"
|
|
loop_dev=$(losetup --find --show "$img")
|
|
mkfs.ext4 -q "$loop_dev"
|
|
mount "$loop_dev" "$CACHE_DIR"
|
|
|
|
echo "$loop_dev" > "$TEST_DIR/cache-loop-dev"
|
|
}
|
|
|
|
teardown_small_cache_disk() {
|
|
if [[ -f "$TEST_DIR/cache-loop-dev" ]]; then
|
|
local loop_dev
|
|
loop_dev=$(cat "$TEST_DIR/cache-loop-dev")
|
|
umount "$CACHE_DIR" 2>/dev/null || true
|
|
losetup -d "$loop_dev" 2>/dev/null || true
|
|
rm -f "$TEST_DIR/cache-disk.img" "$TEST_DIR/cache-loop-dev"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Utility: check if running as root
|
|
# ---------------------------------------------------------------------------
|
|
|
|
require_root() {
|
|
if [[ $EUID -ne 0 ]]; then
|
|
echo "SKIP: test requires root" >&2
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
require_command() {
|
|
local cmd="$1"
|
|
if ! command -v "$cmd" > /dev/null 2>&1; then
|
|
echo "SKIP: required command not found: $cmd" >&2
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
require_long_tests() {
|
|
if [[ "$WARPGATE_TEST_LONG" != "1" ]]; then
|
|
echo "SKIP: slow test (set WARPGATE_TEST_LONG=1)" >&2
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Process detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
is_warpgate_running() {
|
|
[[ -n "${WARPGATE_PID:-}" ]] && kill -0 "$WARPGATE_PID" 2>/dev/null
|
|
}
|
|
|
|
count_smbd_processes() {
|
|
pgrep -c -f "smbd.*--configfile" 2>/dev/null || echo 0
|
|
}
|