Add incremental warmup with cache check and auto-warmup on startup
Warmup now checks the rclone VFS cache directory before reading each file through the FUSE mount, skipping already-cached files for fast re-runs. Also adds WarmupConfig with configurable rules that auto-execute when the supervisor starts (best-effort, non-blocking). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
9b37c88cd5
commit
960ddd20ce
@ -59,9 +59,15 @@ pub fn run(config: &Config, path: &str, newer_than: Option<&str>) -> Result<()>
|
||||
println!("Found {total} files to cache.");
|
||||
|
||||
let mut cached = 0usize;
|
||||
let mut skipped = 0usize;
|
||||
let mut errors = 0usize;
|
||||
|
||||
for file in &files {
|
||||
if is_cached(config, path, file) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let full_path = warmup_path.join(file);
|
||||
match std::fs::File::open(&full_path) {
|
||||
Ok(mut f) => {
|
||||
@ -71,7 +77,7 @@ pub fn run(config: &Config, path: &str, newer_than: Option<&str>) -> Result<()>
|
||||
errors += 1;
|
||||
} else {
|
||||
cached += 1;
|
||||
eprint!("\r Cached {cached}/{total}");
|
||||
eprint!("\r Cached {cached}/{total} (skipped {skipped})");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
@ -82,6 +88,24 @@ pub fn run(config: &Config, path: &str, newer_than: Option<&str>) -> Result<()>
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
println!("Warmup complete: {cached} cached, {errors} errors.");
|
||||
println!(
|
||||
"Warmup complete: skipped {skipped} (already cached), cached {cached}, errors {errors}."
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if a file is already in the rclone VFS cache.
|
||||
///
|
||||
/// `warmup_path` is the subdir passed to `warpgate warmup` (e.g. "Image/2026").
|
||||
/// `relative_path` is the filename from `rclone lsf` (relative to warmup_path).
|
||||
fn is_cached(config: &Config, warmup_path: &str, relative_path: &str) -> bool {
|
||||
let cache_path = config
|
||||
.cache
|
||||
.dir
|
||||
.join("vfs")
|
||||
.join("nas")
|
||||
.join(config.connection.remote_path.trim_start_matches('/'))
|
||||
.join(warmup_path)
|
||||
.join(relative_path);
|
||||
cache_path.exists()
|
||||
}
|
||||
|
||||
@ -22,6 +22,8 @@ pub struct Config {
|
||||
pub directory_cache: DirectoryCacheConfig,
|
||||
pub protocols: ProtocolsConfig,
|
||||
pub mount: MountConfig,
|
||||
#[serde(default)]
|
||||
pub warmup: WarmupConfig,
|
||||
}
|
||||
|
||||
/// SFTP connection to remote NAS.
|
||||
@ -141,6 +143,35 @@ pub struct MountConfig {
|
||||
pub point: PathBuf,
|
||||
}
|
||||
|
||||
/// Warmup configuration — auto-cache paths on startup.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WarmupConfig {
|
||||
/// Auto-warmup on startup (default: true when rules exist).
|
||||
#[serde(default = "default_true")]
|
||||
pub auto: bool,
|
||||
/// Warmup rules — paths to pre-cache.
|
||||
#[serde(default)]
|
||||
pub rules: Vec<WarmupRule>,
|
||||
}
|
||||
|
||||
impl Default for WarmupConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
auto: true,
|
||||
rules: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A single warmup rule specifying a path to pre-cache.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WarmupRule {
|
||||
/// Path relative to remote_path.
|
||||
pub path: String,
|
||||
/// Only cache files newer than this (e.g. "7d", "24h").
|
||||
pub newer_than: Option<String>,
|
||||
}
|
||||
|
||||
// --- Default value functions ---
|
||||
|
||||
fn default_sftp_port() -> u16 {
|
||||
|
||||
@ -106,6 +106,21 @@ pub fn run(config: &Config) -> Result<()> {
|
||||
println!("Starting protocol services...");
|
||||
let mut protocols = start_protocols(config)?;
|
||||
|
||||
// Phase 3.5: Auto-warmup (non-blocking, best-effort)
|
||||
if !config.warmup.rules.is_empty() && config.warmup.auto {
|
||||
println!("Running auto-warmup...");
|
||||
for rule in &config.warmup.rules {
|
||||
if shutdown.load(Ordering::SeqCst) {
|
||||
break;
|
||||
}
|
||||
if let Err(e) =
|
||||
crate::cli::warmup::run(config, &rule.path, rule.newer_than.as_deref())
|
||||
{
|
||||
eprintln!("Warmup warning: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 4: Supervision loop
|
||||
println!("Supervision active. Press Ctrl+C to stop.");
|
||||
let result = supervise(config, &mut mount_child, &mut protocols, Arc::clone(&shutdown));
|
||||
|
||||
@ -70,3 +70,14 @@ webdav_port = 8080
|
||||
[mount]
|
||||
# FUSE mount point (all protocols share this)
|
||||
point = "/mnt/nas-photos"
|
||||
|
||||
[warmup]
|
||||
# Auto-warmup configured paths on startup
|
||||
auto = true
|
||||
|
||||
# [[warmup.rules]]
|
||||
# path = "2024"
|
||||
# newer_than = "30d"
|
||||
#
|
||||
# [[warmup.rules]]
|
||||
# path = "Lightroom/Catalog"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user