Add incremental warmup with cache check and auto-warmup on startup
Warmup now checks the rclone VFS cache directory before reading each file through the FUSE mount, skipping already-cached files for fast re-runs. Also adds WarmupConfig with configurable rules that auto-execute when the supervisor starts (best-effort, non-blocking). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
9b37c88cd5
commit
960ddd20ce
@ -59,9 +59,15 @@ pub fn run(config: &Config, path: &str, newer_than: Option<&str>) -> Result<()>
|
|||||||
println!("Found {total} files to cache.");
|
println!("Found {total} files to cache.");
|
||||||
|
|
||||||
let mut cached = 0usize;
|
let mut cached = 0usize;
|
||||||
|
let mut skipped = 0usize;
|
||||||
let mut errors = 0usize;
|
let mut errors = 0usize;
|
||||||
|
|
||||||
for file in &files {
|
for file in &files {
|
||||||
|
if is_cached(config, path, file) {
|
||||||
|
skipped += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let full_path = warmup_path.join(file);
|
let full_path = warmup_path.join(file);
|
||||||
match std::fs::File::open(&full_path) {
|
match std::fs::File::open(&full_path) {
|
||||||
Ok(mut f) => {
|
Ok(mut f) => {
|
||||||
@ -71,7 +77,7 @@ pub fn run(config: &Config, path: &str, newer_than: Option<&str>) -> Result<()>
|
|||||||
errors += 1;
|
errors += 1;
|
||||||
} else {
|
} else {
|
||||||
cached += 1;
|
cached += 1;
|
||||||
eprint!("\r Cached {cached}/{total}");
|
eprint!("\r Cached {cached}/{total} (skipped {skipped})");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@ -82,6 +88,24 @@ pub fn run(config: &Config, path: &str, newer_than: Option<&str>) -> Result<()>
|
|||||||
}
|
}
|
||||||
|
|
||||||
eprintln!();
|
eprintln!();
|
||||||
println!("Warmup complete: {cached} cached, {errors} errors.");
|
println!(
|
||||||
|
"Warmup complete: skipped {skipped} (already cached), cached {cached}, errors {errors}."
|
||||||
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Check if a file is already in the rclone VFS cache.
|
||||||
|
///
|
||||||
|
/// `warmup_path` is the subdir passed to `warpgate warmup` (e.g. "Image/2026").
|
||||||
|
/// `relative_path` is the filename from `rclone lsf` (relative to warmup_path).
|
||||||
|
fn is_cached(config: &Config, warmup_path: &str, relative_path: &str) -> bool {
|
||||||
|
let cache_path = config
|
||||||
|
.cache
|
||||||
|
.dir
|
||||||
|
.join("vfs")
|
||||||
|
.join("nas")
|
||||||
|
.join(config.connection.remote_path.trim_start_matches('/'))
|
||||||
|
.join(warmup_path)
|
||||||
|
.join(relative_path);
|
||||||
|
cache_path.exists()
|
||||||
|
}
|
||||||
|
|||||||
@ -22,6 +22,8 @@ pub struct Config {
|
|||||||
pub directory_cache: DirectoryCacheConfig,
|
pub directory_cache: DirectoryCacheConfig,
|
||||||
pub protocols: ProtocolsConfig,
|
pub protocols: ProtocolsConfig,
|
||||||
pub mount: MountConfig,
|
pub mount: MountConfig,
|
||||||
|
#[serde(default)]
|
||||||
|
pub warmup: WarmupConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// SFTP connection to remote NAS.
|
/// SFTP connection to remote NAS.
|
||||||
@ -141,6 +143,35 @@ pub struct MountConfig {
|
|||||||
pub point: PathBuf,
|
pub point: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Warmup configuration — auto-cache paths on startup.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct WarmupConfig {
|
||||||
|
/// Auto-warmup on startup (default: true when rules exist).
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub auto: bool,
|
||||||
|
/// Warmup rules — paths to pre-cache.
|
||||||
|
#[serde(default)]
|
||||||
|
pub rules: Vec<WarmupRule>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WarmupConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
auto: true,
|
||||||
|
rules: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A single warmup rule specifying a path to pre-cache.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct WarmupRule {
|
||||||
|
/// Path relative to remote_path.
|
||||||
|
pub path: String,
|
||||||
|
/// Only cache files newer than this (e.g. "7d", "24h").
|
||||||
|
pub newer_than: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
// --- Default value functions ---
|
// --- Default value functions ---
|
||||||
|
|
||||||
fn default_sftp_port() -> u16 {
|
fn default_sftp_port() -> u16 {
|
||||||
|
|||||||
@ -106,6 +106,21 @@ pub fn run(config: &Config) -> Result<()> {
|
|||||||
println!("Starting protocol services...");
|
println!("Starting protocol services...");
|
||||||
let mut protocols = start_protocols(config)?;
|
let mut protocols = start_protocols(config)?;
|
||||||
|
|
||||||
|
// Phase 3.5: Auto-warmup (non-blocking, best-effort)
|
||||||
|
if !config.warmup.rules.is_empty() && config.warmup.auto {
|
||||||
|
println!("Running auto-warmup...");
|
||||||
|
for rule in &config.warmup.rules {
|
||||||
|
if shutdown.load(Ordering::SeqCst) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if let Err(e) =
|
||||||
|
crate::cli::warmup::run(config, &rule.path, rule.newer_than.as_deref())
|
||||||
|
{
|
||||||
|
eprintln!("Warmup warning: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Phase 4: Supervision loop
|
// Phase 4: Supervision loop
|
||||||
println!("Supervision active. Press Ctrl+C to stop.");
|
println!("Supervision active. Press Ctrl+C to stop.");
|
||||||
let result = supervise(config, &mut mount_child, &mut protocols, Arc::clone(&shutdown));
|
let result = supervise(config, &mut mount_child, &mut protocols, Arc::clone(&shutdown));
|
||||||
|
|||||||
@ -70,3 +70,14 @@ webdav_port = 8080
|
|||||||
[mount]
|
[mount]
|
||||||
# FUSE mount point (all protocols share this)
|
# FUSE mount point (all protocols share this)
|
||||||
point = "/mnt/nas-photos"
|
point = "/mnt/nas-photos"
|
||||||
|
|
||||||
|
[warmup]
|
||||||
|
# Auto-warmup configured paths on startup
|
||||||
|
auto = true
|
||||||
|
|
||||||
|
# [[warmup.rules]]
|
||||||
|
# path = "2024"
|
||||||
|
# newer_than = "30d"
|
||||||
|
#
|
||||||
|
# [[warmup.rules]]
|
||||||
|
# path = "Lightroom/Catalog"
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user