use std::{ collections::BTreeMap, path::{Path, PathBuf}, time::Duration, }; use crate::{ app::{AppResult, AppaService}, domain::FolderId, filesystem::{FolderChangeWatcher, FolderLock, lock_app_process, watch_folder}, iroh::NodeHost, storage::FolderConfig, }; const FOLDER_REFRESH_INTERVAL: Duration = Duration::from_secs(1); const LOCAL_CHANGE_DEBOUNCE: Duration = Duration::from_millis(750); const PEER_POLL_INTERVAL: Duration = Duration::from_secs(30); const SAFETY_SCAN_INTERVAL: Duration = Duration::from_secs(10 * 60); const RETRY_BASE_DELAY: Duration = Duration::from_secs(2); const MAX_RETRY_DELAY: Duration = Duration::from_secs(5 * 60); const MAX_RETRY_EXPONENT: u32 = 8; #[derive(Clone, Debug)] struct RetryState { failure_count: u32, retry_at: tokio::time::Instant, } struct FolderWatcher { folder: FolderConfig, events: FolderChangeWatcher, next_local_scan: tokio::time::Instant, next_peer_poll: tokio::time::Instant, next_safety_scan: tokio::time::Instant, } /// Stateful synchronization runtime owned by the daemon process. pub(crate) struct SyncRunner { _lock: FolderLock, node: NodeHost, watchers: Vec, selected_folder_path: Option, next_folder_refresh: tokio::time::Instant, retries: BTreeMap, } impl SyncRunner { pub(crate) async fn start( service: &AppaService, folder_path: Option<&Path>, ) -> AppResult { let folders = service.folders_to_run(folder_path)?; let lock = lock_app_process(&service.paths.data_directory.join("locks"))?; let node = service.load_node().await?; node.wait_until_online().await?; let watchers = folders .into_iter() .map(FolderWatcher::new) .collect::>>()?; tracing::info!(folder_count = watchers.len(), endpoint = %node.endpoint_address().id, "Appa is watching for changes"); Ok(Self { _lock: lock, node, watchers, selected_folder_path: folder_path.map(Path::to_path_buf), next_folder_refresh: tokio::time::Instant::now() + FOLDER_REFRESH_INTERVAL, retries: BTreeMap::new(), }) } pub(crate) async fn tick(&mut self, service: &AppaService) -> AppResult<()> { let now = tokio::time::Instant::now(); if now >= self.next_folder_refresh { service.refresh_watched_folders( &mut self.watchers, self.selected_folder_path.as_deref(), &mut self.retries, )?; self.next_folder_refresh = now + FOLDER_REFRESH_INTERVAL; } service .sync_watched_folders(&mut self.watchers, &self.node, &mut self.retries) .await } pub(crate) async fn shutdown(self) -> AppResult<()> { self.node.shutdown().await } pub(crate) fn node(&self) -> &NodeHost { &self.node } } impl AppaService { fn refresh_watched_folders( &self, watchers: &mut Vec, selected_folder_path: Option<&Path>, retries: &mut BTreeMap, ) -> AppResult<()> { let configured_folders = self.folders_to_watch(selected_folder_path)?; watchers.retain(|watcher| { let is_configured = configured_folders .iter() .any(|folder| folder.id == watcher.folder.id); if !is_configured { retries.remove(&watcher.folder.id); tracing::info!(folder = %watcher.folder.name, "Stopped watching removed folder"); } is_configured }); for folder in configured_folders { refresh_watcher(watchers, folder)?; } Ok(()) } async fn sync_watched_folders( &self, watchers: &mut [FolderWatcher], node: &NodeHost, retries: &mut BTreeMap, ) -> AppResult<()> { let announced_folders = node.take_announced_folders(); for watcher in watchers { if announced_folders.contains(&watcher.folder.id) { retries.remove(&watcher.folder.id); watcher.schedule_remote_sync(); tracing::info!(folder = %watcher.folder.name, "Peer announced folder changes; checking now"); } if watcher.events.has_pending_events() { retries.remove(&watcher.folder.id); watcher.next_local_scan = tokio::time::Instant::now() + LOCAL_CHANGE_DEBOUNCE; tracing::info!(folder = %watcher.folder.name, "Filesystem change detected; checking the folder"); } if retry_is_pending(retries, watcher.folder.id) { continue; } if !watcher.is_sync_due(tokio::time::Instant::now()) { continue; } self.sync_watched_folder(watcher, node, retries).await?; } Ok(()) } async fn sync_watched_folder( &self, watcher: &mut FolderWatcher, node: &NodeHost, retries: &mut BTreeMap, ) -> AppResult<()> { let should_scan_local_files = watcher.is_local_scan_due(tokio::time::Instant::now()); match self .sync_with_node_using_local_scan(&watcher.folder, node, should_scan_local_files) .await { Ok(change_count) => { retries.remove(&watcher.folder.id); watcher.record_successful_sync(should_scan_local_files); if change_count > 0 { tracing::info!(folder = %watcher.folder.name, change_count, "Applied remote changes"); } } Err(error) => { self.state_store .record_sync_failure(watcher.folder.id, &error)?; let retry_delay = schedule_retry(retries, watcher.folder.id); tracing::warn!(folder = %watcher.folder.name, ?retry_delay, %error, "sync pass failed; backing off"); } } Ok(()) } } fn refresh_watcher(watchers: &mut Vec, folder: FolderConfig) -> AppResult<()> { let Some(watcher) = watchers .iter_mut() .find(|watcher| watcher.folder.id == folder.id) else { tracing::info!(folder = %folder.name, "Started watching added folder"); watchers.push(FolderWatcher::new(folder)?); return Ok(()); }; if watcher_matches_folder(watcher, &folder) { return Ok(()); } tracing::info!(folder = %folder.name, "Reloaded changed folder configuration"); *watcher = FolderWatcher::new(folder)?; Ok(()) } fn watcher_matches_folder(watcher: &FolderWatcher, folder: &FolderConfig) -> bool { watcher.folder.name == folder.name && watcher.folder.path == folder.path && watcher.folder.capability == folder.capability && watcher.folder.mode == folder.mode } impl FolderWatcher { fn new(folder: FolderConfig) -> AppResult { let events = watch_folder(&folder.path)?; let now = tokio::time::Instant::now(); Ok(Self { folder, events, next_local_scan: now, next_peer_poll: now, next_safety_scan: now + SAFETY_SCAN_INTERVAL, }) } fn is_sync_due(&self, now: tokio::time::Instant) -> bool { self.is_local_scan_due(now) || now >= self.next_peer_poll } fn is_local_scan_due(&self, now: tokio::time::Instant) -> bool { now >= self.next_local_scan || now >= self.next_safety_scan } fn record_successful_sync(&mut self, scanned_local_files: bool) { let now = tokio::time::Instant::now(); self.next_peer_poll = now + PEER_POLL_INTERVAL; if scanned_local_files { self.next_local_scan = now + SAFETY_SCAN_INTERVAL; self.next_safety_scan = now + SAFETY_SCAN_INTERVAL; } } fn schedule_remote_sync(&mut self) { self.next_peer_poll = tokio::time::Instant::now(); } } fn retry_is_pending(retries: &BTreeMap, folder_id: FolderId) -> bool { retries .get(&folder_id) .is_some_and(|retry| retry.retry_at > tokio::time::Instant::now()) } fn schedule_retry(retries: &mut BTreeMap, folder_id: FolderId) -> Duration { let retry = retries.entry(folder_id).or_insert(RetryState { failure_count: 0, retry_at: tokio::time::Instant::now(), }); retry.failure_count += 1; let delay = retry_delay(retry.failure_count); retry.retry_at = tokio::time::Instant::now() + delay; delay } pub(super) fn retry_delay(failure_count: u32) -> Duration { let exponent = failure_count.saturating_sub(1).min(MAX_RETRY_EXPONENT); RETRY_BASE_DELAY .checked_mul(1_u32 << exponent) .unwrap_or(MAX_RETRY_DELAY) .min(MAX_RETRY_DELAY) } #[cfg(test)] mod tests { use tempfile::TempDir; use uuid::Uuid; use super::{FolderWatcher, refresh_watcher}; use crate::{domain::FolderMode, storage::FolderConfig}; #[test] fn adds_a_watcher_for_a_new_folder() -> anyhow::Result<()> { let directory = TempDir::new()?; let folder = folder_config(directory.path()); let mut watchers = Vec::new(); refresh_watcher(&mut watchers, folder.clone())?; assert_eq!(watchers.len(), 1); assert_eq!(watchers[0].folder.id, folder.id); Ok(()) } #[test] fn replaces_a_watcher_when_folder_settings_change() -> anyhow::Result<()> { let directory = TempDir::new()?; let folder = folder_config(directory.path()); let mut watchers = vec![FolderWatcher::new(folder.clone())?]; let mut changed_folder = folder; changed_folder.capability = "rotated-capability".to_owned(); refresh_watcher(&mut watchers, changed_folder.clone())?; assert_eq!(watchers.len(), 1); assert_eq!(watchers[0].folder.capability, changed_folder.capability); Ok(()) } fn folder_config(path: &std::path::Path) -> FolderConfig { FolderConfig { id: Uuid::new_v4(), name: "test".to_owned(), path: path.to_owned(), capability: "capability".to_owned(), mode: FolderMode::SendReceive, } } }