diff --git a/Cargo.lock b/Cargo.lock index c65db43a..77a1fc83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -462,6 +462,7 @@ dependencies = [ "jsonc-parser", "libc", "libloading", + "notify", "ort", "predicates", "proptest", @@ -875,6 +876,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1277,6 +1287,26 @@ dependencies = [ "web-time", ] +[[package]] +name = "inotify" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "153be1941a183ec9ccd095ddbe17a8b8d435ef6c76e9e02451b933c3999af2c8" +dependencies = [ + "bitflags", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -1335,6 +1365,26 @@ dependencies = [ "serde_json", ] +[[package]] +name = "kqueue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "libc" version = "0.2.186" @@ -1467,6 +1517,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -1524,6 +1575,33 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2269,6 +2347,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" version = "1.0.228" @@ -2933,6 +3020,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -3079,6 +3176,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -3109,6 +3215,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -3142,13 +3257,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3161,6 +3293,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3173,6 +3311,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3185,12 +3329,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3203,6 +3359,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3215,6 +3377,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3227,6 +3395,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3239,6 +3413,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.15" diff --git a/Cargo.toml b/Cargo.toml index 24be46ec..a5dc830b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ clap_mangen = "0.2" directories = "5.0" ed25519-dalek = "2.1" libc = "0.2" +notify = "8.0" predicates = "3.1" proptest = "1" regex = "1.10" diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index 1294d825..36e4e08d 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -22,6 +22,7 @@ chrono.workspace = true clap.workspace = true clap_mangen.workspace = true jsonc-parser = { version = "0.33", features = ["serde", "serde_json"] } +notify.workspace = true ring.workspace = true rusqlite.workspace = true serde.workspace = true diff --git a/crates/ctx-cli/src/commands/import.rs b/crates/ctx-cli/src/commands/import.rs index ecf9d047..9519e6cc 100644 --- a/crates/ctx-cli/src/commands/import.rs +++ b/crates/ctx-cli/src/commands/import.rs @@ -1,9 +1,8 @@ use std::{ + collections::BTreeMap, fs, io::{Cursor, Read}, path::{Path, PathBuf}, - sync::{Arc, Mutex}, - thread, time::{SystemTime, UNIX_EPOCH}, }; @@ -12,44 +11,54 @@ use serde_json::{json, Value}; use uuid::Uuid; use ctx_history_capture::{ - catalog_codex_session_tree, import_antigravity_cli_history, import_astrbot_sqlite, - import_auggie_history, import_claude_projects_jsonl_tree, import_cline_task_json_history, - import_codebuddy_history, import_codex_history_jsonl, import_codex_session_jsonl, - import_codex_session_jsonl_tail, import_codex_session_paths, import_continue_cli_sessions, - import_copilot_cli_session_events, import_crush_sqlite, import_cursor_native_history, - import_custom_history_jsonl_v1, import_custom_history_jsonl_v1_reader, - import_deepagents_sqlite, import_factory_ai_droid_sessions, import_firebender_sqlite, - import_forgecode_sqlite, import_gemini_cli_history, import_goose_sessions_sqlite, - import_hermes_sqlite, import_junie_history, import_kilo_sqlite, import_kimi_code_cli_history, - import_kiro_sqlite, import_lingma_sqlite, import_mimocode_sqlite, import_mistral_vibe_history, - import_mux_history, import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, + catalog_codex_session_tree, import_antigravity_cli_history, + import_append_capable_provider_file, import_astrbot_sqlite, import_auggie_history, + import_claude_projects_jsonl_tree, import_cline_task_json_history, import_codebuddy_history, + import_codex_history_jsonl, import_codex_session_jsonl, import_codex_session_jsonl_tail, + import_codex_session_paths, import_continue_cli_sessions, import_copilot_cli_session_events, + import_crush_sqlite, import_cursor_native_history, import_custom_history_jsonl_v1, + import_custom_history_jsonl_v1_reader, import_deepagents_sqlite, + import_factory_ai_droid_sessions, import_firebender_sqlite, import_forgecode_sqlite, + import_gemini_cli_history, import_goose_sessions_sqlite, import_hermes_sqlite, + import_junie_history, import_kilo_sqlite, import_kimi_code_cli_history, import_kiro_sqlite, + import_lingma_sqlite, import_mimocode_sqlite, import_mistral_vibe_history, import_mux_history, + import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, import_openhands_file_events, import_pi_session_jsonl, import_qoder_history, import_qwen_code_history, import_roo_task_json_history, import_rovodev_history, import_shelley_sqlite, import_tabnine_cli_history, import_trae_history, import_warp_sqlite, - import_windsurf_cascade_hook_transcripts, import_zed_threads_sqlite, provider_source_spec, - stable_capture_uuid, AntigravityCliImportOptions, AstrBotSqliteImportOptions, - AuggieImportOptions, CaptureError, CatalogSummary, ClaudeProjectsImportOptions, - ClineTaskJsonImportOptions, CodeBuddyImportOptions, CodexHistoryImportOptions, - CodexSessionCatalogOptions, CodexSessionImportOptions, CodexSessionImportProgressCallback, - ContinueCliImportOptions, CopilotCliImportOptions, CrushSqliteImportOptions, - CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions, DeepAgentsSqliteImportOptions, - FactoryAiDroidImportOptions, FirebenderSqliteImportOptions, ForgeCodeSqliteImportOptions, - GeminiCliImportOptions, GooseSessionsSqliteImportOptions, HermesSqliteImportOptions, - JunieImportOptions, KiloSqliteImportOptions, KimiCodeCliImportOptions, KiroSqliteImportOptions, - LingmaSqliteImportOptions, MiMoCodeSqliteImportOptions, MistralVibeImportOptions, - MuxImportOptions, NanoClawImportOptions, OpenClawImportOptions, OpenCodeSqliteImportOptions, - OpenHandsImportOptions, PiSessionImportOptions, ProviderImportFailure, ProviderImportSummary, - ProviderImportSupport, ProviderSourceStatus, QoderImportOptions, QwenCodeImportOptions, - RooTaskJsonImportOptions, RovoDevImportOptions, ShelleySqliteImportOptions, - TabnineCliImportOptions, TraeImportOptions, WarpSqliteImportOptions, - WindsurfCascadeHookImportOptions, ZedThreadsSqliteImportOptions, + import_windsurf_cascade_hook_transcripts, import_zed_threads_sqlite, + provider_canonical_material_source_format, provider_file_mutation_contract, + provider_source_spec, stable_capture_uuid, AntigravityCliImportOptions, + AstrBotSqliteImportOptions, AuggieImportOptions, CaptureError, CatalogSummary, + ClaudeProjectsImportOptions, ClineTaskJsonImportOptions, CodeBuddyImportOptions, + CodexHistoryImportOptions, CodexSessionCatalogOptions, CodexSessionImportOptions, + CodexSessionImportProgressCallback, ContinueCliImportOptions, CopilotCliImportOptions, + CrushSqliteImportOptions, CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions, + DeepAgentsSqliteImportOptions, FactoryAiDroidImportOptions, FirebenderSqliteImportOptions, + ForgeCodeSqliteImportOptions, GeminiCliImportOptions, GooseSessionsSqliteImportOptions, + HermesSqliteImportOptions, JunieImportOptions, KiloSqliteImportOptions, + KimiCodeCliImportOptions, KiroSqliteImportOptions, LingmaSqliteImportOptions, + MiMoCodeSqliteImportOptions, MistralVibeImportOptions, MuxImportOptions, NanoClawImportOptions, + OpenClawImportOptions, OpenCodeSqliteImportOptions, OpenHandsImportOptions, + PiSessionImportOptions, ProviderAdmittedJsonlAppendCheckpoint, + ProviderAppendFileImportDecision, ProviderAppendFileImportMode, + ProviderAppendFileImportOptions, ProviderFileMutationContract, ProviderFileStableIdentity, + ProviderImportFailure, ProviderImportSummary, ProviderImportSupport, + ProviderJsonlAppendCheckpoint, ProviderJsonlResumeState, ProviderSourceStatus, + QoderImportOptions, QwenCodeImportOptions, RooTaskJsonImportOptions, RovoDevImportOptions, + ShelleySqliteImportOptions, TabnineCliImportOptions, TraeImportOptions, + WarpSqliteImportOptions, WindsurfCascadeHookImportOptions, ZedThreadsSqliteImportOptions, }; use ctx_history_core::{ database_path, utc_now, CaptureProvider, CtxHistoryJsonlRecord, HistoryRecord, }; use ctx_history_store::{ - CatalogIndexedStatus, CatalogSession, CatalogSourceIndexUpdate, SourceImportFile, - SourceImportFileIndexUpdate, Store, StoreError, + CatalogImportWork, CatalogIndexedStatus, CatalogSession, CatalogSourceIndexUpdate, + ImportPendingReason, ImportWorkClass, ProviderFileCheckpoint, ProviderFileCheckpointKey, + ProviderFileImportOutcome, ProviderFileInventoryObservation, ProviderFilePublicationCommit, + ProviderFilePublicationCompletion, ProviderFilePublicationKind, ProviderFilePublicationPhase, + ProviderFilePublicationRetirementWork, SourceImportFile, SourceImportFileIndexUpdate, + SourceImportFileWork, Store, StoreError, }; use crate::analytics::AnalyticsProperties; @@ -58,9 +67,7 @@ use crate::history_source_plugins::{ HistorySourcePluginSource, }; use crate::output::print_json; -use crate::progress::{ - format_bytes, format_count, plural, ProgressArg, ProgressReporter, SourceProgressSnapshot, -}; +use crate::progress::{format_bytes, format_count, plural, ProgressArg, ProgressReporter}; use crate::provider_args::ImportFormatArg; use crate::provider_sources::{ discovered_sources, discovered_sources_for_provider, explicit_path_source, import_support_json, @@ -78,21 +85,22 @@ mod manifest; mod native; mod report; mod requests; +mod scheduler; #[cfg(test)] pub(crate) use catalog::{catalog_import_checkpoint_matches, sha256_file_prefix_hex}; use catalog::{ import_incremental_codex_session_tree, import_record_for_custom_history, import_record_for_history_source_plugin, import_record_for_source, source_stats, - source_uses_incremental_event_search, }; use explicit::run_explicit_format_import; pub(crate) use inventory::{ inventory_available_sources, inventory_import_sources, ImportInventory, }; -use native::{import_one_source, validate_source_import_supported}; +use native::validate_source_import_supported; pub(crate) use native::{ - import_one_source_for_search_refresh, import_one_source_without_search_refresh, + import_selected_source, publication_recovery_maintenance_warning, + recover_provider_file_publication_retirement, }; use report::{ custom_format_failure_json, custom_format_import_json, history_source_plugin_failure_json, @@ -107,9 +115,13 @@ pub(crate) use report::{ pub(crate) use report::{ImportFailureScope, ImportFailureType, ImportRetryability}; pub(crate) use requests::import_history_source_plugin; use requests::{history_source_plugin_import_requests, import_requests, validate_import_args}; +use scheduler::{bounded_unplanned_root_work_counts, SelectedImportWork}; +pub(crate) use scheduler::{ + ExecutableImportSlice, ImportExecutionPolicy, ImportExecutionResult, ImportExecutionState, + ImportPlan, IMPORT_PENDING_REPORT_LIMIT, +}; include!("import/state.rs"); - pub(crate) fn run_import( args: ImportArgs, data_root: PathBuf, @@ -220,6 +232,8 @@ pub(crate) fn run_import_internal( analytics_properties: &mut AnalyticsProperties, options: ImportRunOptions, ) -> Result { + let _disk_io_pacing = + ctx_history_capture::install_disk_io_pacer(ImportExecutionPolicy::Drain.disk_io_pacer()); validate_import_args(args)?; fs::create_dir_all(&data_root).map_err(|source| CaptureError::SystemIo { operation: "initialize ctx data root", @@ -247,9 +261,15 @@ pub(crate) fn run_import_internal( &data_root, options.include_history_source_plugins, )?; - if requests.is_empty() && plugin_requests.is_empty() { + let has_publication_work = has_provider_file_publication_work(&store)?; + if requests.is_empty() && plugin_requests.is_empty() && !has_publication_work { + let maintenance = repair_import_maintenance(&store, ImportExecutionPolicy::Drain)?; + totals.durable_progress = maintenance.processed_rows > 0; + totals.recovery_units_pending = usize::from(!maintenance.complete); if options.allow_empty_sources { - return Ok(ImportReport::empty(args.resume)); + let mut report = ImportReport::empty(args.resume); + report.totals = totals; + return Ok(report); } return Err(anyhow!( "no importable provider history sources found; use --path, --history-source, or run `ctx sources`" @@ -261,21 +281,23 @@ pub(crate) fn run_import_internal( inventory_progress.message("inventorying", "Preparing local history..."); let inventory = inventory_import_sources(&store, requests, args.resume) .context("inventory local history sources")?; - let planned_sources = inventory.sources; + let plan = ImportPlan::build(&store, inventory.sources)?; + let mut execution_state = ImportExecutionState::for_plan(&plan); let inventory_failures = inventory.failures; + let failed_inventory_pending = failed_inventory_pending_counts(&store, &inventory_failures)?; let planned_total_bytes = inventory.totals.source_bytes; inventory_progress.done( "inventorying", format!( "Found {} history {} ({}).", format_count( - planned_sources + plan.sources .len() .saturating_add(inventory_failures.len()) .saturating_add(plugin_requests.len()), ), plural( - planned_sources + plan.sources .len() .saturating_add(inventory_failures.len()) .saturating_add(plugin_requests.len()), @@ -289,7 +311,7 @@ pub(crate) fn run_import_internal( analytics::insert_count_bucket( analytics_properties, "sources_seen_bucket", - planned_sources + plan.sources .len() .saturating_add(inventory_failures.len()) .saturating_add(plugin_requests.len()) as u64, @@ -309,7 +331,7 @@ pub(crate) fn run_import_internal( if let Some(warning) = low_disk_space_warning(&db_path, planned_total_bytes) { progress.warning(warning); } - if let Some(notice) = large_import_notice(&planned_sources, planned_total_bytes) { + if let Some(notice) = large_import_notice(&plan.sources, planned_total_bytes) { progress.notice(notice); } @@ -331,6 +353,30 @@ pub(crate) fn run_import_internal( imported_sources.push(source_failure_json(&failure)); } + let native_import_requested = !plan.sources.is_empty() || plan.recovery_units > 0; + let mut native_reports = NativeSourceReports::default(); + loop { + execution_state.begin_new_pass(); + let fresh_units = plan.pending_count(&store, ImportWorkClass::Fresh)?; + if fresh_units == 0 { + break; + } + let result = execute_import_plan_class_for_report( + &mut store, + &plan, + &mut execution_state, + ImportWorkClass::Fresh, + fresh_units, + &progress, + options, + &mut totals, + &mut native_reports, + )?; + if !result.made_durable_progress() { + break; + } + } + for plugin_source in plugin_requests { if options.print_human { progress.finish_line(); @@ -404,272 +450,72 @@ pub(crate) fn run_import_internal( } } - let native_import_requested = !planned_sources.is_empty(); - if should_parallelize_import(&planned_sources) { - let final_refresh_required = store.event_search_projection_needs_backfill()? - || planned_sources - .iter() - .any(|plan| !source_uses_incremental_event_search(&plan.source)); - drop(store); - - if options.print_human { - progress.finish_line(); - println!("sources:"); - for plan in &planned_sources { - println!( - " {} {} ({} files, {})", - plan.source.provider.as_str(), - plan.source.path.display(), - plan.stats.files, - format_bytes(plan.stats.bytes) - ); + let mut maintenance_complete = loop { + execution_state.begin_new_pass(); + let maintenance = repair_import_maintenance(&store, ImportExecutionPolicy::Drain)?; + totals.durable_progress |= maintenance.processed_rows > 0; + let recovery_units = plan.pending_count(&store, ImportWorkClass::Recovery)?; + let result = execute_import_plan_class_for_report( + &mut store, + &plan, + &mut execution_state, + ImportWorkClass::Recovery, + recovery_units, + &progress, + options, + &mut totals, + &mut native_reports, + )?; + let fresh_units = plan.pending_count(&store, ImportWorkClass::Fresh)?; + if fresh_units > 0 { + execution_state.begin_new_pass(); + let fresh_result = execute_import_plan_class_for_report( + &mut store, + &plan, + &mut execution_state, + ImportWorkClass::Fresh, + fresh_units, + &progress, + options, + &mut totals, + &mut native_reports, + )?; + if fresh_result.made_durable_progress() { + continue; } - } - - let source_states = Arc::new(Mutex::new( - planned_sources - .iter() - .map(|plan| SourceProgressSnapshot { - completed_bytes: 0, - total_bytes: plan.stats.bytes, - }) - .collect::>(), - )); - let handles = planned_sources - .into_iter() - .enumerate() - .map(|(index, plan)| { - let db_path = db_path.clone(); - let progress_callback = progress.parallel_codex_import_callback( - &plan.source, - index, - Arc::clone(&source_states), - ); - let full_rescan = args.resume; - let join_source = plan.source.clone(); - let join_stats = plan.stats; - let failure_source = plan.source.clone(); - let handle = thread::spawn(move || -> ImportSourceRun { - let result = (|| -> Result { - let mut store = Store::open(&db_path)?; - import_one_source_without_search_refresh( - &mut store, - &plan.source, - progress_callback, - full_rescan, - &plan.preinventory, - ) - .with_context(|| { - format!( - "import {} source {}", - plan.source.provider.as_str(), - plan.source.path.display() - ) - }) - })(); - match result { - Ok(summary) => ImportSourceRun::Imported(ImportSourceOutcome { - index, - source: plan.source, - stats: plan.stats, - summary, - }), - Err(err) => { - let failure_scope = import_error_scope(&err); - let failure_type = import_failure_type(&err); - let rejected_summary = rejected_source_summary(&err); - let error = error_summary(&err); - let system_error = - (failure_scope == ImportFailureScope::System).then_some(err); - ImportSourceRun::Failed(ImportSourceFailure { - index, - source: failure_source, - stats: join_stats, - error, - failure_scope, - failure_type, - rejected_summary, - system_error, - }) - } - } - }); - (index, join_source, join_stats, handle) - }) - .collect::>(); - - let mut runs = Vec::with_capacity(handles.len()); - let mut first_error = None; - for (index, source, stats, handle) in handles { - match handle.join() { - Ok(ImportSourceRun::Imported(outcome)) => { - runs.push(ImportSourceRun::Imported(outcome)) - } - Ok(ImportSourceRun::Failed(mut failure)) => { - if failure.failure_scope == ImportFailureScope::System { - first_error.get_or_insert_with(|| { - failure.system_error.take().unwrap_or_else(|| { - anyhow!( - "import {} source {}: {}", - failure.source.provider.as_str(), - failure.source.path.display(), - failure.error - ) - }) - }); - } - runs.push(ImportSourceRun::Failed(failure)); - } - Err(_) => { - let panic_error = - anyhow::Error::new(CaptureError::WorkerPanicked("provider import")); - let failure = ImportSourceFailure { - index, - source, - stats, - error: error_summary(&panic_error), - failure_scope: ImportFailureScope::System, - failure_type: ImportFailureType::WorkerPanic, - rejected_summary: None, - system_error: Some(panic_error), - }; - first_error.get_or_insert_with(|| { - anyhow::Error::new(CaptureError::WorkerPanicked("provider import")) - }); - runs.push(ImportSourceRun::Failed(failure)); - } + let recovery_units_pending = plan.pending_count(&store, ImportWorkClass::Recovery)?; + if recovery_units_pending > 0 + && (maintenance.processed_rows > 0 || result.made_durable_progress()) + { + continue; } + break maintenance.complete; } - if let Some(err) = first_error { - return Err(err); + let (_, recovery_units_pending) = plan.pending_counts(&store)?; + if maintenance.complete && recovery_units_pending == 0 { + break true; } - - runs.sort_by_key(ImportSourceRun::index); - for run in runs { - match run { - ImportSourceRun::Imported(outcome) => { - totals.add(&outcome.summary, &outcome.stats); - progress.parallel_source_done( - &outcome.source, - outcome.index, - &source_states, - outcome.stats, - &outcome.summary, - ); - if options.print_human { - progress.finish_line(); - print_source_imported(&outcome.source, &outcome.summary); - } - imported_sources.push(source_import_json( - &outcome.source, - &outcome.stats, - &outcome.summary, - )); - } - ImportSourceRun::Failed(failure) => { - if let Some(summary) = failure.rejected_summary.as_ref() { - totals.add_rejected_source(summary, &failure.stats); - } else { - totals.add_source_failure(&failure.stats); - } - progress.parallel_source_failed( - &failure.source, - failure.index, - &source_states, - failure.stats, - &failure.error, - ); - if options.print_human { - progress.finish_line(); - print_source_failed(&failure); - } - imported_sources.push(source_failure_json(&failure)); - } - } + if maintenance.processed_rows == 0 && !result.made_durable_progress() { + break maintenance.complete; } + }; + let trailing_maintenance = repair_import_maintenance(&store, ImportExecutionPolicy::Drain)?; + totals.durable_progress |= trailing_maintenance.processed_rows > 0; + maintenance_complete &= trailing_maintenance.complete; + + let (fresh_units_pending, recovery_units_pending) = plan.pending_counts(&store)?; + totals.fresh_units_pending = + capped_pending_add(fresh_units_pending, failed_inventory_pending.0); + totals.recovery_units_pending = capped_pending_add( + capped_pending_add(recovery_units_pending, failed_inventory_pending.1), + usize::from(!maintenance_complete), + ); + native_reports.apply_totals(&mut totals); + native_reports.append_json(&plan, &mut imported_sources); - if final_refresh_required { - progress.message("finalizing", "Refreshing search index..."); - let store = Store::open(&db_path)?; - store.refresh_search_index()?; - } - } else { - let mut completed_source_bytes = 0u64; - for plan in planned_sources { - if options.print_human { - progress.finish_line(); - println!( - "importing {} {} ({} files, {})", - plan.source.provider.as_str(), - plan.source.path.display(), - plan.stats.files, - format_bytes(plan.stats.bytes) - ); - } - let source_progress = - progress.codex_import_callback(&plan.source, completed_source_bytes); - completed_source_bytes = completed_source_bytes.saturating_add(plan.stats.bytes); - match import_one_source( - &mut store, - &plan.source, - source_progress, - args.resume, - &plan.preinventory, - ) { - Ok(summary) => { - totals.add(&summary, &plan.stats); - progress.done( - "indexing", - format!("Indexed {}.", source_provider_label(&plan.source)), - completed_source_bytes, - ); - if options.print_human { - progress.finish_line(); - print_source_imported(&plan.source, &summary); - } - imported_sources.push(source_import_json(&plan.source, &plan.stats, &summary)); - } - Err(err) => { - let failure_scope = import_error_scope(&err); - let failure_type = import_failure_type(&err); - let rejected_summary = rejected_source_summary(&err); - let error = error_summary(&err); - if failure_scope == ImportFailureScope::Source { - let failure = ImportSourceFailure { - index: imported_sources.len(), - source: plan.source, - stats: plan.stats, - error, - failure_scope, - failure_type, - rejected_summary, - system_error: None, - }; - if let Some(summary) = failure.rejected_summary.as_ref() { - totals.add_rejected_source(summary, &failure.stats); - } else { - totals.add_source_failure(&failure.stats); - } - progress.done( - "indexing", - format!( - "skipped {}: {}", - failure.source.provider.as_str(), - source_error_reason(&failure.source, &failure.error) - ), - completed_source_bytes, - ); - if options.print_human { - progress.finish_line(); - print_source_failed(&failure); - } - imported_sources.push(source_failure_json(&failure)); - } else { - return Err(err); - } - } - } - } + if store.event_search_projection_needs_backfill()? { + progress.message("finalizing", "Refreshing search index..."); + store.refresh_search_index()?; } if totals.imported_sessions > 0 || totals.imported_events > 0 || totals.imported_edges > 0 { @@ -737,50 +583,316 @@ pub(crate) fn run_import_internal( }) } +fn has_provider_file_publication_work(store: &Store) -> Result { + Ok(store.has_pending_provider_file_publications()? + || !store + .list_provider_file_publication_retirement_work(1)? + .is_empty()) +} + +#[allow(clippy::too_many_arguments)] +fn execute_import_plan_class_for_report( + store: &mut Store, + plan: &ImportPlan, + execution_state: &mut ImportExecutionState, + class: ImportWorkClass, + remaining_units: usize, + progress: &ProgressReporter, + options: ImportRunOptions, + totals: &mut ImportTotals, + native_reports: &mut NativeSourceReports, +) -> Result { + execute_import_plan_class_for_report_with_pre_lock_hook( + store, + plan, + execution_state, + class, + remaining_units, + progress, + options, + totals, + native_reports, + || {}, + ) +} + +#[allow(clippy::too_many_arguments)] +fn execute_import_plan_class_for_report_with_pre_lock_hook( + store: &mut Store, + plan: &ImportPlan, + execution_state: &mut ImportExecutionState, + class: ImportWorkClass, + mut remaining_units: usize, + progress: &ProgressReporter, + options: ImportRunOptions, + totals: &mut ImportTotals, + native_reports: &mut NativeSourceReports, + mut before_bulk_lock: impl FnMut(), +) -> Result { + let mut completed_bytes = 0u64; + let mut execution_result = ImportExecutionResult::default(); + while remaining_units > 0 { + let Some(executable) = plan.select_slice_for_execution_with_pre_lock_hook( + store, + class, + remaining_units, + execution_state, + &mut before_bulk_lock, + )? + else { + break; + }; + let ExecutableImportSlice { + slice, + bulk_guard, + validation_failures, + } = executable; + if slice.is_empty() && validation_failures.is_empty() { + store.finish_event_search_bulk_mode(&bulk_guard)?; + break; + } + let validation_units = validation_failures + .iter() + .map(|failure| failure.stats.files) + .sum::(); + let selected_units = slice.units.saturating_add(validation_units); + remaining_units = remaining_units.saturating_sub(selected_units); + let mut system_error = None; + let mut completed_units = 0usize; + let mut deferred_units = 0usize; + let mut maintenance_progress = false; + let mut source_durable_progress = false; + for validation_failure in validation_failures { + let source_plan = &plan.sources[validation_failure.source_index]; + native_reports.record_failure( + validation_failure.source_index, + validation_failure.stats, + &validation_failure.error, + ); + let failure = ImportSourceFailure { + source: source_plan.source.clone(), + stats: validation_failure.stats, + error: error_summary(&validation_failure.error), + failure_type: import_failure_type(&validation_failure.error), + rejected_summary: rejected_source_summary(&validation_failure.error), + }; + progress.done( + "indexing", + format!( + "skipped {}: {}", + failure.source.provider.as_str(), + source_error_reason(&failure.source, &failure.error) + ), + completed_bytes, + ); + if options.print_human { + progress.finish_line(); + print_source_failed(&failure); + } + } + for retirement in &slice.retirements { + execution_state.record_retirement_attempt(retirement); + progress.message("repairing", "repairing prior hidden provider history"); + match recover_provider_file_publication_retirement(store, retirement, true) { + Ok(outcome) => { + maintenance_progress |= outcome.made_durable_progress; + if outcome.completed { + completed_units = completed_units.saturating_add(1); + } + for warning in outcome.maintenance_warnings { + progress.warning(warning.to_string()); + } + } + Err(error) => { + system_error = Some(error); + break; + } + } + } + for selected in slice.sources { + if system_error.is_some() { + break; + } + let source_plan = &plan.sources[selected.source_index]; + let (phase, message) = import_work_progress_message(class, source_plan.source.provider); + progress.message(phase, message); + let source_progress = + progress.codex_import_callback(&source_plan.source, completed_bytes); + execution_state.record_source_attempt(&selected.work); + if let Err(error) = selected.persist_attempt_started(store) { + system_error = Some(error); + break; + } + let import_result = import_selected_source( + store, + &source_plan.source, + source_progress, + &selected.preinventory, + &selected.work, + ); + let (outcome, import_error) = match import_result { + Ok(result) => (Some(result.outcome), result.remaining_error), + Err(error) => (None, Some(error)), + }; + let outcome_has_error = import_error.is_some(); + let mut outcome_completed_units = 0usize; + let mut outcome_completed_bytes = 0u64; + let mut outcome_deferred_units = 0usize; + let had_outcome = outcome.is_some(); + if let Some(outcome) = outcome { + let made_durable_progress = outcome.made_durable_progress(); + execution_state.record_source_outcome( + selected.source_index, + &selected.work, + outcome.post_import_preinventory.clone(), + ); + source_durable_progress |= made_durable_progress; + outcome_completed_units = outcome.completed_units; + outcome_completed_bytes = outcome.completed_bytes; + outcome_deferred_units = outcome.deferred_units; + completed_units = completed_units.saturating_add(outcome.completed_units); + let deferred = outcome.deferred_units; + deferred_units = deferred_units.saturating_add(deferred); + let completed_stats = SourceStats { + files: outcome.completed_units, + bytes: outcome.completed_bytes, + change_token: selected.stats.change_token, + }; + let no_op_stats = (outcome.completed_units == 0 + && deferred == 0 + && !outcome_has_error + && outcome.summary != ProviderImportSummary::default()) + .then_some(selected.stats); + native_reports.record_outcome( + selected.source_index, + &outcome.summary, + completed_stats, + no_op_stats, + ); + if outcome.completed_units > 0 { + completed_bytes = completed_bytes.saturating_add(completed_stats.bytes); + let (phase, message) = import_work_progress_done(class, &source_plan.source); + progress.done(phase, message, completed_bytes); + if options.print_human { + progress.finish_line(); + print_source_imported(&source_plan.source, &outcome.summary); + } + } else if deferred > 0 && !made_durable_progress { + progress.done( + phase, + format!( + "Deferred incomplete {} history.", + source_provider_label(&source_plan.source) + ), + completed_bytes, + ); + } + if deferred > 0 && !made_durable_progress { + progress.warning(format!( + "{} {} history unit(s) remain pending until their current write completes.", + deferred, + source_provider_label(&source_plan.source) + )); + } + } + if let Some(err) = import_error { + if !had_outcome { + execution_state.record_source_outcome( + selected.source_index, + &selected.work, + None, + ); + } + if let Some(warning) = native::publication_recovery_maintenance_warning(&err) { + progress.warning(warning.to_string()); + } + let failure_scope = import_error_scope(&err); + let failure_type = import_failure_type(&err); + let rejected_summary = rejected_source_summary(&err); + let error = error_summary(&err); + if failure_scope == ImportFailureScope::System { + system_error = Some(err); + break; + } + let failure_stats = SourceStats { + files: selected.stats.files.saturating_sub( + outcome_completed_units.saturating_add(outcome_deferred_units), + ), + bytes: selected.stats.bytes.saturating_sub(outcome_completed_bytes), + change_token: selected.stats.change_token, + }; + native_reports.record_failure(selected.source_index, failure_stats, &err); + let failure = ImportSourceFailure { + source: source_plan.source.clone(), + stats: failure_stats, + error, + failure_type, + rejected_summary, + }; + progress.done( + "indexing", + format!( + "skipped {}: {}", + failure.source.provider.as_str(), + source_error_reason(&failure.source, &failure.error) + ), + completed_bytes, + ); + if options.print_human { + progress.finish_line(); + print_source_failed(&failure); + } + } + if outcome_deferred_units > 0 && store.has_pending_provider_file_publications()? { + break; + } + } + let finish_result = store.finish_event_search_bulk_mode(&bulk_guard); + if let Err(error) = finish_result { + return Err(error.into()); + } + match class { + ImportWorkClass::Fresh => { + totals.fresh_units_processed = + totals.fresh_units_processed.saturating_add(completed_units); + } + ImportWorkClass::Recovery => { + totals.recovery_units_processed = totals + .recovery_units_processed + .saturating_add(completed_units); + } + } + execution_result.add_slice( + selected_units, + completed_units, + deferred_units, + maintenance_progress || source_durable_progress, + ); + totals.durable_progress |= + completed_units > 0 || maintenance_progress || source_durable_progress; + if let Some(error) = system_error { + return Err(error); + } + if deferred_units > 0 && store.has_pending_provider_file_publications()? { + break; + } + } + Ok(execution_result) +} + fn source_provider_label(source: &SourceInfo) -> &'static str { provider_source_spec(source.provider) .map(|spec| spec.display_name) .unwrap_or_else(|| source.provider.as_str()) } -#[derive(Debug)] -pub(crate) struct ImportSourceOutcome { - pub(crate) index: usize, - pub(crate) source: SourceInfo, - pub(crate) stats: SourceStats, - pub(crate) summary: ProviderImportSummary, -} - -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct ImportSourceFailure { - pub(crate) index: usize, pub(crate) source: SourceInfo, pub(crate) stats: SourceStats, pub(crate) error: String, - pub(crate) failure_scope: ImportFailureScope, pub(crate) failure_type: ImportFailureType, pub(crate) rejected_summary: Option, - pub(crate) system_error: Option, -} - -#[derive(Debug)] -enum ImportSourceRun { - Imported(ImportSourceOutcome), - Failed(ImportSourceFailure), -} - -impl ImportSourceRun { - pub(crate) fn index(&self) -> usize { - match self { - Self::Imported(outcome) => outcome.index, - Self::Failed(failure) => failure.index, - } - } -} - -pub(crate) fn should_parallelize_import(planned_sources: &[PlannedImportSource]) -> bool { - let _ = planned_sources; - false } pub(crate) fn large_import_notice( diff --git a/crates/ctx-cli/src/commands/import/catalog.rs b/crates/ctx-cli/src/commands/import/catalog.rs index 76876fcc..2ad0b8a7 100644 --- a/crates/ctx-cli/src/commands/import/catalog.rs +++ b/crates/ctx-cli/src/commands/import/catalog.rs @@ -1,6 +1,8 @@ use super::*; -use crate::commands::import::manifest::collect_source_import_paths; -use ctx_history_capture::{observe_sqlite_source_generation, SqliteObservedFile}; +use ctx_history_capture::{ + observe_ordinary_file, observe_sqlite_source_generation, OrdinaryFileObservation, + SqliteObservedFile, +}; use sha2::{Digest, Sha256}; pub(crate) fn system_time_ms(time: SystemTime) -> i64 { @@ -9,16 +11,70 @@ pub(crate) fn system_time_ms(time: SystemTime) -> i64 { .unwrap_or(0) } +fn catalog_batch_outcome( + summary: &ProviderImportSummary, + completed_units: usize, + completed_bytes: u64, + deferred_units: usize, +) -> super::native::ProviderImportBatchOutcome { + super::native::ProviderImportBatchOutcome { + summary: summary.clone(), + completed_units, + completed_bytes, + deferred_units, + durable_progress: false, + post_import_inventory_generation: None, + post_import_preinventory: None, + } +} + +fn catalog_batch_error_with_progress( + summary: &ProviderImportSummary, + completed_units: usize, + completed_bytes: u64, + deferred_units: usize, + durable_progress: bool, + error: anyhow::Error, +) -> anyhow::Error { + super::native::provider_import_batch_error( + super::native::ProviderImportBatchOutcome { + summary: summary.clone(), + completed_units, + completed_bytes, + deferred_units, + durable_progress, + post_import_inventory_generation: None, + post_import_preinventory: None, + }, + error, + ) +} + +fn catalog_batch_error( + summary: &ProviderImportSummary, + completed_units: usize, + completed_bytes: u64, + deferred_units: usize, + error: anyhow::Error, +) -> anyhow::Error { + super::native::provider_import_batch_error( + catalog_batch_outcome(summary, completed_units, completed_bytes, deferred_units), + error, + ) +} + #[allow(clippy::too_many_arguments)] pub(crate) fn import_incremental_codex_session_tree( store: &mut Store, source: &SourceInfo, - record_id: Uuid, + record: &HistoryRecord, progress: Option, preinventory_catalog: Option<&CatalogSummary>, preinventory_generation: Option, force_selection: bool, -) -> Result { + selection: Option<&SelectedImportWork>, +) -> Result { + let record_id = record.id; let source_root = codex_catalog_root_identity(&source.path)?.to_owned(); let inventory_generation = match preinventory_generation { Some(generation) => generation, @@ -27,6 +83,10 @@ pub(crate) fn import_incremental_codex_session_tree( } }; let mut summary = ProviderImportSummary::default(); + let mut completed_units = 0; + let mut completed_bytes = 0_u64; + let mut deferred_units = 0; + let mut durable_progress = false; if let Some(catalog) = preinventory_catalog { summary.failed += catalog.failed_sessions; summary.failures.extend(catalog.failures.clone()); @@ -45,10 +105,118 @@ pub(crate) fn import_incremental_codex_session_tree( summary.failures.extend(catalog.failures); } - let selected_sessions = if force_selection { - store.list_active_catalog_sessions_for_source(CaptureProvider::Codex, &source_root)? - } else { - store.list_pending_catalog_sessions(CaptureProvider::Codex, &source_root)? + if let Some(SelectedImportWork::Catalog(work)) = selection { + for unit in work { + let outcome = match super::native::import_append_capable_catalog_work( + store, + source, + unit, + inventory_generation, + record, + ) { + Ok(outcome) => outcome, + Err(error) => { + if super::native::publication_recovery_required(&error) { + return Err(catalog_batch_error_with_progress( + &summary, + completed_units, + completed_bytes, + deferred_units, + durable_progress, + error, + )); + } + let rejected_summary = rejected_source_summary(&error); + let status = if rejected_summary.is_some() { + CatalogIndexedStatus::Rejected + } else { + catalog_import_error_status(&error) + }; + if let Err(persist_error) = store.record_observed_catalog_source_import_result( + unit.session.provider, + CatalogSourceIndexUpdate { + source_root: &unit.session.source_root, + source_path: &unit.session.source_path, + file_size_bytes: unit.session.file_size_bytes, + file_modified_at_ms: unit.session.file_modified_at_ms, + import_revision: unit.session.import_revision, + inventory_generation, + file_sha256: None, + event_count: None, + indexed_at_ms: utc_now().timestamp_millis(), + }, + &unit.session.metadata, + status, + Some(&error_summary(&error)), + ) { + return Err(catalog_batch_error_with_progress( + &summary, + completed_units, + completed_bytes, + deferred_units, + durable_progress, + persist_error.into(), + )); + } + if let Some(mut rejected_summary) = rejected_summary { + completed_units += 1; + completed_bytes = completed_bytes.saturating_add(unit.estimated_bytes); + for failure in &mut rejected_summary.failures { + failure.error = + format!("{}: {}", unit.session.source_path, failure.error); + } + summary.merge_from(rejected_summary); + continue; + } + return Err(catalog_batch_error_with_progress( + &summary, + completed_units, + completed_bytes, + deferred_units, + durable_progress, + error, + )); + } + }; + match outcome { + super::native::AppendImportOutcome::Imported(unit_summary) => { + summary.merge_from(unit_summary); + completed_units += 1; + completed_bytes = completed_bytes.saturating_add(unit.estimated_bytes); + } + super::native::AppendImportOutcome::Deferred { + durable_progress: unit_progress, + } => { + deferred_units += 1; + durable_progress |= unit_progress; + break; + } + } + } + return Ok(super::native::ProviderImportBatchOutcome { + summary, + completed_units, + completed_bytes, + deferred_units, + durable_progress, + post_import_inventory_generation: None, + post_import_preinventory: None, + }); + } + + let selected_sessions = match selection { + Some(SelectedImportWork::Catalog(work)) => { + work.iter().map(|work| work.session.clone()).collect() + } + Some(SelectedImportWork::SourceFiles(_)) => { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "source-file work selected for a catalog source", + ))) + } + None if force_selection => { + store.list_active_catalog_sessions_for_source(CaptureProvider::Codex, &source_root)? + } + None => store.list_pending_catalog_sessions(CaptureProvider::Codex, &source_root)?, }; if selected_sessions.is_empty() { if !store.catalog_inventory_generation_is_complete( @@ -58,7 +226,9 @@ pub(crate) fn import_incremental_codex_session_tree( )? { return Err(anyhow::Error::new(CaptureError::InventorySuperseded)); } - return Ok(summary); + return Ok(super::native::ProviderImportBatchOutcome::completed( + summary, 0, + )); } let mut full_import_sessions = Vec::new(); @@ -67,11 +237,22 @@ pub(crate) fn import_incremental_codex_session_tree( full_import_sessions.push(session.clone()); continue; } - let state = store.catalog_source_index_state( + let state = match store.catalog_source_index_state( CaptureProvider::Codex, &source_root, &session.source_path, - )?; + ) { + Ok(state) => state, + Err(error) => { + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + error.into(), + )) + } + }; let tail_start = state .as_ref() .and_then(|state| state.last_imported_file_size_bytes) @@ -88,14 +269,28 @@ pub(crate) fn import_incremental_codex_session_tree( Ok(matches) => matches, Err(err) => { let error = error_summary(&err); - mark_catalog_sessions_error( + if let Err(persist_error) = mark_catalog_sessions_error( store, std::slice::from_ref(session), &error, catalog_import_error_status(&err), inventory_generation, - )?; - return Err(err); + ) { + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + persist_error, + )); + } + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + err, + )); } }; if !checkpoint_matches { @@ -117,14 +312,28 @@ pub(crate) fn import_incremental_codex_session_tree( { Ok(summary) => summary, Err(err) => { - mark_catalog_sessions_error( + if let Err(persist_error) = mark_catalog_sessions_error( store, std::slice::from_ref(session), &err.to_string(), catalog_import_error_status(&err), inventory_generation, - )?; - return Err(err); + ) { + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + persist_error, + )); + } + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + err, + )); } }; let tail_event_count = tail_summary @@ -141,7 +350,7 @@ pub(crate) fn import_incremental_codex_session_tree( }; let error = (tail_summary.failed > 0).then(|| catalog_session_import_failure(&tail_summary)); - mark_catalog_session_result( + if let Err(persist_error) = mark_catalog_session_result( store, session, event_count, @@ -149,7 +358,19 @@ pub(crate) fn import_incremental_codex_session_tree( status, error.as_deref(), inventory_generation, - )?; + ) { + let mut partial_summary = summary.clone(); + partial_summary.merge_from(tail_summary); + return Err(catalog_batch_error( + &partial_summary, + completed_units, + completed_bytes, + deferred_units, + persist_error, + )); + } + completed_units += 1; + completed_bytes = completed_bytes.saturating_add(session.file_size_bytes); summary.merge_from(tail_summary); } else { full_import_sessions.push(session.clone()); @@ -175,16 +396,32 @@ pub(crate) fn import_incremental_codex_session_tree( Err(err) => { let failure_scope = import_error_scope(&err); let error = error_summary(&err); - mark_catalog_sessions_error( + if let Err(persist_error) = mark_catalog_sessions_error( store, std::slice::from_ref(session), &error, catalog_import_error_status(&err), inventory_generation, - )?; + ) { + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + persist_error, + )); + } if failure_scope == ImportFailureScope::System { - return Err(err); + return Err(catalog_batch_error( + &summary, + completed_units, + completed_bytes, + deferred_units, + err, + )); } + completed_units += 1; + completed_bytes = completed_bytes.saturating_add(session.file_size_bytes); summary.failed += 1; summary .failures @@ -192,16 +429,33 @@ pub(crate) fn import_incremental_codex_session_tree( continue; } }; - mark_catalog_sessions_result( + if let Err(persist_error) = mark_catalog_sessions_result( store, std::slice::from_ref(session), &file_summary, inventory_generation, - )?; + ) { + let mut partial_summary = summary.clone(); + partial_summary.merge_from(file_summary); + return Err(catalog_batch_error( + &partial_summary, + completed_units, + completed_bytes, + deferred_units, + persist_error, + )); + } + completed_units += 1; + completed_bytes = completed_bytes.saturating_add(session.file_size_bytes); summary.merge_from(file_summary); } } - Ok(summary) + Ok(catalog_batch_outcome( + &summary, + completed_units, + completed_bytes, + deferred_units, + )) } pub(crate) fn codex_catalog_root_identity(path: &Path) -> Result<&str> { @@ -284,7 +538,7 @@ pub(crate) fn mark_catalog_session_result( } else { None }; - let changed = store.record_catalog_source_import_result( + let changed = store.record_observed_catalog_source_import_result( session.provider, CatalogSourceIndexUpdate { source_root: &session.source_root, @@ -297,6 +551,7 @@ pub(crate) fn mark_catalog_session_result( event_count, indexed_at_ms, }, + &session.metadata, status, error, )?; @@ -347,7 +602,7 @@ pub(crate) fn mark_catalog_sessions_error( ) -> Result<()> { let indexed_at_ms = utc_now().timestamp_millis(); for session in sessions { - let changed = store.record_catalog_source_import_result( + let changed = store.record_observed_catalog_source_import_result( session.provider, CatalogSourceIndexUpdate { source_root: &session.source_root, @@ -360,6 +615,7 @@ pub(crate) fn mark_catalog_sessions_error( event_count: None, indexed_at_ms, }, + &session.metadata, status, Some(error), )?; @@ -377,6 +633,7 @@ fn catalog_import_error_status(error: &anyhow::Error) -> CatalogIndexedStatus { } } +#[cfg(test)] pub(crate) fn source_uses_incremental_event_search(source: &SourceInfo) -> bool { // Every importable provider persists events through Store APIs that update // the event-search projection transactionally. Unsupported sources have no @@ -401,13 +658,14 @@ pub(crate) fn source_stats(path: &Path) -> Result { stats.change_token = Some(source_change_token(change_entries)); return Ok(stats); } - add_source_stat( + let observation = observe_ordinary_file(path) + .with_context(|| format!("observe import source {}", path.display()))?; + add_source_observation( &mut stats, &mut change_entries, path.parent().unwrap_or(path), path, - &metadata, - true, + &observation, true, ); stats.change_token = Some(source_change_token(change_entries)); @@ -440,6 +698,11 @@ pub(crate) fn source_stats(path: &Path) -> Result { }) } }; + if is_sqlite_sidecar_path(&entry_path) { + stats.files += 1; + stats.bytes = stats.bytes.saturating_add(metadata.len()); + continue; + } if is_sqlite_main_path(&entry_path) { stats.files += 1; stats.bytes = stats.bytes.saturating_add(metadata.len()); @@ -459,15 +722,16 @@ pub(crate) fn source_stats(path: &Path) -> Result { } continue; } - let include_in_token = !is_sqlite_sidecar_path(&entry_path); - add_source_stat( + let observation = observe_ordinary_file(&entry_path).with_context(|| { + format!("observe import source file {}", entry_path.display()) + })?; + add_source_observation( &mut stats, &mut change_entries, path, &entry_path, - &metadata, + &observation, true, - include_in_token, ); } } @@ -500,6 +764,24 @@ impl SourceChangeEntry { } } + pub(crate) fn from_observation( + base: &Path, + path: &Path, + observation: &OrdinaryFileObservation, + ) -> Self { + let modified = observation + .modified_at() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + Self { + path: path.strip_prefix(base).unwrap_or(path).to_path_buf(), + len: observation.len(), + modified_secs: modified.as_secs(), + modified_nanos: modified.subsec_nanos(), + sentinel: observation.token().to_vec(), + } + } + pub(crate) fn from_sqlite_observed(base: &Path, file: &SqliteObservedFile) -> Self { Self { path: file @@ -534,23 +816,20 @@ fn is_sqlite_sidecar_path(path: &Path) -> bool { }) } -fn add_source_stat( +fn add_source_observation( stats: &mut SourceStats, change_entries: &mut Vec, base: &Path, path: &Path, - metadata: &fs::Metadata, - include_in_totals: bool, + observation: &OrdinaryFileObservation, include_in_token: bool, ) { - if include_in_totals { - stats.files += 1; - stats.bytes = stats.bytes.saturating_add(metadata.len()); - } + stats.files += 1; + stats.bytes = stats.bytes.saturating_add(observation.len()); if !include_in_token { return; } - change_entries.push(SourceChangeEntry::from_metadata(base, path, metadata)); + change_entries.push(SourceChangeEntry::from_observation(base, path, observation)); } pub(crate) fn source_change_token(mut entries: Vec) -> [u8; 32] { @@ -571,17 +850,6 @@ pub(crate) fn source_change_token(mut entries: Vec) -> [u8; 3 hasher.finalize().into() } -pub(crate) fn source_import_stats(source: &SourceInfo) -> Result { - let mut stats = SourceStats::default(); - for path in collect_source_import_paths(source)? { - let metadata = fs::metadata(&path) - .with_context(|| format!("stat import source file {}", path.display()))?; - stats.files += 1; - stats.bytes = stats.bytes.saturating_add(metadata.len()); - } - Ok(stats) -} - pub(crate) fn import_record_for_source(source: &SourceInfo) -> HistoryRecord { let key = format!( "agent-history:{}:{}", @@ -658,231 +926,4 @@ pub(crate) fn import_record_for_history_source_plugin( record } -#[cfg(test)] -mod tests { - use super::*; - use crate::provider_sources::explicit_path_source; - use ctx_history_capture::provider_source_specs; - use ctx_history_core::AgentType; - - #[test] - fn every_importable_provider_uses_incremental_event_search() { - for spec in provider_source_specs() { - let source = explicit_path_source( - spec.provider, - PathBuf::from(format!("{}-history", spec.provider.as_str())), - ); - - assert_eq!(source.import_support, spec.import_support); - assert!( - source_uses_incremental_event_search(&source), - "{} import must maintain event search incrementally", - spec.provider - ); - } - } - - #[test] - fn unsupported_source_does_not_claim_incremental_event_search() { - let source = explicit_path_source(CaptureProvider::Shell, PathBuf::from("shell-history")); - - assert!(!source.import_support.is_importable()); - assert!(!source_uses_incremental_event_search(&source)); - } - - #[test] - fn codex_result_rejects_a_generation_superseded_after_normalization() { - let temp = tempfile::tempdir().unwrap(); - let db_path = temp.path().join("work.sqlite"); - let source_path = temp.path().join("session.jsonl"); - fs::write(&source_path, b"{}\n").unwrap(); - let source_root = temp.path().join("sessions").display().to_string(); - let source_path = source_path.display().to_string(); - let session = CatalogSession { - provider: CaptureProvider::Codex, - source_format: "codex_session_jsonl".to_owned(), - source_root: source_root.clone(), - source_path, - external_session_id: Some("superseded-result".to_owned()), - parent_external_session_id: None, - agent_type: AgentType::Primary, - role_hint: None, - external_agent_id: None, - cwd: None, - session_started_at_ms: Some(1), - file_size_bytes: 3, - file_modified_at_ms: 1, - import_revision: 1, - cataloged_at_ms: 1, - metadata: serde_json::json!({}), - }; - let store = Store::open(db_path).unwrap(); - let superseded = store - .allocate_catalog_inventory_generation(CaptureProvider::Codex, &source_root) - .unwrap(); - store - .upsert_catalog_sessions(superseded, std::slice::from_ref(&session)) - .unwrap(); - store - .complete_catalog_inventory_generation(CaptureProvider::Codex, &source_root, superseded) - .unwrap(); - store - .allocate_catalog_inventory_generation(CaptureProvider::Codex, &source_root) - .unwrap(); - - let error = mark_catalog_session_result( - &store, - &session, - Some(1), - 2, - CatalogIndexedStatus::Indexed, - None, - superseded, - ) - .unwrap_err(); - - assert!(error.chain().any(|cause| matches!( - cause.downcast_ref::(), - Some(CaptureError::InventorySuperseded) - ))); - } - - #[test] - fn sqlite_source_stats_observe_durable_sidecars_but_ignore_shm() { - let temp = tempfile::tempdir().unwrap(); - let db = temp.path().join("state.db"); - fs::write(&db, b"main").unwrap(); - let initial = source_stats(&db).unwrap().change_token.unwrap(); - - fs::write(sqlite_sidecar(&db, "-shm"), b"volatile coordination state").unwrap(); - assert_eq!(source_stats(&db).unwrap().change_token.unwrap(), initial); - - fs::write(sqlite_sidecar(&db, "-wal"), b"committed wal frame").unwrap(); - assert_ne!(source_stats(&db).unwrap().change_token.unwrap(), initial); - - let root = temp.path().join("project"); - fs::create_dir(&root).unwrap(); - let nested_db = root.join("session.db"); - fs::write(&nested_db, b"main").unwrap(); - let root_initial = source_stats(&root).unwrap().change_token.unwrap(); - fs::write(sqlite_sidecar(&nested_db, "-shm"), b"volatile").unwrap(); - assert_eq!( - source_stats(&root).unwrap().change_token.unwrap(), - root_initial - ); - fs::write(sqlite_sidecar(&nested_db, "-journal"), b"committed journal").unwrap(); - assert_ne!( - source_stats(&root).unwrap().change_token.unwrap(), - root_initial - ); - } - - #[test] - fn sqlite_source_stats_detect_same_stat_wal_generation_and_disappearance() { - use std::fs::FileTimes; - - let temp = tempfile::tempdir().unwrap(); - let db = temp.path().join("state.db"); - let first_fixture = real_wal_generation(temp.path(), "first", "omega"); - let second_fixture = real_wal_generation(temp.path(), "second", "sigma"); - assert_eq!(first_fixture.0, second_fixture.0); - assert_eq!(first_fixture.1.len(), second_fixture.1.len()); - fs::write(&db, first_fixture.0).unwrap(); - let wal = sqlite_sidecar(&db, "-wal"); - fs::write(&wal, first_fixture.1).unwrap(); - let original_metadata = fs::metadata(&wal).unwrap(); - let original_modified = original_metadata.modified().unwrap(); - let first = source_stats(&db).unwrap().change_token.unwrap(); - - fs::write(&wal, second_fixture.1).unwrap(); - fs::File::options() - .write(true) - .open(&wal) - .unwrap() - .set_times(FileTimes::new().set_modified(original_modified)) - .unwrap(); - let replacement_metadata = fs::metadata(&wal).unwrap(); - assert_eq!(replacement_metadata.len(), original_metadata.len()); - assert_eq!(replacement_metadata.modified().unwrap(), original_modified); - let replaced = source_stats(&db).unwrap().change_token.unwrap(); - assert_ne!(replaced, first); - - fs::remove_file(&wal).unwrap(); - let disappeared = source_stats(&db).unwrap().change_token.unwrap(); - assert_ne!(disappeared, replaced); - } - - #[test] - fn ordinary_source_change_tokens_keep_the_stat_only_encoding() { - let path = PathBuf::from("session.jsonl"); - let entry = SourceChangeEntry { - path: path.clone(), - len: 42, - modified_secs: 123, - modified_nanos: 456, - sentinel: Vec::new(), - }; - let mut expected = Sha256::new(); - let path = path.as_os_str().as_encoded_bytes(); - expected.update((path.len() as u64).to_le_bytes()); - expected.update(path); - expected.update(42_u64.to_le_bytes()); - expected.update(123_u64.to_le_bytes()); - expected.update(456_u32.to_le_bytes()); - let expected: [u8; 32] = expected.finalize().into(); - - assert_eq!(source_change_token(vec![entry]), expected); - } - - #[cfg(unix)] - #[test] - fn source_change_tokens_distinguish_lossy_non_utf8_path_labels() { - use std::ffi::OsString; - use std::os::unix::ffi::OsStringExt; - - let path_a = PathBuf::from(OsString::from_vec(b"session-\x80.jsonl".to_vec())); - let path_b = PathBuf::from(OsString::from_vec(b"session-\x81.jsonl".to_vec())); - assert_eq!(path_a.display().to_string(), path_b.display().to_string()); - - let entry = |path| SourceChangeEntry { - path, - len: 42, - modified_secs: 123, - modified_nanos: 456, - sentinel: Vec::new(), - }; - assert_ne!( - source_change_token(vec![entry(path_a)]), - source_change_token(vec![entry(path_b)]) - ); - } - - fn sqlite_sidecar(path: &Path, suffix: &str) -> PathBuf { - let mut sidecar = path.as_os_str().to_owned(); - sidecar.push(suffix); - PathBuf::from(sidecar) - } - - fn real_wal_generation(root: &Path, name: &str, value: &str) -> (Vec, Vec) { - let path = root.join(format!("{name}.db")); - let writer = rusqlite::Connection::open(&path).unwrap(); - writer - .execute_batch( - "PRAGMA page_size = 512; - VACUUM; - CREATE TABLE entries (id INTEGER PRIMARY KEY, value TEXT); - INSERT INTO entries VALUES (1, 'alpha'); - PRAGMA journal_mode = WAL; - PRAGMA wal_autocheckpoint = 0; - PRAGMA wal_checkpoint(TRUNCATE);", - ) - .unwrap(); - writer - .execute("UPDATE entries SET value = ?1 WHERE id = 1", [value]) - .unwrap(); - ( - fs::read(&path).unwrap(), - fs::read(sqlite_sidecar(&path, "-wal")).unwrap(), - ) - } -} +include!("catalog_tests.rs"); diff --git a/crates/ctx-cli/src/commands/import/catalog_tests.rs b/crates/ctx-cli/src/commands/import/catalog_tests.rs new file mode 100644 index 00000000..e90b9c23 --- /dev/null +++ b/crates/ctx-cli/src/commands/import/catalog_tests.rs @@ -0,0 +1,292 @@ +#[cfg(test)] +mod tests { + use super::*; + use crate::provider_sources::explicit_path_source; + use ctx_history_capture::provider_source_specs; + use ctx_history_core::AgentType; + + #[test] + fn every_importable_provider_uses_incremental_event_search() { + for spec in provider_source_specs() { + let source = explicit_path_source( + spec.provider, + PathBuf::from(format!("{}-history", spec.provider.as_str())), + ); + + assert_eq!(source.import_support, spec.import_support); + assert!( + source_uses_incremental_event_search(&source), + "{} import must maintain event search incrementally", + spec.provider + ); + } + } + + #[test] + fn unsupported_source_does_not_claim_incremental_event_search() { + let source = explicit_path_source(CaptureProvider::Shell, PathBuf::from("shell-history")); + + assert!(!source.import_support.is_importable()); + assert!(!source_uses_incremental_event_search(&source)); + } + + #[test] + fn codex_result_rejects_a_generation_superseded_after_normalization() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("work.sqlite"); + let source_path = temp.path().join("session.jsonl"); + fs::write(&source_path, b"{}\n").unwrap(); + let source_root = temp.path().join("sessions").display().to_string(); + let source_path = source_path.display().to_string(); + let session = CatalogSession { + provider: CaptureProvider::Codex, + source_format: "codex_session_jsonl".to_owned(), + source_root: source_root.clone(), + source_path, + external_session_id: Some("superseded-result".to_owned()), + parent_external_session_id: None, + agent_type: AgentType::Primary, + role_hint: None, + external_agent_id: None, + cwd: None, + session_started_at_ms: Some(1), + file_size_bytes: 3, + file_modified_at_ms: 1, + import_revision: 1, + cataloged_at_ms: 1, + metadata: serde_json::json!({ + "file_observation_token_v1": "superseded-result-token" + }), + }; + let store = Store::open(db_path).unwrap(); + let superseded = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, &source_root) + .unwrap(); + store + .upsert_catalog_sessions(superseded, std::slice::from_ref(&session)) + .unwrap(); + store + .complete_catalog_inventory_generation(CaptureProvider::Codex, &source_root, superseded) + .unwrap(); + store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, &source_root) + .unwrap(); + + let error = mark_catalog_session_result( + &store, + &session, + Some(1), + 2, + CatalogIndexedStatus::Indexed, + None, + superseded, + ) + .unwrap_err(); + + assert!(error.chain().any(|cause| matches!( + cause.downcast_ref::(), + Some(CaptureError::InventorySuperseded) + ))); + } + + #[test] + fn sqlite_source_stats_observe_durable_sidecars_but_ignore_shm() { + let temp = tempfile::tempdir().unwrap(); + let db = temp.path().join("state.db"); + fs::write(&db, b"main").unwrap(); + let initial = source_stats(&db).unwrap().change_token.unwrap(); + + fs::write(sqlite_sidecar(&db, "-shm"), b"volatile coordination state").unwrap(); + assert_eq!(source_stats(&db).unwrap().change_token.unwrap(), initial); + + fs::write(sqlite_sidecar(&db, "-wal"), b"committed wal frame").unwrap(); + assert_ne!(source_stats(&db).unwrap().change_token.unwrap(), initial); + + let root = temp.path().join("project"); + fs::create_dir(&root).unwrap(); + let nested_db = root.join("session.db"); + fs::write(&nested_db, b"main").unwrap(); + let root_initial = source_stats(&root).unwrap().change_token.unwrap(); + fs::write(sqlite_sidecar(&nested_db, "-shm"), b"volatile").unwrap(); + assert_eq!( + source_stats(&root).unwrap().change_token.unwrap(), + root_initial + ); + fs::write(sqlite_sidecar(&nested_db, "-journal"), b"committed journal").unwrap(); + assert_ne!( + source_stats(&root).unwrap().change_token.unwrap(), + root_initial + ); + } + + #[test] + fn sqlite_source_stats_ignore_orphan_sidecar_contents_and_churn() { + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("session.jsonl"); + let sidecar = temp.path().join("orphan.db-wal"); + fs::write(&source, b"stable\n").unwrap(); + let initial = source_stats(temp.path()).unwrap(); + + fs::write(&sidecar, b"first generation").unwrap(); + let present = source_stats(temp.path()).unwrap(); + assert_eq!(present.change_token, initial.change_token); + assert_eq!(present.files, initial.files + 1); + assert_eq!(present.bytes, initial.bytes + 16); + + fs::remove_file(&sidecar).unwrap(); + let disappeared = source_stats(temp.path()).unwrap(); + assert_eq!(disappeared.change_token, initial.change_token); + assert_eq!(disappeared.files, initial.files); + assert_eq!(disappeared.bytes, initial.bytes); + } + + #[cfg(unix)] + #[test] + fn sqlite_source_stats_do_not_reopen_orphan_sidecars() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let sidecar = temp.path().join("orphan.db-journal"); + fs::write(&sidecar, b"transient journal").unwrap(); + fs::set_permissions(&sidecar, fs::Permissions::from_mode(0o000)).unwrap(); + + let stats = source_stats(temp.path()).unwrap(); + + assert_eq!(stats.files, 1); + assert_eq!(stats.bytes, 17); + assert_eq!(stats.change_token, Some(source_change_token(Vec::new()))); + } + + #[test] + fn sqlite_source_stats_detect_same_stat_wal_generation_and_disappearance() { + use std::fs::FileTimes; + + let temp = tempfile::tempdir().unwrap(); + let db = temp.path().join("state.db"); + let first_fixture = real_wal_generation(temp.path(), "first", "omega"); + let second_fixture = real_wal_generation(temp.path(), "second", "sigma"); + assert_eq!(first_fixture.0, second_fixture.0); + assert_eq!(first_fixture.1.len(), second_fixture.1.len()); + fs::write(&db, first_fixture.0).unwrap(); + let wal = sqlite_sidecar(&db, "-wal"); + fs::write(&wal, first_fixture.1).unwrap(); + let original_metadata = fs::metadata(&wal).unwrap(); + let original_modified = original_metadata.modified().unwrap(); + let first = source_stats(&db).unwrap().change_token.unwrap(); + + fs::write(&wal, second_fixture.1).unwrap(); + fs::File::options() + .write(true) + .open(&wal) + .unwrap() + .set_times(FileTimes::new().set_modified(original_modified)) + .unwrap(); + let replacement_metadata = fs::metadata(&wal).unwrap(); + assert_eq!(replacement_metadata.len(), original_metadata.len()); + assert_eq!(replacement_metadata.modified().unwrap(), original_modified); + let replaced = source_stats(&db).unwrap().change_token.unwrap(); + assert_ne!(replaced, first); + + fs::remove_file(&wal).unwrap(); + let disappeared = source_stats(&db).unwrap().change_token.unwrap(); + assert_ne!(disappeared, replaced); + } + + #[test] + fn source_change_tokens_include_the_ordinary_file_observation() { + let path = PathBuf::from("session.jsonl"); + let entry = SourceChangeEntry { + path: path.clone(), + len: 42, + modified_secs: 123, + modified_nanos: 456, + sentinel: b"ordinary-token".to_vec(), + }; + let mut expected = Sha256::new(); + let path = path.as_os_str().as_encoded_bytes(); + expected.update((path.len() as u64).to_le_bytes()); + expected.update(path); + expected.update(42_u64.to_le_bytes()); + expected.update(123_u64.to_le_bytes()); + expected.update(456_u32.to_le_bytes()); + expected.update(14_u64.to_le_bytes()); + expected.update(b"ordinary-token"); + let expected: [u8; 32] = expected.finalize().into(); + + assert_eq!(source_change_token(vec![entry]), expected); + } + + #[cfg(unix)] + #[test] + fn ordinary_source_stats_detect_same_size_rewrite_with_restored_mtime() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("session.jsonl"); + fs::write(&path, b"alpha\n").unwrap(); + let original_modified = fs::metadata(&path).unwrap().modified().unwrap(); + let first = source_stats(&path).unwrap(); + + fs::write(&path, b"omega\n").unwrap(); + fs::File::options() + .write(true) + .open(&path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(original_modified)) + .unwrap(); + let second = source_stats(&path).unwrap(); + + assert_eq!(first.bytes, second.bytes); + assert_ne!(first.change_token, second.change_token); + } + + #[cfg(unix)] + #[test] + fn source_change_tokens_distinguish_lossy_non_utf8_path_labels() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let path_a = PathBuf::from(OsString::from_vec(b"session-\x80.jsonl".to_vec())); + let path_b = PathBuf::from(OsString::from_vec(b"session-\x81.jsonl".to_vec())); + assert_eq!(path_a.display().to_string(), path_b.display().to_string()); + + let entry = |path| SourceChangeEntry { + path, + len: 42, + modified_secs: 123, + modified_nanos: 456, + sentinel: Vec::new(), + }; + assert_ne!( + source_change_token(vec![entry(path_a)]), + source_change_token(vec![entry(path_b)]) + ); + } + + fn sqlite_sidecar(path: &Path, suffix: &str) -> PathBuf { + let mut sidecar = path.as_os_str().to_owned(); + sidecar.push(suffix); + PathBuf::from(sidecar) + } + + fn real_wal_generation(root: &Path, name: &str, value: &str) -> (Vec, Vec) { + let path = root.join(format!("{name}.db")); + let writer = rusqlite::Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA page_size = 512; + VACUUM; + CREATE TABLE entries (id INTEGER PRIMARY KEY, value TEXT); + INSERT INTO entries VALUES (1, 'alpha'); + PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + PRAGMA wal_checkpoint(TRUNCATE);", + ) + .unwrap(); + writer + .execute("UPDATE entries SET value = ?1 WHERE id = 1", [value]) + .unwrap(); + ( + fs::read(&path).unwrap(), + fs::read(sqlite_sidecar(&path, "-wal")).unwrap(), + ) + } +} diff --git a/crates/ctx-cli/src/commands/import/inventory.rs b/crates/ctx-cli/src/commands/import/inventory.rs index bbe66844..818026a9 100644 --- a/crates/ctx-cli/src/commands/import/inventory.rs +++ b/crates/ctx-cli/src/commands/import/inventory.rs @@ -1,5 +1,6 @@ use std::{ fs, + path::PathBuf, time::{SystemTime, UNIX_EPOCH}, }; @@ -7,14 +8,17 @@ use anyhow::{Context, Result}; use serde_json::{json, Value}; use ctx_history_capture::{ - catalog_codex_session_tree, CatalogSummary, CodexSessionCatalogOptions, ProviderImportSupport, - ProviderImportUnitSpec, ProviderSourceStatus, + catalog_codex_session_tree, provider_source_for_persisted_format, CaptureError, CatalogSummary, + CodexSessionCatalogOptions, ProviderImportSupport, ProviderSourceStatus, +}; +use ctx_history_core::{utc_now, CaptureProvider}; +use ctx_history_store::{ + CatalogIndexedStatus, CatalogSourceIndexUpdate, ProviderFileInventoryFamily, + ProviderFilePublicationInventoryOwner, SourceImportFile, SourceImportFileIndexUpdate, Store, }; -use ctx_history_core::CaptureProvider; -use ctx_history_store::{SourceImportFile, Store}; use crate::commands::import::catalog::codex_catalog_root_identity; -use crate::commands::import::catalog::{source_import_stats, source_stats, system_time_ms}; +use crate::commands::import::catalog::{source_stats, system_time_ms}; use crate::commands::import::manifest::{ collect_source_import_files, persist_new_source_import_observation, persisted_import_identity, source_uses_import_file_manifest, @@ -36,25 +40,39 @@ pub(crate) struct ImportInventory { pub(crate) fn inventory_import_sources( store: &Store, - sources: Vec, + mut sources: Vec, full_rescan: bool, ) -> Result { let mut inventory = ImportInventory::default(); - for (index, source) in sources.into_iter().enumerate() { + if let Some(owner) = store.effective_provider_file_publication_inventory_owner()? { + sources.retain(|source| !source_matches_publication_owner(source, &owner)); + let plan = publication_owner_plan(owner)?; + inventory.totals.sources += 1; + inventory.totals.source_files += 1; + inventory.totals.source_bytes = plan.stats.bytes; + match &plan.preinventory { + SourcePreinventory::CodexSessionCatalog { .. } => { + inventory.totals.codex_catalog_sources += 1; + } + SourcePreinventory::SourceImportFiles { .. } => { + inventory.totals.source_import_files += 1; + } + SourcePreinventory::None | SourcePreinventory::SourceRoot { .. } => {} + } + inventory.sources.push(plan); + } + for source in sources { inventory.totals.sources += 1; let failure_source = source.clone(); let (plan, cataloged) = match inventory_import_source(store, source, full_rescan) { Ok(inventoried) => inventoried, Err(error) if import_error_scope(&error) == ImportFailureScope::Source => { inventory.failures.push(ImportSourceFailure { - index, source: failure_source, stats: SourceStats::default(), error: error_summary(&error), - failure_scope: ImportFailureScope::Source, failure_type: import_failure_type(&error), rejected_summary: None, - system_error: None, }); continue; } @@ -85,6 +103,49 @@ pub(crate) fn inventory_import_sources( Ok(inventory) } +fn source_matches_publication_owner( + source: &SourceInfo, + owner: &ProviderFilePublicationInventoryOwner, +) -> bool { + source.provider == owner.provider + && persisted_import_identity(&source.path, "source root") + .is_ok_and(|source_root| source_root == owner.source_root) +} + +fn publication_owner_plan( + owner: ProviderFilePublicationInventoryOwner, +) -> Result { + let source = provider_source_for_persisted_format( + owner.provider, + PathBuf::from(&owner.source_root), + &owner.source_format, + ) + .ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "persisted publication owner has an unsupported source format", + )) + })?; + let preinventory = match owner.inventory_family { + ProviderFileInventoryFamily::Catalog => SourcePreinventory::CodexSessionCatalog { + summary: CatalogSummary::default(), + inventory_generation: owner.inventory_generation, + }, + ProviderFileInventoryFamily::SourceImport => SourcePreinventory::SourceImportFiles { + files: Vec::new(), + inventory_generation: owner.inventory_generation, + }, + }; + Ok(PlannedImportSource { + source, + stats: SourceStats { + files: 1, + bytes: owner.file_size_bytes, + change_token: None, + }, + preinventory, + }) +} + pub(crate) fn inventory_available_sources( store: &Store, sources: &[SourceInfo], @@ -104,12 +165,12 @@ pub(crate) fn inventory_available_sources( fn inventory_import_source( store: &Store, source: SourceInfo, - full_rescan: bool, + resume: bool, ) -> Result<(PlannedImportSource, Option<(CatalogSummary, Value)>)> { if source.provider == CaptureProvider::Codex { codex_catalog_root_identity(&source.path)?; } - if !full_rescan && is_incremental_codex_session_tree(&source) { + if is_incremental_codex_session_tree(&source) { let source_root = persisted_import_identity(&source.path, "source root")?.to_owned(); let mut cataloged = None; for _ in 0..3 { @@ -152,6 +213,14 @@ fn inventory_import_source( bytes: summary.source_bytes, change_token: None, }; + if resume { + schedule_pending_catalog_resume( + store, + CaptureProvider::Codex, + &source_root, + inventory_generation, + )?; + } let plan = PlannedImportSource { source, stats, @@ -178,7 +247,22 @@ fn inventory_import_source( if source_uses_import_file_manifest(&source) { let files = collect_source_import_files(&source) .with_context(|| format!("inventory import files from {}", source.path.display()))?; + if files.is_empty() { + return Err(anyhow::anyhow!( + "no importable {} history files found under {}", + source.provider.as_str(), + source.path.display() + )); + } let persisted = persist_new_source_import_observation(store, &source, &files)?; + if resume { + schedule_pending_source_resume( + store, + source.provider, + persisted_import_identity(&source.path, "source root")?, + persisted.inventory_generation, + )?; + } let stats = source_stats_from_import_files(&files); return Ok(( PlannedImportSource { @@ -193,41 +277,93 @@ fn inventory_import_source( )); } - let inventory_whole_source_rescan = full_rescan - && source.provider != CaptureProvider::Codex - && source.import_unit == ProviderImportUnitSpec::WholeSource; - if !full_rescan || inventory_whole_source_rescan { - let (stats, root_file) = observe_source_root(&source)?; - let persisted = persist_new_source_import_observation( + let (stats, root_file) = observe_source_root(&source)?; + let persisted = + persist_new_source_import_observation(store, &source, std::slice::from_ref(&root_file))?; + if resume { + schedule_pending_source_resume( store, - &source, - std::slice::from_ref(&root_file), + source.provider, + &root_file.source_root, + persisted.inventory_generation, )?; - return Ok(( - PlannedImportSource { - source, - stats, - preinventory: SourcePreinventory::SourceRoot { - file: root_file, - inventory_generation: persisted.inventory_generation, - }, - }, - None, - )); } - - let stats = source_import_stats(&source) - .with_context(|| format!("inventory import source {}", source.path.display()))?; Ok(( PlannedImportSource { source, stats, - preinventory: SourcePreinventory::None, + preinventory: SourcePreinventory::SourceRoot { + file: root_file, + inventory_generation: persisted.inventory_generation, + }, }, None, )) } +fn schedule_pending_catalog_resume( + store: &Store, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, +) -> Result<()> { + for session in store.list_pending_catalog_sessions(provider, source_root)? { + let state = store.catalog_source_index_state( + session.provider, + &session.source_root, + &session.source_path, + )?; + store.record_observed_catalog_source_import_result( + session.provider, + CatalogSourceIndexUpdate { + source_root: &session.source_root, + source_path: &session.source_path, + file_size_bytes: session.file_size_bytes, + file_modified_at_ms: session.file_modified_at_ms, + import_revision: session.import_revision, + inventory_generation, + file_sha256: state + .as_ref() + .and_then(|state| state.last_imported_file_sha256.as_deref()), + event_count: state + .as_ref() + .and_then(|state| state.last_imported_event_count), + indexed_at_ms: utc_now().timestamp_millis(), + }, + &session.metadata, + CatalogIndexedStatus::Pending, + None, + )?; + } + Ok(()) +} + +fn schedule_pending_source_resume( + store: &Store, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, +) -> Result<()> { + for file in store.list_pending_source_import_files(provider, source_root)? { + store.record_source_import_file_result( + file.provider, + SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation, + metadata: &file.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + CatalogIndexedStatus::Pending, + None, + )?; + } + Ok(()) +} + fn is_incremental_codex_session_tree(source: &SourceInfo) -> bool { source.provider == CaptureProvider::Codex && source.source_format == "codex_session_jsonl_tree" } diff --git a/crates/ctx-cli/src/commands/import/manifest.rs b/crates/ctx-cli/src/commands/import/manifest.rs index 2a765d25..c8160932 100644 --- a/crates/ctx-cli/src/commands/import/manifest.rs +++ b/crates/ctx-cli/src/commands/import/manifest.rs @@ -3,8 +3,9 @@ use std::collections::{BTreeMap, BTreeSet}; use super::*; use crate::commands::import::catalog::{source_change_token, system_time_ms, SourceChangeEntry}; use ctx_history_capture::{ - observe_sqlite_source_generation, ProviderImportDependency, ProviderImportUnitGrouping, - ProviderImportUnitOwner, ProviderImportUnitSpec, + observe_ordinary_file, observe_sqlite_source_generation, pace_current_disk_io, + ProviderImportDependency, ProviderImportUnitGrouping, ProviderImportUnitOwner, + ProviderImportUnitSpec, }; pub(crate) struct PersistedSourceImportObservation { @@ -73,6 +74,13 @@ fn persist_source_import_observation_with_outcomes_inner( )?; before_outcomes(); for outcome in outcomes { + pace_current_disk_io( + 256u64 + .saturating_add(outcome.file.source_format.len() as u64) + .saturating_add(outcome.file.source_root.len() as u64) + .saturating_add(outcome.file.source_path.len() as u64) + .saturating_add(serde_json::to_vec(&outcome.file.metadata)?.len() as u64), + ); let changed = store.record_source_import_file_result( outcome.file.provider, SourceImportFileIndexUpdate { @@ -149,13 +157,18 @@ fn persist_source_import_files_in_batch( .iter() .map(|file| file.source_path.clone()) .collect::>(); - store.upsert_source_import_files(inventory_generation, files)?; - store.mark_source_import_missing_paths_stale( + store.upsert_source_import_files_with_pacing( + inventory_generation, + files, + pace_current_disk_io, + )?; + store.mark_source_import_missing_paths_stale_with_pacing( source.provider, source_root, ¤t_paths, utc_now().timestamp_millis(), inventory_generation, + pace_current_disk_io, )?; Ok(()) } @@ -170,49 +183,111 @@ pub(crate) fn collect_source_import_files(source: &SourceInfo) -> Result>(); - let absent_dependency_paths = fingerprint - .absent_dependencies - .iter() - .map(|path| import_unit_path_label(fingerprint_base, path)) - .collect::>(); - files.push(SourceImportFile { - provider: source.provider, - source_format: source.source_format.to_owned(), - source_root: source_root.clone(), - source_path: owner_identity.to_owned(), - file_size_bytes: fingerprint.owner_len, - file_modified_at_ms: system_time_ms(fingerprint.owner_modified_at), - import_revision: source.import_revision, + files.push(observe_collected_source_import_unit( + source, + &source_root, + &unit, observed_at_ms, - metadata: json!({ - "inventory_unit": "logical_import_unit", - "change_token_v1": hex_change_token(fingerprint.change_token), - "dependencies": dependency_paths, - "absent_dependencies": absent_dependency_paths, - }), - }); + )?); } Ok(files) } -pub(crate) fn collect_source_import_paths(source: &SourceInfo) -> Result> { - Ok(collect_source_import_units(source)? - .into_iter() - .map(|unit| unit.owner) - .collect()) +pub(crate) fn observe_selected_source_import_file( + source: &SourceInfo, + source_path: &str, +) -> Result> { + let ProviderImportUnitSpec::PerFile { + owner, + dependencies, + .. + } = source.import_unit + else { + return Err(anyhow!( + "selected import file does not belong to a manifested source" + )); + }; + let path = PathBuf::from(source_path); + let belongs_to_source = path == source.path || path.starts_with(&source.path); + if !belongs_to_source || !import_unit_owner_matches(owner, &source.path, &path) { + return Err(anyhow!( + "selected import unit is outside its manifested source: {}", + path.display() + )); + } + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => { + return Err(anyhow!( + "import unit owner is not a regular file: {}", + path.display() + )) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("stat import unit owner {}", path.display())) + } + } + let source_root = persisted_import_identity(&source.path, "source root")?; + let unit = collected_import_unit(path, dependencies)?; + observe_collected_source_import_unit(source, source_root, &unit, utc_now().timestamp_millis()) + .map(Some) +} + +pub(crate) fn same_source_import_observation( + left: &SourceImportFile, + right: &SourceImportFile, +) -> bool { + left.provider == right.provider + && left.source_format == right.source_format + && left.source_root == right.source_root + && left.source_path == right.source_path + && left.file_size_bytes == right.file_size_bytes + && left.file_modified_at_ms == right.file_modified_at_ms + && left.import_revision == right.import_revision + && left.metadata == right.metadata +} + +fn observe_collected_source_import_unit( + source: &SourceInfo, + source_root: &str, + unit: &CollectedImportUnit, + observed_at_ms: i64, +) -> Result { + let owner_identity = persisted_import_identity(&unit.owner, "import unit owner")?; + let fingerprint_base = if source.path.is_dir() { + source.path.as_path() + } else { + source.path.parent().unwrap_or(source.path.as_path()) + }; + let fingerprint = import_unit_fingerprint(fingerprint_base, unit)?; + let dependency_paths = fingerprint + .dependencies + .iter() + .filter(|path| *path != &unit.owner) + .map(|path| import_unit_path_label(fingerprint_base, path)) + .collect::>(); + let absent_dependency_paths = fingerprint + .absent_dependencies + .iter() + .map(|path| import_unit_path_label(fingerprint_base, path)) + .collect::>(); + Ok(SourceImportFile { + provider: source.provider, + source_format: source.source_format.to_owned(), + source_root: source_root.to_owned(), + source_path: owner_identity.to_owned(), + file_size_bytes: fingerprint.owner_len, + file_modified_at_ms: system_time_ms(fingerprint.owner_modified_at), + import_revision: source.import_revision, + observed_at_ms, + metadata: json!({ + "inventory_unit": "logical_import_unit", + "change_token_v1": hex_change_token(fingerprint.change_token), + "dependencies": dependency_paths, + "absent_dependencies": absent_dependency_paths, + }), + }) } struct CollectedImportUnit { @@ -538,31 +613,23 @@ fn import_unit_fingerprint( } (generation.main().len(), generation.main().modified_at()) } else { - let metadata = fs::symlink_metadata(&unit.owner) - .with_context(|| format!("stat import unit owner {}", unit.owner.display()))?; - if !metadata.file_type().is_file() { - return Err(anyhow!( - "import unit owner is not a regular file: {}", - unit.owner.display() - )); - } - entries.push(SourceChangeEntry::from_metadata( + let observation = observe_ordinary_file(&unit.owner) + .with_context(|| format!("observe import unit owner {}", unit.owner.display()))?; + entries.push(SourceChangeEntry::from_observation( base, &unit.owner, - &metadata, + &observation, )); - (metadata.len(), metadata.modified().unwrap_or(UNIX_EPOCH)) + (observation.len(), observation.modified_at()) }; for path in unit.dependencies.iter().filter(|path| *path != &unit.owner) { - let metadata = fs::symlink_metadata(path) - .with_context(|| format!("stat import unit dependency {}", path.display()))?; - if !metadata.file_type().is_file() { - return Err(anyhow!( - "import unit dependency is not a regular file: {}", - path.display() - )); - } - entries.push(SourceChangeEntry::from_metadata(base, path, &metadata)); + let observation = observe_ordinary_file(path) + .with_context(|| format!("observe import unit dependency {}", path.display()))?; + entries.push(SourceChangeEntry::from_observation( + base, + path, + &observation, + )); } let observed_absences = unit .absence_watches diff --git a/crates/ctx-cli/src/commands/import/native.rs b/crates/ctx-cli/src/commands/import/native.rs index 20f7907b..188faa2e 100644 --- a/crates/ctx-cli/src/commands/import/native.rs +++ b/crates/ctx-cli/src/commands/import/native.rs @@ -1,17 +1,32 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::ops::{ControlFlow, Deref}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; + +use super::catalog::sha256_file_prefix_hex; use super::*; use crate::commands::import::inventory::observe_source_root; use crate::commands::import::manifest::{ - collect_source_import_files, persist_new_source_import_observation, - persist_source_import_observation_with_outcomes, persisted_import_identity, - source_uses_import_file_manifest, SourceImportObservationOutcome, + collect_source_import_files, observe_selected_source_import_file, + persist_new_source_import_observation, persist_source_import_observation_with_outcomes, + persisted_import_identity, same_source_import_observation, source_uses_import_file_manifest, + SourceImportObservationOutcome, }; #[cfg(test)] use crate::commands::import::manifest::{ persist_source_import_files, persist_source_import_observation_with_outcomes_and_hook, }; +use ctx_history_capture::{ + provider_jsonl_checkpoint_matches_file, provider_jsonl_range_has_complete_line, + ProviderImportMaintenanceKind, ProviderImportMaintenanceWarning, +}; +use ctx_history_store::ProviderFileMaintenanceWarning; + +const PROVIDER_PUBLICATION_SLICE_ROWS: usize = 1024; +const PROVIDER_RETIREMENT_SLICE_ROWS: usize = 64; +include!("native/types_and_recovery.rs"); +include!("native/append.rs"); include!("native/selection.rs"); include!("native/batching.rs"); include!("native/manifested.rs"); diff --git a/crates/ctx-cli/src/commands/import/native/append.rs b/crates/ctx-cli/src/commands/import/native/append.rs new file mode 100644 index 00000000..22bd36dc --- /dev/null +++ b/crates/ctx-cli/src/commands/import/native/append.rs @@ -0,0 +1,815 @@ +pub(crate) fn import_append_capable_catalog_work( + store: &mut Store, + source: &SourceInfo, + work: &CatalogImportWork, + inventory_generation: u64, + record: &HistoryRecord, +) -> Result { + import_append_capable_work( + store, + AppendInventoryUnit::Catalog { + source, + work, + inventory_generation, + }, + record, + ) +} + +fn import_append_capable_source_file_work( + store: &mut Store, + source: &SourceInfo, + work: &SourceImportFileWork, + inventory_generation: u64, + record: &HistoryRecord, +) -> Result { + import_append_capable_work( + store, + AppendInventoryUnit::SourceFile { + source, + work, + inventory_generation, + }, + record, + ) +} + +fn import_manifested_append_source_file_work( + store: &mut Store, + source: &SourceInfo, + pending_source: &SourceInfo, + work: &SourceImportFileWork, + inventory_generation: u64, +) -> Result { + let record = import_record_for_source(pending_source); + let record_existed = history_record_exists(store, record.id) + .context("inspect manifested append history record")?; + let result = + import_append_capable_source_file_work(store, source, work, inventory_generation, &record) + .context("run manifested append publication"); + let remove_orphan = match &result { + Ok(AppendImportOutcome::Imported(summary)) => summary == &ProviderImportSummary::default(), + Ok(AppendImportOutcome::Deferred { durable_progress }) => !durable_progress, + Err(_) => !store.has_pending_provider_file_publications()?, + }; + if remove_orphan && !record_existed { + store + .delete_orphan_record(record.id) + .context("clean up manifested append history record")?; + } + result +} + +fn import_append_capable_work( + store: &mut Store, + unit: AppendInventoryUnit<'_>, + record: &HistoryRecord, +) -> Result { + let provider = unit.provider(); + let source_format = unit.source_format().to_owned(); + let material_source_format = + provider_canonical_material_source_format(provider, &source_format).ok_or_else(|| { + anyhow!( + "missing canonical material format for {}:{source_format}", + provider.as_str() + ) + })?; + if provider_file_mutation_contract(provider, &source_format) + != ProviderFileMutationContract::AppendOnlyNewlineDelimited + { + return Err(anyhow!( + "{}:{source_format} is not append-capable", + provider.as_str() + )); + } + + let mut replacement = unit.reason().requires_replacement() || unit.has_active_publication(); + for _ in 0..2 { + let admitted_checkpoint = if replacement { + None + } else { + load_admitted_append_checkpoint(store, &unit)? + }; + let admitted_offset = admitted_checkpoint + .as_ref() + .map(|checkpoint| checkpoint.checkpoint().committed_offset); + if admitted_checkpoint.is_none() { + replacement = true; + } + let observed_offset = admitted_offset.unwrap_or(0); + if unit.file_size_bytes() > observed_offset + && !provider_jsonl_range_has_complete_line( + std::path::Path::new(unit.source_path()), + observed_offset, + unit.file_size_bytes(), + )? + { + return Ok(AppendImportOutcome::Deferred { + durable_progress: false, + }); + } + let kind = if replacement { + ProviderFilePublicationKind::Replacement + } else { + ProviderFilePublicationKind::Incremental + }; + let observation = unit.observation(utc_now().timestamp_millis(), None); + let scope = store + .begin_provider_file_publication( + provider, + observation, + material_source_format, + kind, + utc_now().timestamp_millis(), + ) + .context("begin append-capable provider publication")?; + let mut scope = Some(scope); + let attempt = (|| -> Result { + let publication = scope + .as_ref() + .expect("append publication scope must remain owned until completion"); + let effective_replacement = + publication.kind() == ProviderFilePublicationKind::Replacement; + if effective_replacement { + match store.provider_file_publication_phase(publication)? { + ProviderFilePublicationPhase::Preparing => { + let preparation = store + .prepare_provider_file_publication_slice( + publication, + PROVIDER_PUBLICATION_SLICE_ROWS, + ) + .context("prepare append-capable replacement publication")?; + if !preparation.complete { + store.abandon_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: preparation.rows_processed > 0, + }); + } + } + ProviderFilePublicationPhase::Reconciling => { + store + .reconcile_provider_file_publication_slice( + publication, + PROVIDER_PUBLICATION_SLICE_ROWS, + ) + .context("reconcile append-capable replacement publication")?; + store.abandon_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: true, + }); + } + ProviderFilePublicationPhase::ReadyToFinalize => { + let completion = store + .load_provider_file_publication_completion(publication)? + .ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "ready provider publication has no staged completion", + )) + })?; + let staged = decode_staged_append_completion(completion)?; + if !staged_append_completion_matches_current_file(&unit, &staged)? { + invalidate_active_append_publication_observation(store, &unit)?; + queue_current_append_observation(store, &unit, true)?; + store.abandon_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: true, + }); + } + let (mut summary, checkpoint, indexed_at_ms) = staged.into_restored(); + let status = provider_summary_import_status(&summary); + let outcome_error = + (summary.failed > 0).then(|| source_import_file_failure(&summary)); + let event_count = Some( + summary + .imported_events + .saturating_add(summary.skipped_events) + as u64, + ); + let outcome = ProviderFileImportOutcome { + provider, + observation: unit.observation(indexed_at_ms, event_count), + status, + error: outcome_error.as_deref(), + }; + let checkpoint = checkpoint + .map(|checkpoint| checkpoint.into_store_checkpoint(&unit)) + .transpose()?; + let commit = ProviderFilePublicationCommit::Replacement( + (summary.failed == 0) + .then_some(checkpoint.as_ref()) + .flatten(), + ); + let finalized = store.finalize_provider_file_publication( + scope.take().expect("append publication scope must exist"), + outcome, + commit, + )?; + if let Some(warning) = finalized.maintenance_warning { + push_publication_maintenance_warning(&mut summary, warning); + } + queue_current_append_observation(store, &unit, false)?; + return Ok(AppendPublicationAttempt::Imported(summary)); + } + ProviderFilePublicationPhase::Importing => {} + } + } + + let mode = if effective_replacement { + ProviderAppendFileImportMode::AppendCapableReplacement + } else { + ProviderAppendFileImportMode::Append(admitted_checkpoint.ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "incremental publication has no admitted checkpoint", + )) + })?) + }; + let decision = store + .with_provider_file_publication_writes_mut(publication, |store| { + if !store + .provider_file_publication_history_record_exists(publication, record.id)? + { + store.upsert_record(record)?; + } + import_append_capable_provider_file( + provider, + store, + ProviderAppendFileImportOptions { + machine_id: CodexSessionImportOptions::default().machine_id, + inventory_source_format: source_format.clone(), + material_source_format: material_source_format.to_owned(), + source_path: PathBuf::from(unit.source_path()), + source_root: PathBuf::from(unit.material_source_root()), + imported_at: utc_now(), + history_record_id: Some(record.id), + observed_size: unit.file_size_bytes(), + mode, + }, + ) + }) + .map_err(anyhow::Error::new) + .context("write append-capable provider publication")?; + #[cfg(test)] + if take_append_source_failure_after_mutation() { + return Err(anyhow::Error::new(CaptureError::InvalidPayload( + "injected append source failure after publication mutation".to_owned(), + ))); + } + + if let ProviderAppendFileImportDecision::ReplacementRequired(_) = decision { + let abort = store.abort_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + if let ControlFlow::Break(warning) = abort { + return Err(publication_recovery_required_error( + anyhow::Error::new(CaptureError::SystemInvariant( + "append importer requested replacement after mutating its publication", + )), + warning, + )); + } + if effective_replacement { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "append-capable replacement importer requested replacement", + ))); + } + return Ok(AppendPublicationAttempt::RetryReplacement); + } + + let deferred_without_boundary_progress = !effective_replacement + && admitted_offset.is_some_and(|prior_offset| { + unit.file_size_bytes() > prior_offset + && match &decision { + ProviderAppendFileImportDecision::Imported(result) => { + result.summary == ProviderImportSummary::default() + && result.checkpoint.committed_offset == prior_offset + } + ProviderAppendFileImportDecision::ImportedWithoutCheckpoint(result) => { + result.summary == ProviderImportSummary::default() + } + ProviderAppendFileImportDecision::DeferredPartial + | ProviderAppendFileImportDecision::ReplacementRequired(_) => false, + } + }); + if deferred_without_boundary_progress { + let abort = store.abort_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + if let ControlFlow::Break(warning) = abort { + return Err(publication_recovery_required_error( + anyhow::Error::new(CaptureError::SystemInvariant( + "append tail deferred after mutating its publication", + )), + warning, + )); + } + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: false, + }); + } + + let (mut summary, checkpoint, source_prefix_sha256, retain_checkpoint) = match decision + { + ProviderAppendFileImportDecision::Imported(result) => { + (result.summary, Some(result.checkpoint), None, false) + } + ProviderAppendFileImportDecision::ImportedWithoutCheckpoint(result) => ( + result.summary, + None, + result.source_prefix_sha256, + !effective_replacement, + ), + ProviderAppendFileImportDecision::DeferredPartial => { + let abort = store.abort_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + if let ControlFlow::Break(warning) = abort { + return Err(publication_recovery_required_error( + anyhow::Error::new(CaptureError::SystemInvariant( + "partial append deferred after mutating its publication", + )), + warning, + )); + } + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: false, + }); + } + ProviderAppendFileImportDecision::ReplacementRequired(_) => unreachable!(), + }; + let mut status = provider_summary_import_status(&summary); + if status == CatalogIndexedStatus::Rejected + && effective_replacement + && publication.tracks_prior_material() + { + let abort = store.abort_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + match abort { + ControlFlow::Continue(warning) => { + summary.mark_retained_existing_content(); + if let Some(warning) = warning { + push_publication_maintenance_warning(&mut summary, warning); + } + return Ok(AppendPublicationAttempt::Imported(summary)); + } + ControlFlow::Break(warning) => { + return Err(publication_recovery_required_error( + provider_import_summary_failure_for_unit( + provider, + unit.source_path(), + &summary, + ), + warning, + )); + } + } + } + if status == CatalogIndexedStatus::Rejected && !effective_replacement { + summary.mark_retained_existing_content(); + status = provider_summary_import_status(&summary); + } + if status == CatalogIndexedStatus::Rejected { + for failure in &mut summary.failures { + failure.error = format!("{}: {}", unit.source_path(), failure.error); + } + let deleted = store + .discard_provider_file_publication_orphan_record(publication, record.id)?; + if deleted == Some(false) + && store + .provider_file_publication_history_record_exists(publication, record.id)? + { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "rejected replacement left material attached to its history record", + ))); + } + store.discard_provider_file_publication_orphan_capture_sources(publication)?; + let completion = encode_staged_append_completion( + summary, + None, + None, + utc_now().timestamp_millis(), + )?; + store.stage_provider_file_publication_completion(publication, &completion)?; + store.abandon_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: true, + }); + } + let event_count = Some( + summary + .imported_events + .saturating_add(summary.skipped_events) as u64, + ); + let indexed_at_ms = utc_now().timestamp_millis(); + let outcome_error = (summary.failed > 0).then(|| source_import_file_failure(&summary)); + let outcome = ProviderFileImportOutcome { + provider, + observation: unit.observation(indexed_at_ms, event_count), + status, + error: outcome_error.as_deref(), + }; + let stored_checkpoint = checkpoint + .as_ref() + .map(|checkpoint| store_checkpoint_from_capture(&unit, checkpoint, indexed_at_ms)) + .transpose()?; + let retain_safe_checkpoint = summary.failed > 0; + if effective_replacement { + let completion = encode_staged_append_completion( + summary, + stored_checkpoint, + source_prefix_sha256, + indexed_at_ms, + )?; + store.stage_provider_file_publication_completion(publication, &completion)?; + store.abandon_provider_file_publication( + scope.take().expect("append publication scope must exist"), + )?; + return Ok(AppendPublicationAttempt::Deferred { + durable_progress: true, + }); + } + let commit = if effective_replacement { + ProviderFilePublicationCommit::Replacement(if retain_safe_checkpoint { + None + } else { + stored_checkpoint.as_ref() + }) + } else if retain_checkpoint || retain_safe_checkpoint { + ProviderFilePublicationCommit::RetainCheckpoint + } else { + ProviderFilePublicationCommit::Append(stored_checkpoint.as_ref().ok_or_else( + || { + anyhow::Error::new(CaptureError::SystemInvariant( + "append import completed without a checkpoint decision", + )) + }, + )?) + }; + let finalized = store.finalize_provider_file_publication( + scope.take().expect("append publication scope must exist"), + outcome, + commit, + )?; + if let Some(warning) = finalized.maintenance_warning { + push_publication_maintenance_warning(&mut summary, warning); + } + queue_current_append_observation(store, &unit, false)?; + Ok(AppendPublicationAttempt::Imported(summary)) + })(); + + match attempt { + Ok(AppendPublicationAttempt::Imported(summary)) => { + return Ok(AppendImportOutcome::Imported(summary)); + } + Ok(AppendPublicationAttempt::Deferred { durable_progress }) => { + return Ok(AppendImportOutcome::Deferred { durable_progress }); + } + Ok(AppendPublicationAttempt::RetryReplacement) => { + replacement = true; + } + Err(error) => { + if let Some(scope) = scope.take() { + match store.abort_provider_file_publication(scope) { + Ok(ControlFlow::Continue(_)) => {} + Ok(ControlFlow::Break(warning)) => { + return Err(publication_recovery_required_error(error, warning)); + } + Err(abort_error) => { + return Err(error.context(format!( + "release failed append publication: {abort_error}" + ))); + } + } + } + return Err(error); + } + } + } + Err(anyhow::Error::new(CaptureError::SystemInvariant( + "append import replacement retry did not converge", + ))) +} + +fn staged_append_completion_matches_current_file( + unit: &AppendInventoryUnit<'_>, + staged: &StagedAppendPublicationCompletion, +) -> Result { + let metadata = fs::symlink_metadata(unit.source_path()) + .with_context(|| format!("stat active append source {}", unit.source_path()))?; + if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { + return Ok(false); + } + if metadata.len() < unit.file_size_bytes() { + return Ok(false); + } + let Some(checkpoint) = staged.checkpoint.as_ref() else { + if !staged.has_accepted_content { + return Ok(true); + } + let Some(expected_sha256) = staged.source_prefix_sha256.as_deref() else { + return Ok(false); + }; + return Ok(sha256_file_prefix_hex( + std::path::Path::new(unit.source_path()), + unit.file_size_bytes(), + )? == expected_sha256); + }; + if checkpoint.import_revision != unit.import_revision() + || checkpoint.committed_byte_offset != unit.file_size_bytes() + { + return Ok(false); + } + let checkpoint = checkpoint.to_capture_checkpoint()?; + provider_jsonl_checkpoint_matches_file(unit.source_path(), &checkpoint).map_err(Into::into) +} + +fn invalidate_active_append_publication_observation( + store: &Store, + unit: &AppendInventoryUnit<'_>, +) -> Result { + let Some(owner) = store.effective_provider_file_publication_inventory_owner()? else { + return Ok(false); + }; + if owner.provider != unit.provider() + || owner.source_format != unit.source_format() + || owner.source_root != unit.source_root() + || owner.source_path != unit.source_path() + || owner.file_size_bytes != unit.file_size_bytes() + || owner.import_revision != unit.import_revision() + { + return Ok(false); + } + store + .invalidate_effective_provider_file_publication_observation( + &owner, + utc_now().timestamp_millis(), + ) + .map_err(Into::into) +} + +fn queue_current_append_observation( + store: &Store, + unit: &AppendInventoryUnit<'_>, + include_source_roots: bool, +) -> Result<()> { + match unit { + AppendInventoryUnit::Catalog { + source, + inventory_generation, + .. + } => { + catalog_codex_session_tree( + &source.path, + store, + CodexSessionCatalogOptions { + source_root: Some(source.path.clone()), + observation_generation: Some(*inventory_generation), + ..CodexSessionCatalogOptions::default() + }, + ) + .with_context(|| { + format!( + "reobserve Codex sessions after publishing {}", + source.path.display() + ) + })?; + } + AppendInventoryUnit::SourceFile { source, work, .. } + if source_uses_import_file_manifest(source) => + { + if let Some(current) = + observe_selected_source_import_file(source, &work.file.source_path)? + { + store.upsert_source_import_files( + unit.inventory_generation(), + std::slice::from_ref(¤t), + )?; + } + } + AppendInventoryUnit::SourceFile { source, .. } if include_source_roots => { + let (_, current) = observe_source_root(source)?; + persist_new_source_import_observation(store, source, std::slice::from_ref(¤t))?; + } + AppendInventoryUnit::SourceFile { .. } => {} + } + Ok(()) +} + +fn load_admitted_append_checkpoint( + store: &Store, + unit: &AppendInventoryUnit<'_>, +) -> Result> { + let Some(checkpoint) = store.provider_file_checkpoint(ProviderFileCheckpointKey { + provider: unit.provider(), + source_format: unit.source_format(), + source_root: unit.source_root(), + source_path: unit.source_path(), + })? + else { + return Ok(None); + }; + if checkpoint.import_revision != unit.import_revision() { + return Ok(None); + } + let Some(stable_identity) = + ProviderFileStableIdentity::from_storage_key(&checkpoint.stable_file_identity) + else { + return Ok(None); + }; + let resume_state = match checkpoint.resume_state { + Some(bytes) => { + let Ok(json) = std::str::from_utf8(&bytes) else { + return Ok(None); + }; + let Ok(state) = ProviderJsonlResumeState::decode_persisted_json(json) else { + return Ok(None); + }; + Some(state) + } + None => None, + }; + Ok(Some( + ProviderAdmittedJsonlAppendCheckpoint::from_persisted_admitted_replacement( + ProviderJsonlAppendCheckpoint { + version: checkpoint.checkpoint_version, + stable_identity, + committed_offset: checkpoint.committed_byte_offset, + complete_line_count: checkpoint.committed_complete_line_count, + head_sha256: checkpoint.head_sha256, + boundary_sha256: checkpoint.boundary_sha256, + resume_state, + }, + ), + )) +} + +fn store_checkpoint_from_capture( + unit: &AppendInventoryUnit<'_>, + checkpoint: &ProviderJsonlAppendCheckpoint, + updated_at_ms: i64, +) -> Result { + let resume_state = checkpoint + .resume_state + .as_ref() + .map(ProviderJsonlResumeState::encode_persisted_json) + .transpose()? + .map(String::into_bytes); + Ok(ProviderFileCheckpoint { + provider: unit.provider(), + source_format: unit.source_format().to_owned(), + source_root: unit.source_root().to_owned(), + source_path: unit.source_path().to_owned(), + import_revision: unit.import_revision(), + checkpoint_version: checkpoint.version, + stable_file_identity: checkpoint.stable_identity.to_storage_key(), + committed_byte_offset: checkpoint.committed_offset, + committed_complete_line_count: checkpoint.complete_line_count, + head_sha256: checkpoint.head_sha256.clone(), + boundary_sha256: checkpoint.boundary_sha256.clone(), + resume_state, + updated_at_ms, + }) +} + +fn encode_staged_append_completion( + summary: ProviderImportSummary, + checkpoint: Option, + source_prefix_sha256: Option, + indexed_at_ms: i64, +) -> Result { + let has_accepted_content = summary.has_accepted_content(); + if has_accepted_content && checkpoint.is_none() && source_prefix_sha256.is_none() { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "accepted staged append completion has no content certificate", + ))); + } + let staged = StagedAppendPublicationCompletion { + summary, + has_accepted_content, + checkpoint: checkpoint.map(StagedProviderFileCheckpoint::from_store_checkpoint), + source_prefix_sha256, + indexed_at_ms, + }; + Ok(ProviderFilePublicationCompletion { + version: STAGED_APPEND_PUBLICATION_VERSION, + payload: serde_json::to_value(staged).context("encode staged append completion")?, + }) +} + +fn decode_staged_append_completion( + completion: ProviderFilePublicationCompletion, +) -> Result { + if completion.version != STAGED_APPEND_PUBLICATION_VERSION { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "unsupported staged append publication version", + ))); + } + serde_json::from_value(completion.payload).context("decode staged append completion") +} + +impl StagedProviderFileCheckpoint { + fn from_store_checkpoint(checkpoint: ProviderFileCheckpoint) -> Self { + Self { + import_revision: checkpoint.import_revision, + checkpoint_version: checkpoint.checkpoint_version, + stable_file_identity: checkpoint.stable_file_identity, + committed_byte_offset: checkpoint.committed_byte_offset, + committed_complete_line_count: checkpoint.committed_complete_line_count, + head_sha256: checkpoint.head_sha256, + boundary_sha256: checkpoint.boundary_sha256, + resume_state_base64: checkpoint.resume_state.map(|bytes| BASE64.encode(bytes)), + updated_at_ms: checkpoint.updated_at_ms, + } + } + + fn into_store_checkpoint( + self, + unit: &AppendInventoryUnit<'_>, + ) -> Result { + let resume_state = self + .resume_state_base64 + .map(|value| { + BASE64 + .decode(value) + .context("decode staged append resume state") + }) + .transpose()?; + Ok(ProviderFileCheckpoint { + provider: unit.provider(), + source_format: unit.source_format().to_owned(), + source_root: unit.source_root().to_owned(), + source_path: unit.source_path().to_owned(), + import_revision: self.import_revision, + checkpoint_version: self.checkpoint_version, + stable_file_identity: self.stable_file_identity, + committed_byte_offset: self.committed_byte_offset, + committed_complete_line_count: self.committed_complete_line_count, + head_sha256: self.head_sha256, + boundary_sha256: self.boundary_sha256, + resume_state, + updated_at_ms: self.updated_at_ms, + }) + } + + fn to_capture_checkpoint(&self) -> Result { + let stable_identity = ProviderFileStableIdentity::from_storage_key( + &self.stable_file_identity, + ) + .ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "staged append checkpoint has invalid file identity", + )) + })?; + let resume_state = self + .resume_state_base64 + .as_ref() + .map(|value| { + let bytes = BASE64 + .decode(value) + .context("decode staged append resume state")?; + let json = std::str::from_utf8(&bytes) + .context("decode staged append resume state as UTF-8")?; + ProviderJsonlResumeState::decode_persisted_json(json) + .context("decode staged append resume state payload") + }) + .transpose()?; + Ok(ProviderJsonlAppendCheckpoint { + version: self.checkpoint_version, + stable_identity, + committed_offset: self.committed_byte_offset, + complete_line_count: self.committed_complete_line_count, + head_sha256: self.head_sha256.clone(), + boundary_sha256: self.boundary_sha256.clone(), + resume_state, + }) + } +} + +fn provider_import_summary_failure_for_unit( + provider: CaptureProvider, + source_path: &str, + summary: &ProviderImportSummary, +) -> anyhow::Error { + let detail = summary + .failures + .first() + .map(|failure| format!("line {}: {}", failure.line, failure.error)) + .unwrap_or_else(|| "unknown provider import failure".to_owned()); + rejected_source_error( + format!( + "import {} source {} failed with {} failure(s); first failure: {detail}", + provider.as_str(), + source_path, + summary.failed + ), + summary, + ) +} diff --git a/crates/ctx-cli/src/commands/import/native/batching.rs b/crates/ctx-cli/src/commands/import/native/batching.rs index ac951beb..ba9314de 100644 --- a/crates/ctx-cli/src/commands/import/native/batching.rs +++ b/crates/ctx-cli/src/commands/import/native/batching.rs @@ -4,26 +4,164 @@ fn import_one_source_inner_batched( progress: Option, full_rescan: bool, preinventory: &SourcePreinventory, -) -> Result { + selection: Option<&SelectedImportWork>, +) -> Result { if source.provider == CaptureProvider::Codex { super::catalog::codex_catalog_root_identity(&source.path)?; } let record = import_record_for_source(source); let record_id = record.id; - let record_existed = history_record_exists(store, record_id)?; - store.upsert_record(&record)?; + let record_existed = + history_record_exists(store, record_id).context("inspect source import history record")?; + let selected_append_publication = match selection { + Some(SelectedImportWork::Catalog(_)) => true, + Some(SelectedImportWork::SourceFiles(_)) => { + provider_file_mutation_contract(source.provider, source.source_format) + == ProviderFileMutationContract::AppendOnlyNewlineDelimited + } + None => false, + }; + if !record_existed && !selected_append_publication { + store.upsert_record(&record)?; + } + if !source_uses_import_file_manifest(source) + && provider_file_mutation_contract(source.provider, source.source_format) + == ProviderFileMutationContract::AppendOnlyNewlineDelimited + { + if let Some(SelectedImportWork::SourceFiles(work)) = selection { + if work.len() != 1 { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "single-file append source selected more than one inventory unit", + ))); + } + let import_result = import_append_capable_source_file_work( + store, + source, + &work[0], + preinventory.inventory_generation().ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "selected append source has no inventory generation", + )) + })?, + &record, + ) + .context("run selected append publication"); + return match import_result { + Ok(AppendImportOutcome::Imported(summary)) => { + let status = provider_summary_import_status(&summary); + let error = (summary.failed > 0).then(|| source_import_file_failure(&summary)); + let post_import = persist_reobserved_source_root_result( + store, + source, + preinventory, + status, + error.as_deref().unwrap_or(""), + )?; + Ok(ProviderImportBatchOutcome { + summary, + completed_units: post_import + .as_ref() + .map_or(0, |observation| observation.persisted_current_outcomes), + completed_bytes: post_import.as_ref().map_or(0, |observation| { + if observation.persisted_current_outcomes > 0 { + work[0].estimated_bytes + } else { + 0 + } + }), + deferred_units: 0, + durable_progress: false, + post_import_inventory_generation: post_import + .as_ref() + .map(|observation| observation.inventory_generation), + post_import_preinventory: post_import + .map(|observation| observation.preinventory), + }) + } + Ok(AppendImportOutcome::Deferred { durable_progress }) => { + if !record_existed && !durable_progress { + store.delete_orphan_record(record_id)?; + } + let post_import = if durable_progress { + None + } else { + persist_reobserved_source_root_result( + store, + source, + preinventory, + CatalogIndexedStatus::Pending, + "", + )? + }; + Ok(ProviderImportBatchOutcome { + summary: ProviderImportSummary::default(), + completed_units: 0, + completed_bytes: 0, + deferred_units: 1, + durable_progress, + post_import_inventory_generation: post_import + .as_ref() + .map(|observation| observation.inventory_generation), + post_import_preinventory: post_import + .map(|observation| observation.preinventory), + }) + } + Err(error) => { + if publication_recovery_required(&error) { + return Err(error); + } + let observation_result = persist_reobserved_source_root_result( + store, + source, + preinventory, + import_error_status(&error), + &error.to_string(), + ); + if let Some(observation_error) = + final_observation_system_error(observation_result) + { + return Err(observation_error); + } + cleanup_rejected_history_record(store, record_id, record_existed)?; + Err(error) + } + }; + } + } + let mut completed_units = 0; + let mut completed_bytes = 0_u64; + let mut deferred_units = 0; + let mut durable_progress = false; + let mut post_import_inventory_generation = None; + let mut post_import_preinventory = None; let summary = if source_uses_import_file_manifest(source) && (source.path.is_dir() || preinventory.source_import_files().is_some()) { import_manifested_source( store, source, - record_id, progress, - preinventory.source_import_files(), - preinventory.inventory_generation(), - full_rescan, + ManifestedImportOptions::new( + preinventory.source_import_files(), + preinventory.inventory_generation(), + full_rescan, + selection, + ), ) + .map(|outcome| { + completed_units = outcome.completed_units; + completed_bytes = outcome.completed_bytes; + deferred_units = outcome.deferred_units; + durable_progress = outcome.durable_progress; + post_import_inventory_generation = outcome.post_import_inventory_generation; + post_import_preinventory = outcome.post_import_preinventory.or_else(|| { + (outcome.deferred_units == 0 + && outcome.post_import_inventory_generation + == preinventory.inventory_generation()) + .then(|| preinventory.clone()) + }); + outcome.summary + }) } else { match source.provider { CaptureProvider::Codex => { @@ -31,12 +169,22 @@ fn import_one_source_inner_batched( import_incremental_codex_session_tree( store, source, - record_id, + &record, progress.clone(), preinventory.codex_session_catalog(), preinventory.inventory_generation(), full_rescan, + selection, ) + .map(|outcome| { + completed_units = outcome.completed_units; + completed_bytes = outcome.completed_bytes; + deferred_units = outcome.deferred_units; + durable_progress = outcome.durable_progress; + post_import_inventory_generation = outcome.post_import_inventory_generation; + post_import_preinventory = outcome.post_import_preinventory; + outcome.summary + }) } else if source .path .file_name() @@ -475,6 +623,17 @@ fn import_one_source_inner_batched( }; let summary = match summary { Ok(mut summary) => { + if deferred_units > 0 { + return Ok(ProviderImportBatchOutcome { + summary, + completed_units, + completed_bytes, + deferred_units, + durable_progress, + post_import_inventory_generation, + post_import_preinventory, + }); + } // A manifested-source retry can contain only rejected files even though earlier // files under the same stable history record are already indexed. Preserve that // as a completed source with rejections; an orphan record is still cleaned up and @@ -506,19 +665,29 @@ fn import_one_source_inner_batched( } let status = provider_summary_import_status(&summary); let error = (summary.failed > 0).then(|| source_import_file_failure(&summary)); - persist_reobserved_source_root_result( + if let Some(post_import) = persist_reobserved_source_root_result( store, source, preinventory, status, error.as_deref().unwrap_or(""), - )?; + )? { + completed_units = post_import.persisted_current_outcomes; + post_import_inventory_generation = Some(post_import.inventory_generation); + post_import_preinventory = Some(post_import.preinventory); + } if !record_existed && summary == ProviderImportSummary::default() { store.delete_orphan_record(record_id)?; } summary } Err(err) => { + if err.downcast_ref::().is_some() { + return Err(err); + } + if publication_recovery_required(&err) { + return Err(err); + } let failure_scope = import_error_scope(&err); let status = import_error_status(&err); let observation_result = persist_reobserved_source_root_result( @@ -531,10 +700,13 @@ fn import_one_source_inner_batched( if let Some(observation_error) = final_observation_system_error(observation_result) { return Err(observation_error); } - let deleted = store.delete_orphan_record(record_id)?; + let deleted = store.delete_orphan_record(record_id).with_context(|| { + format!("clean up history record after provider import failed: {err:#}") + })?; if failure_scope == ImportFailureScope::Source && !deleted && !record_existed + && !source_uses_import_file_manifest(source) && history_record_exists(store, record_id)? { return Err(anyhow::Error::new(CaptureError::SystemInvariant( @@ -544,7 +716,22 @@ fn import_one_source_inner_batched( return Err(err); } }; - Ok(summary) + Ok(ProviderImportBatchOutcome { + summary, + completed_units, + completed_bytes, + deferred_units, + durable_progress, + post_import_inventory_generation, + post_import_preinventory, + }) +} + +#[derive(Debug)] +struct PostImportInventoryObservation { + inventory_generation: u64, + persisted_current_outcomes: usize, + preinventory: SourcePreinventory, } fn persist_reobserved_source_root_result( @@ -553,9 +740,9 @@ fn persist_reobserved_source_root_result( preinventory: &SourcePreinventory, status: CatalogIndexedStatus, error: &str, -) -> Result<()> { +) -> Result> { let Some((observed, _)) = preinventory.source_root_observation() else { - return Ok(()); + return Ok(None); }; let (_, current) = observe_source_root(source)?; let outcomes = same_source_import_observation(observed, ¤t) @@ -566,16 +753,23 @@ fn persist_reobserved_source_root_result( }) .into_iter() .collect::>(); - persist_source_import_observation_with_outcomes( + let persisted = persist_source_import_observation_with_outcomes( store, source, std::slice::from_ref(¤t), &outcomes, )?; - Ok(()) + Ok(Some(PostImportInventoryObservation { + inventory_generation: persisted.inventory_generation, + persisted_current_outcomes: outcomes.len(), + preinventory: SourcePreinventory::SourceRoot { + file: current, + inventory_generation: persisted.inventory_generation, + }, + })) } -fn final_observation_system_error(observation_result: Result<()>) -> Option { +fn final_observation_system_error(observation_result: Result) -> Option { observation_result .err() .filter(|error| import_error_scope(error) == ImportFailureScope::System) diff --git a/crates/ctx-cli/src/commands/import/native/manifested.rs b/crates/ctx-cli/src/commands/import/native/manifested.rs index 8b3ebcc2..7a911725 100644 --- a/crates/ctx-cli/src/commands/import/native/manifested.rs +++ b/crates/ctx-cli/src/commands/import/native/manifested.rs @@ -1,12 +1,9 @@ -pub(crate) fn import_manifested_source( +fn import_manifested_source( store: &mut Store, source: &SourceInfo, - record_id: Uuid, progress: Option, - preinventoried_files: Option<&[SourceImportFile]>, - preinventory_generation: Option, - force_selection: bool, -) -> Result { + options: ManifestedImportOptions<'_>, +) -> Result { let mut import_file = |store: &mut Store, pending_source: &SourceInfo| { import_one_source_inner( store, @@ -17,15 +14,31 @@ pub(crate) fn import_manifested_source( &SourcePreinventory::None, ) }; - import_manifested_source_with_importer( - store, - source, - record_id, - preinventoried_files, - preinventory_generation, - force_selection, - &mut import_file, - ) + import_manifested_source_with_importer(store, source, options, &mut import_file) +} + +#[derive(Clone, Copy)] +struct ManifestedImportOptions<'a> { + preinventoried_files: Option<&'a [SourceImportFile]>, + preinventory_generation: Option, + force_selection: bool, + selection: Option<&'a SelectedImportWork>, +} + +impl<'a> ManifestedImportOptions<'a> { + fn new( + preinventoried_files: Option<&'a [SourceImportFile]>, + preinventory_generation: Option, + force_selection: bool, + selection: Option<&'a SelectedImportWork>, + ) -> Self { + Self { + preinventoried_files, + preinventory_generation, + force_selection, + selection, + } + } } struct ManifestedImportOutcome { @@ -44,48 +57,109 @@ enum ManifestedImportResult { fn import_manifested_source_with_importer( store: &mut Store, source: &SourceInfo, - record_id: Uuid, - preinventoried_files: Option<&[SourceImportFile]>, - _preinventory_generation: Option, - force_selection: bool, + options: ManifestedImportOptions<'_>, import_file: &mut dyn FnMut(&mut Store, &SourceInfo) -> Result, -) -> Result { +) -> Result { + let ManifestedImportOptions { + preinventoried_files, + preinventory_generation, + force_selection, + selection, + } = options; let source_root = persisted_import_identity(&source.path, "source root")?.to_owned(); let collected_files; + let mut outcome_generation = preinventory_generation; let files = match preinventoried_files { Some(files) => files, None => { collected_files = collect_source_import_files(source).with_context(|| { format!("inventory import files from {}", source.path.display()) })?; - persist_new_source_import_observation(store, source, &collected_files)?; + let persisted = persist_new_source_import_observation(store, source, &collected_files)?; + outcome_generation = Some(persisted.inventory_generation); &collected_files } }; - if files.is_empty() { + if files.is_empty() && selection.is_none() { return Err(anyhow!( "no importable {} history files found under {}", source.provider.as_str(), source.path.display() )); } - let selected_files = if force_selection { - files.to_vec() - } else { - store.list_pending_source_import_files(source.provider, &source_root)? + let selected_files = match selection { + Some(SelectedImportWork::SourceFiles(work)) => { + work.iter().map(|work| work.file.clone()).collect() + } + Some(SelectedImportWork::Catalog(_)) => { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "catalog work selected for a manifested source", + ))) + } + None if force_selection => files.to_vec(), + None => store.list_pending_source_import_files(source.provider, &source_root)?, }; if selected_files.is_empty() { - return Ok(ProviderImportSummary::default()); + return Ok(ProviderImportBatchOutcome::completed( + ProviderImportSummary::default(), + 0, + )); } + let selected_work_by_path = match selection { + Some(SelectedImportWork::SourceFiles(work)) => work + .iter() + .map(|work| (work.file.source_path.as_str(), work)) + .collect::>(), + Some(SelectedImportWork::Catalog(_)) | None => BTreeMap::new(), + }; let mut summary = ProviderImportSummary::default(); + let mut deferred_units = 0; + let mut durable_progress = false; let mut outcomes = Vec::with_capacity(selected_files.len()); let mut system_error = None; + let append_only = provider_file_mutation_contract(source.provider, source.source_format) + == ProviderFileMutationContract::AppendOnlyNewlineDelimited; for pending_file in selected_files { let path = PathBuf::from(&pending_file.source_path); let mut pending_source = explicit_path_source(source.provider, path); pending_source.source_format = source.source_format; - let imported = import_file(store, &pending_source); + let uses_bounded_publication = + append_only && selected_work_by_path.contains_key(pending_file.source_path.as_str()); + let imported = if append_only { + match selected_work_by_path.get(pending_file.source_path.as_str()) { + Some(work) => match import_manifested_append_source_file_work( + store, + source, + &pending_source, + work, + preinventory_generation.ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "selected source-file work has no inventory generation", + )) + })?, + ) { + Ok(AppendImportOutcome::Imported(summary)) => Some(Ok(summary)), + Ok(AppendImportOutcome::Deferred { + durable_progress: unit_progress, + }) => { + deferred_units += 1; + durable_progress |= unit_progress; + if unit_progress { + break; + } + continue; + } + Err(error) => Some(Err(error)), + }, + None => Some(import_file(store, &pending_source)), + } + } else { + Some(import_file(store, &pending_source)) + }; + let Some(imported) = imported else { + continue; + }; match imported { Ok(file_summary) => { let status = provider_summary_import_status(&file_summary); @@ -99,6 +173,12 @@ fn import_manifested_source_with_importer( }); } Err(err) => { + if publication_recovery_required(&err) { + if system_error.is_none() { + system_error = Some(err); + } + continue; + } if let Some(file_summary) = rejected_source_summary(&err) { let status = provider_summary_import_status(&file_summary); let error = (file_summary.failed > 0) @@ -134,79 +214,159 @@ fn import_manifested_source_with_importer( } } } + if uses_bounded_publication { + break; + } } - let persisted_outcomes = persist_reobserved_manifested_outcomes(store, source, &outcomes); - if let Some(error) = system_error { - return Err(error); + if deferred_units > 0 && durable_progress { + return Ok(ProviderImportBatchOutcome { + summary, + completed_units: 0, + completed_bytes: 0, + deferred_units, + durable_progress, + post_import_inventory_generation: None, + post_import_preinventory: None, + }); } - let current_outcomes = persisted_outcomes?; + let outcome_generation = outcome_generation.ok_or_else(|| { + anyhow::Error::new(CaptureError::SystemInvariant( + "manifested import outcomes have no inventory generation", + )) + })?; + let persisted = match persist_reobserved_manifested_outcomes( + store, + source, + outcome_generation, + &outcomes, + ) { + Ok(persisted) => persisted, + Err(persist_error) => { + let mut partial_summary = ProviderImportSummary::default(); + for outcome in &outcomes { + if let ManifestedImportResult::Imported(file_summary) = &outcome.result { + partial_summary.merge_from(file_summary.clone()); + } + } + return Err(provider_import_batch_error( + ProviderImportBatchOutcome { + summary: partial_summary, + completed_units: 0, + completed_bytes: 0, + deferred_units, + durable_progress, + post_import_inventory_generation: None, + post_import_preinventory: None, + }, + system_error.unwrap_or(persist_error), + )); + } + }; + durable_progress |= persisted.durable_progress; let mut source_error = None; + let mut completed_paths = BTreeSet::new(); for outcome in outcomes { + let source_path = outcome.observation.source_path.clone(); match outcome.result { - ManifestedImportResult::Imported(file_summary) => summary.merge_from(file_summary), + ManifestedImportResult::Imported(file_summary) => { + if persisted.current_outcomes.contains(&source_path) { + completed_paths.insert(source_path); + } + summary.merge_from(file_summary); + } ManifestedImportResult::SourceFailure(error) - if current_outcomes.contains(&outcome.observation.source_path) => + if persisted + .current_outcomes + .contains(&outcome.observation.source_path) => { - if source_error.is_none() { + if import_error_retryability(&error) == ImportRetryability::Terminal { + completed_paths.insert(source_path.clone()); + summary.failed = summary.failed.saturating_add(1); + summary.failures.push(ProviderImportFailure { + line: 0, + error: format!("{source_path}: {}", error_summary(&error)), + }); + } else if source_error.is_none() { source_error = Some(error); } } ManifestedImportResult::SourceFailure(_) | ManifestedImportResult::SystemFailure => {} } } - let _ = record_id; + let completed_bytes = completed_paths + .iter() + .filter_map(|source_path| selected_work_by_path.get(source_path.as_str())) + .fold(0_u64, |total, work| { + total.saturating_add(work.estimated_bytes) + }); + let outcome = ProviderImportBatchOutcome { + summary, + completed_units: completed_paths.len(), + completed_bytes, + deferred_units, + durable_progress, + post_import_inventory_generation: Some(persisted.inventory_generation), + post_import_preinventory: None, + }; + if let Some(error) = system_error { + return Err(provider_import_batch_error(outcome, error)); + } if let Some(error) = source_error { - return Err(error); + return Err(provider_import_batch_error(outcome, error)); } - Ok(summary) + Ok(outcome) +} + +struct PersistedManifestedOutcomes { + current_outcomes: BTreeSet, + inventory_generation: u64, + durable_progress: bool, } fn persist_reobserved_manifested_outcomes( store: &Store, source: &SourceInfo, + inventory_generation: u64, outcomes: &[ManifestedImportOutcome], -) -> Result> { - let current_files = collect_source_import_files(source) - .with_context(|| format!("re-inventory import files from {}", source.path.display()))?; - let current_by_path = current_files - .iter() - .map(|file| (file.source_path.as_str(), file)) - .collect::>(); - let mut persisted_outcomes = Vec::new(); +) -> Result { + let mut current_outcomes = BTreeSet::new(); + let mut durable_progress = false; for outcome in outcomes { - let Some(current) = current_by_path.get(outcome.observation.source_path.as_str()) else { + let Some(current) = + observe_selected_source_import_file(source, &outcome.observation.source_path)? + else { continue; }; - if !same_source_import_observation(&outcome.observation, current) { + if !same_source_import_observation(&outcome.observation, ¤t) { + store + .upsert_source_import_files(inventory_generation, std::slice::from_ref(¤t))?; + durable_progress = true; continue; } - persisted_outcomes.push(SourceImportObservationOutcome { - file: current, - status: outcome.status, - error: outcome.error.as_deref(), - }); + let changed = store.record_source_import_file_result( + current.provider, + SourceImportFileIndexUpdate { + source_root: ¤t.source_root, + source_path: ¤t.source_path, + file_size_bytes: current.file_size_bytes, + file_modified_at_ms: current.file_modified_at_ms, + import_revision: current.import_revision, + inventory_generation, + metadata: ¤t.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + outcome.status, + outcome.error.as_deref(), + )?; + if changed == 1 { + current_outcomes.insert(current.source_path); + } } - persist_source_import_observation_with_outcomes( - store, - source, - ¤t_files, - &persisted_outcomes, - )?; - Ok(persisted_outcomes - .iter() - .map(|outcome| outcome.file.source_path.clone()) - .collect()) -} - -fn same_source_import_observation(left: &SourceImportFile, right: &SourceImportFile) -> bool { - left.provider == right.provider - && left.source_format == right.source_format - && left.source_root == right.source_root - && left.source_path == right.source_path - && left.file_size_bytes == right.file_size_bytes - && left.file_modified_at_ms == right.file_modified_at_ms - && left.import_revision == right.import_revision - && left.metadata == right.metadata + Ok(PersistedManifestedOutcomes { + current_outcomes, + inventory_generation, + durable_progress, + }) } fn import_error_status(error: &anyhow::Error) -> CatalogIndexedStatus { diff --git a/crates/ctx-cli/src/commands/import/native/selection.rs b/crates/ctx-cli/src/commands/import/native/selection.rs index e8115f8e..7d5859e9 100644 --- a/crates/ctx-cli/src/commands/import/native/selection.rs +++ b/crates/ctx-cli/src/commands/import/native/selection.rs @@ -14,51 +14,106 @@ pub(crate) fn validate_source_import_supported(source: &SourceInfo) -> Result<() } } -pub(crate) fn import_one_source( +#[cfg(test)] +fn import_one_source_for_search_refresh( store: &mut Store, source: &SourceInfo, progress: Option, - full_rescan: bool, preinventory: &SourcePreinventory, ) -> Result { - let event_search_needs_backfill = store.event_search_projection_needs_backfill()?; - let refresh_search_after_import = - event_search_needs_backfill || !source_uses_incremental_event_search(source); + let refresh_search_after_import = store.event_search_projection_needs_backfill()?; import_one_source_inner( store, source, progress, refresh_search_after_import, - full_rescan, + false, preinventory, ) } -pub(crate) fn import_one_source_without_search_refresh( - store: &mut Store, - source: &SourceInfo, - progress: Option, - full_rescan: bool, - preinventory: &SourcePreinventory, -) -> Result { - import_one_source_inner(store, source, progress, false, full_rescan, preinventory) -} - -pub(crate) fn import_one_source_for_search_refresh( +pub(crate) fn import_selected_source( store: &mut Store, source: &SourceInfo, progress: Option, preinventory: &SourcePreinventory, -) -> Result { - let refresh_search_after_import = store.event_search_projection_needs_backfill()?; - import_one_source_inner( + selection: &SelectedImportWork, +) -> Result { + let (outcome, remaining_error) = match import_one_source_inner_batched( store, source, progress, - refresh_search_after_import, false, preinventory, - ) + Some(selection), + ) { + Ok(outcome) => (outcome, None), + Err(error) => match error.downcast::() { + Ok(error) => { + let (outcome, source) = error.into_parts(); + (outcome, Some(source)) + } + Err(error) => return Err(error), + }, + }; + let (selected_units, selected_bytes) = match selection { + SelectedImportWork::Catalog(work) => ( + work.len(), + work.iter().fold(0_u64, |total, work| { + total.saturating_add(work.estimated_bytes) + }), + ), + SelectedImportWork::SourceFiles(work) => ( + work.len(), + work.iter().fold(0_u64, |total, work| { + total.saturating_add(work.estimated_bytes) + }), + ), + }; + let mut summary = outcome.summary; + if selected_work_is_explicit_rescan(selection) { + summary.skipped = summary.skipped.saturating_add(summary.imported); + summary.imported = 0; + summary.skipped_sessions = summary + .skipped_sessions + .saturating_add(summary.imported_sessions); + summary.imported_sessions = 0; + summary.skipped_events = summary + .skipped_events + .saturating_add(summary.imported_events); + summary.imported_events = 0; + summary.skipped_edges = summary.skipped_edges.saturating_add(summary.imported_edges); + summary.imported_edges = 0; + } + Ok(SelectedSourceImportResult { + outcome: SelectedSourceImportOutcome { + summary, + completed_units: outcome.completed_units, + completed_bytes: if outcome.completed_units == selected_units + && outcome.deferred_units == 0 + { + selected_bytes + } else { + outcome.completed_bytes + }, + deferred_units: outcome.deferred_units, + durable_progress: outcome.durable_progress, + post_import_inventory_generation: outcome.post_import_inventory_generation, + post_import_preinventory: outcome.post_import_preinventory, + }, + remaining_error, + }) +} + +fn selected_work_is_explicit_rescan(selection: &SelectedImportWork) -> bool { + match selection { + SelectedImportWork::Catalog(work) => work + .iter() + .all(|unit| unit.reason == ImportPendingReason::ExplicitRescan), + SelectedImportWork::SourceFiles(work) => work + .iter() + .all(|unit| unit.reason == ImportPendingReason::ExplicitRescan), + } } pub(crate) fn import_one_source_inner( @@ -100,6 +155,8 @@ fn import_one_source_inner_with_pre_lock_hook( let import_result = (|| { let mut revalidated = if full_rescan { RevalidatedSourcePreinventory::Import(preinventory.clone()) + } else if preinventory_is_complete(store, source, preinventory)? { + RevalidatedSourcePreinventory::Complete } else { revalidate_source_preinventory(store, source, preinventory)? }; @@ -115,11 +172,12 @@ fn import_one_source_inner_with_pre_lock_hook( progress.clone(), full_rescan, current, + None, ) { Err(error) if is_inventory_superseded(&error) => { revalidated = revalidate_source_preinventory(store, source, current)?; } - result => return result, + result => return result.map(|outcome| outcome.summary), } } } @@ -133,9 +191,14 @@ fn import_one_source_inner_with_pre_lock_hook( })(); let finish_result = store.finish_event_search_bulk_mode(&bulk_guard); let summary = match (import_result, finish_result) { - (Ok(summary), Ok(())) => Ok(summary), + (Ok(summary), Ok(ctx_history_store::EventSearchBulkMaintenanceOutcome::Complete)) => { + Ok(summary) + } + (Ok(summary), Ok(ctx_history_store::EventSearchBulkMaintenanceOutcome::Pending)) => { + Ok(summary) + } (_, Err(error)) => Err(error.into()), - (Err(error), Ok(())) => Err(error), + (Err(error), Ok(_)) => Err(error), }?; if refresh_search_after_import { store.refresh_search_index()?; diff --git a/crates/ctx-cli/src/commands/import/native/types_and_recovery.rs b/crates/ctx-cli/src/commands/import/native/types_and_recovery.rs new file mode 100644 index 00000000..23326bab --- /dev/null +++ b/crates/ctx-cli/src/commands/import/native/types_and_recovery.rs @@ -0,0 +1,444 @@ +#[derive(Debug, Default)] +pub(crate) struct ProviderFileRetirementRecoveryOutcome { + pub(crate) completed: bool, + pub(crate) made_durable_progress: bool, + pub(crate) maintenance_warnings: Vec, +} + +pub(crate) fn recover_provider_file_publication_retirement( + store: &Store, + work: &ProviderFilePublicationRetirementWork, + drain: bool, +) -> Result { + let mut outcome = ProviderFileRetirementRecoveryOutcome::default(); + loop { + let Some(scope) = store.begin_provider_file_publication_retirement( + work.provider, + &work.material_source_format, + &work.material_source_root, + &work.source_path, + utc_now().timestamp_millis(), + )? + else { + outcome.completed = true; + return Ok(outcome); + }; + match store.provider_file_publication_phase(&scope)? { + ProviderFilePublicationPhase::Preparing => { + let preparation = store.prepare_provider_file_publication_slice( + &scope, + PROVIDER_RETIREMENT_SLICE_ROWS, + )?; + outcome.made_durable_progress |= + preparation.rows_processed > 0 || preparation.complete; + if let Some(warning) = store.abandon_provider_file_publication(scope)? { + outcome.maintenance_warnings.push(warning); + } + if drain { + continue; + } + return Ok(outcome); + } + ProviderFilePublicationPhase::Reconciling => { + let reconciliation = store.reconcile_provider_file_publication_slice( + &scope, + PROVIDER_RETIREMENT_SLICE_ROWS, + )?; + outcome.made_durable_progress |= reconciliation.rows_scanned > 0; + if let Some(warning) = store.abandon_provider_file_publication(scope)? { + outcome.maintenance_warnings.push(warning); + } + if drain { + continue; + } + return Ok(outcome); + } + ProviderFilePublicationPhase::ReadyToFinalize => {} + ProviderFilePublicationPhase::Importing => { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "retirement publication entered importer phase", + ))) + } + } + let finalized = store.retire_provider_file_publication(scope)?; + if let Some(warning) = finalized.maintenance_warning { + outcome.maintenance_warnings.push(warning); + } + outcome.completed = true; + outcome.made_durable_progress = true; + return Ok(outcome); + } +} + +#[derive(Debug)] +pub(crate) struct SelectedSourceImportOutcome { + pub(crate) summary: ProviderImportSummary, + pub(crate) completed_units: usize, + pub(crate) completed_bytes: u64, + pub(crate) deferred_units: usize, + pub(crate) durable_progress: bool, + #[allow(dead_code)] + pub(crate) post_import_inventory_generation: Option, + pub(crate) post_import_preinventory: Option, +} + +impl SelectedSourceImportOutcome { + pub(crate) fn made_durable_progress(&self) -> bool { + self.durable_progress || self.completed_units > 0 || self.summary.has_accepted_content() + } +} + +impl Deref for SelectedSourceImportOutcome { + type Target = ProviderImportSummary; + + fn deref(&self) -> &Self::Target { + &self.summary + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ProviderImportBatchOutcome { + pub(crate) summary: ProviderImportSummary, + pub(crate) completed_units: usize, + pub(crate) completed_bytes: u64, + pub(crate) deferred_units: usize, + pub(crate) durable_progress: bool, + pub(crate) post_import_inventory_generation: Option, + pub(crate) post_import_preinventory: Option, +} + +impl ProviderImportBatchOutcome { + pub(crate) fn completed(summary: ProviderImportSummary, completed_units: usize) -> Self { + Self { + summary, + completed_units, + completed_bytes: 0, + deferred_units: 0, + durable_progress: false, + post_import_inventory_generation: None, + post_import_preinventory: None, + } + } + + fn made_durable_progress(&self) -> bool { + self.durable_progress || self.completed_units > 0 || self.summary.has_accepted_content() + } +} + +impl Deref for ProviderImportBatchOutcome { + type Target = ProviderImportSummary; + + fn deref(&self) -> &Self::Target { + &self.summary + } +} + +#[derive(Debug)] +pub(crate) struct ProviderImportBatchError { + outcome: ProviderImportBatchOutcome, + source: anyhow::Error, +} + +impl ProviderImportBatchError { + fn into_parts(self) -> (ProviderImportBatchOutcome, anyhow::Error) { + (self.outcome, self.source) + } +} + +impl std::fmt::Display for ProviderImportBatchError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.source.fmt(formatter) + } +} + +impl std::error::Error for ProviderImportBatchError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +pub(crate) fn provider_import_batch_error( + outcome: ProviderImportBatchOutcome, + source: anyhow::Error, +) -> anyhow::Error { + if !outcome.made_durable_progress() { + return source; + } + anyhow::Error::new(ProviderImportBatchError { outcome, source }) +} + +#[derive(Debug)] +pub(crate) struct SelectedSourceImportResult { + pub(crate) outcome: SelectedSourceImportOutcome, + pub(crate) remaining_error: Option, +} + +impl Deref for SelectedSourceImportResult { + type Target = SelectedSourceImportOutcome; + + fn deref(&self) -> &Self::Target { + &self.outcome + } +} + +#[allow(dead_code)] +#[derive(Debug)] +struct PublicationRecoveryRequiredError { + source: anyhow::Error, + maintenance_warning: Option, +} + +impl std::fmt::Display for PublicationRecoveryRequiredError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.source.fmt(formatter) + } +} + +impl std::error::Error for PublicationRecoveryRequiredError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +fn publication_recovery_required_error( + source: anyhow::Error, + maintenance_warning: Option, +) -> anyhow::Error { + anyhow::Error::new(PublicationRecoveryRequiredError { + source, + maintenance_warning, + }) +} + +pub(crate) fn publication_recovery_required(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some() + }) +} + +#[allow(dead_code)] +pub(crate) fn publication_recovery_maintenance_warning( + error: &anyhow::Error, +) -> Option<&ProviderFileMaintenanceWarning> { + error.chain().find_map(|cause| { + cause + .downcast_ref::() + .and_then(|error| error.maintenance_warning.as_ref()) + }) +} + +fn push_publication_maintenance_warning( + summary: &mut ProviderImportSummary, + warning: ProviderFileMaintenanceWarning, +) { + summary + .maintenance_warnings + .push(ProviderImportMaintenanceWarning { + kind: ProviderImportMaintenanceKind::ImportInterruptedAfterCommit, + error: warning.to_string(), + }); +} + +#[cfg(test)] +thread_local! { + static APPEND_SOURCE_FAILURE_AFTER_MUTATION: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +fn inject_append_source_failure_after_mutation() { + APPEND_SOURCE_FAILURE_AFTER_MUTATION.with(|fault| fault.set(true)); +} + +#[cfg(test)] +fn take_append_source_failure_after_mutation() -> bool { + APPEND_SOURCE_FAILURE_AFTER_MUTATION.with(|fault| fault.replace(false)) +} + +enum AppendInventoryUnit<'a> { + Catalog { + source: &'a SourceInfo, + work: &'a CatalogImportWork, + inventory_generation: u64, + }, + SourceFile { + source: &'a SourceInfo, + work: &'a SourceImportFileWork, + inventory_generation: u64, + }, +} + +pub(crate) enum AppendImportOutcome { + Imported(ProviderImportSummary), + Deferred { durable_progress: bool }, +} + +enum AppendPublicationAttempt { + Imported(ProviderImportSummary), + Deferred { durable_progress: bool }, + RetryReplacement, +} + +const STAGED_APPEND_PUBLICATION_VERSION: u32 = 2; + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct StagedAppendPublicationCompletion { + summary: ProviderImportSummary, + has_accepted_content: bool, + checkpoint: Option, + source_prefix_sha256: Option, + indexed_at_ms: i64, +} + +impl StagedAppendPublicationCompletion { + fn into_restored( + self, + ) -> ( + ProviderImportSummary, + Option, + i64, + ) { + let mut summary = self.summary; + if self.has_accepted_content { + summary.mark_retained_existing_content(); + } + (summary, self.checkpoint, self.indexed_at_ms) + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct StagedProviderFileCheckpoint { + import_revision: u32, + checkpoint_version: u32, + stable_file_identity: String, + committed_byte_offset: u64, + committed_complete_line_count: u64, + head_sha256: String, + boundary_sha256: String, + resume_state_base64: Option, + updated_at_ms: i64, +} + +impl AppendInventoryUnit<'_> { + fn provider(&self) -> CaptureProvider { + match self { + Self::Catalog { work, .. } => work.session.provider, + Self::SourceFile { work, .. } => work.file.provider, + } + } + + fn reason(&self) -> ImportPendingReason { + match self { + Self::Catalog { work, .. } => work.reason, + Self::SourceFile { work, .. } => work.reason, + } + } + + fn has_active_publication(&self) -> bool { + match self { + Self::Catalog { work, .. } => work.has_active_publication, + Self::SourceFile { work, .. } => work.has_active_publication, + } + } + + fn source_format(&self) -> &str { + match self { + Self::Catalog { work, .. } => &work.session.source_format, + Self::SourceFile { work, .. } => &work.file.source_format, + } + } + + fn source_root(&self) -> &str { + match self { + Self::Catalog { work, .. } => &work.session.source_root, + Self::SourceFile { work, .. } => &work.file.source_root, + } + } + + fn source_path(&self) -> &str { + match self { + Self::Catalog { work, .. } => &work.session.source_path, + Self::SourceFile { work, .. } => &work.file.source_path, + } + } + + fn material_source_root(&self) -> &str { + match self { + Self::Catalog { work, .. } => &work.session.source_root, + Self::SourceFile { work, .. } => &work.file.source_path, + } + } + + fn import_revision(&self) -> u32 { + match self { + Self::Catalog { work, .. } => work.session.import_revision, + Self::SourceFile { work, .. } => work.file.import_revision, + } + } + + fn file_size_bytes(&self) -> u64 { + match self { + Self::Catalog { work, .. } => work.session.file_size_bytes, + Self::SourceFile { work, .. } => work.file.file_size_bytes, + } + } + + fn observation( + &self, + indexed_at_ms: i64, + event_count: Option, + ) -> ProviderFileInventoryObservation<'_> { + match self { + Self::Catalog { + work, + inventory_generation, + .. + } => ProviderFileInventoryObservation::ObservedCatalog { + source_format: &work.session.source_format, + update: CatalogSourceIndexUpdate { + source_root: &work.session.source_root, + source_path: &work.session.source_path, + file_size_bytes: work.session.file_size_bytes, + file_modified_at_ms: work.session.file_modified_at_ms, + import_revision: work.session.import_revision, + inventory_generation: *inventory_generation, + file_sha256: None, + event_count, + indexed_at_ms, + }, + metadata: &work.session.metadata, + }, + Self::SourceFile { + work, + inventory_generation, + .. + } => ProviderFileInventoryObservation::SourceImport { + source_format: &work.file.source_format, + update: SourceImportFileIndexUpdate { + source_root: &work.file.source_root, + source_path: &work.file.source_path, + file_size_bytes: work.file.file_size_bytes, + file_modified_at_ms: work.file.file_modified_at_ms, + import_revision: work.file.import_revision, + inventory_generation: *inventory_generation, + metadata: &work.file.metadata, + indexed_at_ms, + }, + }, + } + } + + fn inventory_generation(&self) -> u64 { + match self { + Self::Catalog { + inventory_generation, + .. + } + | Self::SourceFile { + inventory_generation, + .. + } => *inventory_generation, + } + } +} diff --git a/crates/ctx-cli/src/commands/import/native_tests.rs b/crates/ctx-cli/src/commands/import/native_tests.rs index 28df5e76..d0c4c1cb 100644 --- a/crates/ctx-cli/src/commands/import/native_tests.rs +++ b/crates/ctx-cli/src/commands/import/native_tests.rs @@ -1,21 +1,19 @@ use super::*; +use crate::commands::import::scheduler::{SelectedImportSource, IMPORT_SLICE_MAX_UNITS}; +use crate::provider_args::NativeProviderArg; use crate::provider_sources::explicit_path_source; +use ctx_history_capture::MAX_PROVIDER_JSONL_LINE_BYTES; use ctx_history_core::{ - new_id, Event, EventRole, EventType, Fidelity, SyncMetadata, SyncState, Visibility, + new_id, CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, Event, EventRole, EventType, + Fidelity, SyncMetadata, SyncState, Visibility, }; use ctx_history_store::{SourceImportFile, SourceImportFileIndexUpdate}; use serde_json::json; - -fn tempdir() -> tempfile::TempDir { - let temp_root = fs::canonicalize(std::env::temp_dir()) - .expect("system temporary directory should be canonicalizable"); - tempfile::Builder::new() - .prefix("ctx-native-import-") - .tempdir_in(temp_root) - .unwrap() -} +use std::io::Write; +use std::sync::Arc; include!("native_tests/selection_and_append.rs"); +include!("native_tests/retirement.rs"); include!("native_tests/append_recovery.rs"); include!("native_tests/root_and_inventory.rs"); include!("native_tests/manifest_and_sqlite.rs"); diff --git a/crates/ctx-cli/src/commands/import/native_tests/append_recovery.rs b/crates/ctx-cli/src/commands/import/native_tests/append_recovery.rs index f35aeb62..244680da 100644 --- a/crates/ctx-cli/src/commands/import/native_tests/append_recovery.rs +++ b/crates/ctx-cli/src/commands/import/native_tests/append_recovery.rs @@ -1,13 +1,544 @@ -fn write_valid_codex_session(root: &Path, session_id: &str) { - fs::create_dir_all(root).unwrap(); +#[test] +fn codex_partial_append_defers_then_shrink_replacement_removes_stale_material() { + let temp = tempdir(); + let root = temp.path().join("sessions"); + let file = root.join("session.jsonl"); + fs::create_dir_all(&root).unwrap(); + let header = jsonl(json!({ + "timestamp": "2026-07-14T12:00:00Z", + "type": "session_meta", + "payload": {"id": "codex-partial", "timestamp": "2026-07-14T12:00:00Z", "cwd": "/repo"} + })); + let initial_message = jsonl(json!({ + "timestamp": "2026-07-14T12:00:01Z", + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "keep me"}]} + })); + fs::write(&file, format!("{header}{initial_message}")).unwrap(); + let source = explicit_path_source(CaptureProvider::Codex, root); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (_, initial_checkpoint) = run_single_fresh_unit(&mut store, source.clone()); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + let completed_tail = serde_json::to_string(&json!({ + "timestamp": "2026-07-14T12:00:02Z", + "type": "response_item", + "payload": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "remove me after shrink"}]} + })) + .unwrap(); + fs::OpenOptions::new() + .append(true) + .open(&file) + .unwrap() + .write_all(completed_tail.as_bytes()) + .unwrap(); + let inventory = inventory_import_sources(&store, vec![source.clone()], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + let selected = &slice.sources[0]; + let source_plan = &plan.sources[selected.source_index]; + let deferred = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + assert_eq!(deferred.summary, ProviderImportSummary::default()); + assert_eq!(deferred.completed_units, 0); + assert_eq!(deferred.completed_bytes, 0); + assert_eq!(deferred.deferred_units, 1); + assert_eq!(deferred.post_import_inventory_generation, None); + let retained = store + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider: initial_checkpoint.provider, + source_format: &initial_checkpoint.source_format, + source_root: &initial_checkpoint.source_root, + source_path: &initial_checkpoint.source_path, + }) + .unwrap() + .unwrap(); + assert_eq!( + retained.committed_byte_offset, + initial_checkpoint.committed_byte_offset + ); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + fs::OpenOptions::new() + .append(true) + .open(&file) + .unwrap() + .write_all(b"\n") + .unwrap(); + let (_, completed_checkpoint) = run_single_fresh_unit(&mut store, source.clone()); + assert_eq!( + completed_checkpoint.committed_byte_offset, + fs::metadata(&file).unwrap().len() + ); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("remove me after shrink")); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + fs::write(&file, format!("{header}{initial_message}")).unwrap(); + let (_, replacement_checkpoint) = run_single_fresh_unit(&mut store, source.clone()); + assert_eq!( + replacement_checkpoint.committed_byte_offset, + fs::metadata(&file).unwrap().len() + ); + let archive = serde_json::to_string(&store.export_archive().unwrap()).unwrap(); + assert!(archive.contains("keep me")); + assert!(!archive.contains("remove me after shrink")); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert_unchanged_source_has_no_work(&store, source); +} + +#[test] +fn mixed_codex_append_slice_reports_only_completed_bytes() { + let temp = tempdir(); + let root = temp.path().join("sessions"); + write_valid_codex_session(&root, "a-complete"); + let completed_file = root.join("a-complete.jsonl"); + let deferred_file = root.join("b-deferred.jsonl"); + let source = explicit_path_source(CaptureProvider::Codex, root.clone()); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + run_single_fresh_unit(&mut store, source.clone()); + write_valid_codex_session(&root, "b-deferred"); + run_single_fresh_unit(&mut store, source.clone()); + + fs::OpenOptions::new() + .append(true) + .open(&completed_file) + .unwrap() + .write_all( + jsonl(json!({ + "timestamp": "2026-07-14T12:00:02Z", + "type": "response_item", + "payload": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "complete tail"}]} + })) + .as_bytes(), + ) + .unwrap(); + fs::OpenOptions::new() + .append(true) + .open(&deferred_file) + .unwrap() + .write_all( + serde_json::to_string(&json!({ + "timestamp": "2026-07-14T12:00:02Z", + "type": "response_item", + "payload": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "partial tail"}]} + })) + .unwrap() + .as_bytes(), + ) + .unwrap(); + + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 2); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + let selected = &slice.sources[0]; + let expected_completed_bytes = match &selected.work { + SelectedImportWork::Catalog(work) => { + work.iter() + .find(|work| Path::new(&work.session.source_path) == completed_file) + .unwrap() + .estimated_bytes + } + SelectedImportWork::SourceFiles(_) => panic!("Codex tree must select catalog work"), + }; + let source_plan = &plan.sources[selected.source_index]; + let outcome = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + + assert_eq!(outcome.completed_units, 1); + assert_eq!(outcome.deferred_units, 1); + assert_eq!(outcome.completed_bytes, expected_completed_bytes); + assert_eq!( + store + .list_catalog_import_work( + CaptureProvider::Codex, + root.to_str().unwrap(), + ImportWorkClass::Fresh, + 10, + ) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn rejected_codex_growth_keeps_later_growth_append_capable() { + let temp = tempdir(); + let root = temp.path().join("sessions"); + let file = root.join("session.jsonl"); + fs::create_dir_all(&root).unwrap(); + let header = jsonl(json!({ + "timestamp": "2026-07-14T12:00:00Z", + "type": "session_meta", + "payload": {"id": "codex-rejected-growth", "timestamp": "2026-07-14T12:00:00Z", "cwd": "/repo"} + })); + let initial_message = jsonl(json!({ + "timestamp": "2026-07-14T12:00:01Z", + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "initial"}]} + })); + let initial = format!("{header}{initial_message}"); + fs::write(&file, &initial).unwrap(); + let source = explicit_path_source(CaptureProvider::Codex, root); + let db_path = temp.path().join("work.sqlite"); + let mut store = Store::open(&db_path).unwrap(); + let (_, initial_checkpoint) = run_single_fresh_unit(&mut store, source.clone()); + + let rejected = concat!( + r#"{"timestamp":"2026-07-14T12:00:02Z","type":"response_item","payload":{"type":"message","role":"assistant","content":["#, + "\n" + ); + fs::write(&file, format!("{initial}{rejected}")).unwrap(); + let inventory = inventory_import_sources(&store, vec![source.clone()], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + let selected = &slice.sources[0]; + let first_reason = match &selected.work { + SelectedImportWork::Catalog(work) => work[0].reason, + SelectedImportWork::SourceFiles(_) => panic!("Codex tree must select catalog work"), + }; + assert_eq!(first_reason, ImportPendingReason::FreshAppend); + let source_plan = &plan.sources[selected.source_index]; + let rejected_outcome = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + assert_eq!(rejected_outcome.summary.failed, 1); + assert_eq!( + store + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider: initial_checkpoint.provider, + source_format: &initial_checkpoint.source_format, + source_root: &initial_checkpoint.source_root, + source_path: &initial_checkpoint.source_path, + }) + .unwrap() + .unwrap() + .committed_byte_offset, + initial_checkpoint.committed_byte_offset + ); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let post_rejection_state: (String, i64, i64, i64) = conn + .query_row( + r#" + SELECT catalog.indexed_status, + catalog.file_size_bytes, + catalog.indexed_file_size_bytes, + EXISTS ( + SELECT 1 + FROM sessions AS material_session + JOIN capture_sources AS source + ON source.id = material_session.capture_source_id + WHERE material_session.provider = catalog.provider + AND material_session.external_session_id = catalog.external_session_id + AND source.provider = catalog.provider + AND source.source_format = 'codex_session_jsonl' + AND source.raw_source_path = catalog.source_path + ) + FROM catalog_sessions AS catalog + WHERE catalog.source_path = ?1 + "#, + [file.to_str().unwrap()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!(post_rejection_state.0, "completed_with_rejections"); + assert_eq!(post_rejection_state.1, post_rejection_state.2); + assert_eq!(post_rejection_state.3, 1); + drop(conn); + + let accepted = jsonl(json!({ + "timestamp": "2026-07-14T12:00:03Z", + "type": "response_item", + "payload": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "accepted after rejection"}]} + })); + fs::write(&file, format!("{initial}{rejected}{accepted}")).unwrap(); + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + let selected = &slice.sources[0]; + let second_reason = match &selected.work { + SelectedImportWork::Catalog(work) => work[0].reason, + SelectedImportWork::SourceFiles(_) => panic!("Codex tree must select catalog work"), + }; + assert_eq!(second_reason, ImportPendingReason::FreshAppend); + let source_plan = &plan.sources[selected.source_index]; + let accepted_outcome = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + assert_eq!(accepted_outcome.summary.failed, 1); + assert_eq!(accepted_outcome.summary.imported_events, 1); + assert_eq!( + store + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider: initial_checkpoint.provider, + source_format: &initial_checkpoint.source_format, + source_root: &initial_checkpoint.source_root, + source_path: &initial_checkpoint.source_path, + }) + .unwrap() + .unwrap() + .committed_byte_offset, + initial_checkpoint.committed_byte_offset + ); +} + +#[test] +fn deferred_source_file_reports_zero_completion_and_post_import_generation() { + let temp = tempdir(); + let file = temp.path().join("session.jsonl"); + fs::write( + &file, + format!( + "{}{}", + jsonl(json!({ + "timestamp": "2026-07-14T12:00:00Z", + "type": "session_meta", + "payload": {"id": "codex-source-file-deferred", "timestamp": "2026-07-14T12:00:00Z", "cwd": "/repo"} + })), + jsonl(json!({ + "timestamp": "2026-07-14T12:00:01Z", + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "initial"}]} + })) + ), + ) + .unwrap(); + let source = explicit_path_source(CaptureProvider::Codex, file.clone()); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + run_single_fresh_unit(&mut store, source.clone()); + + let unterminated_tail = serde_json::to_string(&json!({ + "timestamp": "2026-07-14T12:00:02Z", + "type": "response_item", + "payload": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "wait for newline"}]} + })) + .unwrap(); + fs::OpenOptions::new() + .append(true) + .open(&file) + .unwrap() + .write_all(unterminated_tail.as_bytes()) + .unwrap(); + + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + let selected = &slice.sources[0]; + assert!(matches!(&selected.work, SelectedImportWork::SourceFiles(_))); + let pre_import_generation = selected.preinventory.inventory_generation().unwrap(); + let source_plan = &plan.sources[selected.source_index]; + let outcome = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + + assert_eq!(outcome.completed_units, 0); + assert_eq!(outcome.completed_bytes, 0); + assert_eq!(outcome.deferred_units, 1); + assert!(!outcome.durable_progress); + assert_eq!( + outcome.post_import_inventory_generation, + Some(pre_import_generation) + ); +} + +#[test] +fn append_source_failure_after_mutation_fences_other_work_until_recovered() { + let temp = tempdir(); + let root = temp.path().join("sessions"); + let file = root.join("session.jsonl"); + fs::create_dir_all(&root).unwrap(); fs::write( - root.join(format!("{session_id}.jsonl")), + &file, format!( - "{{\"timestamp\":\"2026-07-13T12:00:00Z\",\"type\":\"session_meta\",\"payload\":{{\"id\":\"{session_id}\",\"timestamp\":\"2026-07-13T12:00:00Z\",\"cwd\":\"/repo\",\"originator\":\"codex-cli\",\"source\":\"cli\"}}}}\n\ - {{\"timestamp\":\"2026-07-13T12:00:01Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"message\",\"role\":\"user\",\"content\":[{{\"type\":\"input_text\",\"text\":\"supersession retry oracle\"}}]}}}}\n" + "{}{}", + jsonl(json!({ + "timestamp": "2026-07-14T12:00:00Z", + "type": "session_meta", + "payload": {"id": "codex-recovery-required", "timestamp": "2026-07-14T12:00:00Z", "cwd": "/repo"} + })), + jsonl(json!({ + "timestamp": "2026-07-14T12:00:01Z", + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "initial"}]} + })) ), ) .unwrap(); + let source = explicit_path_source(CaptureProvider::Codex, root.clone()); + let db_path = temp.path().join("work.sqlite"); + let mut store = Store::open(&db_path).unwrap(); + run_single_fresh_unit(&mut store, source.clone()); + + fs::OpenOptions::new() + .append(true) + .open(&file) + .unwrap() + .write_all( + jsonl(json!({ + "timestamp": "2026-07-14T12:00:02Z", + "type": "response_item", + "payload": {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "durable before injected failure"}]} + })) + .as_bytes(), + ) + .unwrap(); + let inventory = inventory_import_sources(&store, vec![source.clone()], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + let selected = &slice.sources[0]; + let source_plan = &plan.sources[selected.source_index]; + let catalog_outcome_before = rusqlite::Connection::open(&db_path) + .unwrap() + .query_row( + "SELECT indexed_status, indexed_error FROM catalog_sessions WHERE source_path = ?1", + [file.display().to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + ) + .unwrap(); + inject_append_source_failure_after_mutation(); + let error = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap_err(); + + assert!(publication_recovery_required(&error)); + assert!(publication_recovery_maintenance_warning(&error).is_none()); + assert_eq!(import_error_scope(&error), ImportFailureScope::Source); + assert_eq!( + error.to_string(), + "invalid capture payload: injected append source failure after publication mutation" + ); + assert!(store.has_pending_provider_file_publications().unwrap()); + assert_eq!( + rusqlite::Connection::open(&db_path) + .unwrap() + .query_row( + "SELECT indexed_status, indexed_error FROM catalog_sessions WHERE source_path = ?1", + [file.display().to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + ) + .unwrap(), + catalog_outcome_before, + "recovery-required errors must not record an ordinary source outcome" + ); + + let pi_root = temp.path().join("pi-sessions"); + fs::create_dir_all(&pi_root).unwrap(); + fs::write( + pi_root.join("session.jsonl"), + format!( + "{}{}", + jsonl(json!({ + "type": "session", "id": "pi-unrelated", "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", "id": "pi-unrelated-user", "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "unrelated work continues"} + })) + ), + ) + .unwrap(); + let unrelated = explicit_path_source(CaptureProvider::Pi, pi_root); + let inventory = inventory_import_sources(&store, vec![unrelated.clone()], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + assert_eq!(slice.sources.len(), 1); + assert_eq!(slice.sources[0].source_index, 0); + assert_eq!(plan.sources[0].source.path, source.path); + + let mut recovered = None; + for _ in 0..32 { + let result = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + assert!(result.remaining_error.is_none()); + if result.outcome.completed_units == 1 { + recovered = Some(result.outcome); + break; + } + assert_eq!(result.outcome.deferred_units, 1); + } + recovered.expect("the interrupted publication must resume"); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("durable before injected failure")); + let (unrelated_summary, _) = run_single_fresh_unit(&mut store, unrelated); + assert!(unrelated_summary.imported_events > 0); +} + +#[test] +fn provider_finalize_cleanup_warning_maps_to_import_summary_maintenance() { + let mut summary = ProviderImportSummary::default(); + push_publication_maintenance_warning( + &mut summary, + ProviderFileMaintenanceWarning::StagingCleanupDeferred { + publication_id: "publication-1".to_owned(), + operation: "remove-directory", + }, + ); + + assert_eq!(summary.maintenance_warnings.len(), 1); + assert_eq!( + summary.maintenance_warnings[0].kind, + ProviderImportMaintenanceKind::ImportInterruptedAfterCommit + ); + assert!(summary.maintenance_warnings[0] + .error + .contains("staging cleanup deferred during remove-directory")); } #[test] diff --git a/crates/ctx-cli/src/commands/import/native_tests/manifest_and_sqlite.rs b/crates/ctx-cli/src/commands/import/native_tests/manifest_and_sqlite.rs index 21ec461d..b3e2f246 100644 --- a/crates/ctx-cli/src/commands/import/native_tests/manifest_and_sqlite.rs +++ b/crates/ctx-cli/src/commands/import/native_tests/manifest_and_sqlite.rs @@ -5,31 +5,36 @@ fn manifested_completion_requires_exact_post_import_observation() { fs::create_dir(&source_path).unwrap(); fs::write(source_path.join("messages.jsonl"), b"{}\n").unwrap(); fs::write(source_path.join("meta.json"), b"{}\n").unwrap(); + let companion_modified = fs::metadata(source_path.join("meta.json")) + .unwrap() + .modified() + .unwrap(); let source = explicit_path_source(CaptureProvider::MistralVibe, source_path.clone()); let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); let mut import_file = |_store: &mut Store, _pending_source: &SourceInfo| { - let mut meta = fs::OpenOptions::new() - .append(true) - .open(source_path.join("meta.json")) + let meta_path = source_path.join("meta.json"); + fs::write(&meta_path, b"[]\n").unwrap(); + fs::File::options() + .write(true) + .open(&meta_path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(companion_modified)) .unwrap(); - use std::io::Write as _; - meta.write_all(b"changed\n").unwrap(); - meta.sync_all().unwrap(); Ok(successful_file_summary()) }; let summary = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .unwrap(); assert_eq!(summary.imported_events, 1); + assert_eq!(summary.completed_units, 0); + assert_eq!(summary.deferred_units, 0); + assert!(summary.post_import_inventory_generation.is_some()); let pending = store .list_pending_source_import_files(source.provider, source.path.to_str().unwrap()) .unwrap(); @@ -37,6 +42,117 @@ fn manifested_completion_requires_exact_post_import_observation() { assert_eq!(store.source_import_file_counts().unwrap().indexed, 0); } +#[test] +fn manifested_current_result_reports_exact_completion_and_post_import_generation() { + let temp = tempdir(); + let source_path = temp.path().join("sessions"); + fs::create_dir(&source_path).unwrap(); + fs::write(source_path.join("messages.jsonl"), b"{}\n").unwrap(); + let source = explicit_path_source(CaptureProvider::MistralVibe, source_path); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let files = collect_source_import_files(&source).unwrap(); + assert_eq!(files.len(), 1); + let pre_import = persist_new_source_import_observation(&store, &source, &files).unwrap(); + let mut import_file = + |_store: &mut Store, _pending_source: &SourceInfo| Ok(successful_file_summary()); + + let outcome = import_manifested_source_with_importer( + &mut store, + &source, + ManifestedImportOptions::new( + Some(&files), + Some(pre_import.inventory_generation), + false, + None, + ), + &mut import_file, + ) + .unwrap(); + + assert_eq!(outcome.completed_units, 1); + assert_eq!(outcome.deferred_units, 0); + assert_eq!( + outcome.post_import_inventory_generation, + Some(pre_import.inventory_generation) + ); + assert!(outcome.post_import_preinventory.is_none()); + assert_eq!(outcome.imported_events, 1); +} + +#[test] +fn manifested_130_unit_drain_keeps_one_generation_across_three_slices() { + let temp = tempdir(); + let source_path = temp.path().join("sessions"); + fs::create_dir(&source_path).unwrap(); + for index in 0..130 { + let session_path = source_path.join(format!("session-{index:03}")); + fs::create_dir(&session_path).unwrap(); + fs::write(session_path.join("messages.jsonl"), b"{}\n").unwrap(); + } + let source = explicit_path_source(CaptureProvider::MistralVibe, source_path); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let files = collect_source_import_files(&source).unwrap(); + assert_eq!(files.len(), 130); + let persisted = persist_new_source_import_observation(&store, &source, &files).unwrap(); + let plan = ImportPlan::build( + &store, + vec![PlannedImportSource { + source: source.clone(), + stats: SourceStats::default(), + preinventory: SourcePreinventory::SourceImportFiles { + files: files.clone(), + inventory_generation: persisted.inventory_generation, + }, + }], + ) + .unwrap(); + assert_eq!(plan.fresh_units, 130); + let mut completed = 0usize; + let mut slices = 0usize; + let mut import_file = + |_store: &mut Store, _pending_source: &SourceInfo| Ok(successful_file_summary()); + + loop { + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, IMPORT_SLICE_MAX_UNITS) + .unwrap(); + if slice.is_empty() { + break; + } + slices += 1; + let selected = &slice.sources[0]; + let outcome = import_manifested_source_with_importer( + &mut store, + &source, + ManifestedImportOptions::new( + Some(&files), + Some(persisted.inventory_generation), + false, + Some(&selected.work), + ), + &mut import_file, + ) + .unwrap(); + assert_eq!( + outcome.post_import_inventory_generation, + Some(persisted.inventory_generation) + ); + completed += outcome.completed_units; + } + + assert_eq!(slices, 3); + assert_eq!(completed, 130); + assert!(store + .list_source_import_file_work( + source.provider, + source.path.to_str().unwrap(), + ImportWorkClass::Fresh, + 1, + ) + .unwrap() + .is_empty()); +} + #[test] fn whole_source_completion_requires_exact_post_import_observation() { let temp = tempdir(); @@ -101,7 +217,7 @@ fn whole_source_post_import_observation_failure_is_not_success() { #[test] fn whole_source_store_failure_takes_precedence_over_source_failure() { let observation_error = anyhow::Error::new(StoreError::BulkSearchImportBusy); - let selected = final_observation_system_error(Err(observation_error)) + let selected = final_observation_system_error::<()>(Err(observation_error)) .expect("a store failure during final observation must abort the whole run"); assert_eq!(import_error_scope(&selected), ImportFailureScope::System); @@ -126,10 +242,7 @@ fn manifested_system_error_survives_failed_post_observation() { let error = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .expect_err("the original system error must abort the manifested import"); @@ -162,10 +275,7 @@ fn manifested_source_error_keeps_its_typed_identity() { let error = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .expect_err("a source adapter failure must remain an error"); @@ -180,6 +290,83 @@ fn manifested_source_error_keeps_its_typed_identity() { assert_eq!(import_error_scope(&error), ImportFailureScope::Source); } +#[test] +fn manifested_terminal_file_failure_preserves_sibling_success() { + let temp = tempdir(); + let source_path = temp.path().join("sessions"); + fs::create_dir(&source_path).unwrap(); + fs::write(source_path.join("good.jsonl"), b"{}\n").unwrap(); + fs::write(source_path.join("bad.jsonl"), b"{}\n").unwrap(); + let source = explicit_path_source(CaptureProvider::Pi, source_path); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let mut import_file = |_store: &mut Store, pending_source: &SourceInfo| { + if pending_source.path.ends_with("bad.jsonl") { + return Err(anyhow::Error::new( + CaptureError::InvalidProviderTranscriptPath { + path: pending_source.path.clone(), + reason: "deterministic malformed fixture", + }, + )); + } + Ok(successful_file_summary()) + }; + + let outcome = import_manifested_source_with_importer( + &mut store, + &source, + ManifestedImportOptions::new(None, None, false, None), + &mut import_file, + ) + .unwrap(); + + assert_eq!(outcome.completed_units, 2); + assert_eq!(outcome.summary.imported_events, 1); + assert_eq!(outcome.summary.failed, 1); + assert_eq!(outcome.summary.failures.len(), 1); + assert!(outcome.summary.failures[0].error.contains("bad.jsonl")); +} + +#[test] +fn manifested_retryable_file_failure_preserves_original_error_after_sibling_success() { + let temp = tempdir(); + let source_path = temp.path().join("sessions"); + fs::create_dir(&source_path).unwrap(); + fs::write(source_path.join("good.jsonl"), b"{}\n").unwrap(); + fs::write(source_path.join("retry.jsonl"), b"{}\n").unwrap(); + let source = explicit_path_source(CaptureProvider::Pi, source_path); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let mut import_file = |_store: &mut Store, pending_source: &SourceInfo| { + if pending_source.path.ends_with("retry.jsonl") { + return Err(anyhow::Error::new(CaptureError::Io(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "retry this exact file", + )))); + } + Ok(successful_file_summary()) + }; + + let error = import_manifested_source_with_importer( + &mut store, + &source, + ManifestedImportOptions::new(None, None, false, None), + &mut import_file, + ) + .expect_err("a retryable manifested file failure must remain retryable"); + + let partial = error + .downcast::() + .expect("sibling success must be attached to the remaining retryable error"); + let (outcome, error) = partial.into_parts(); + assert_eq!(outcome.completed_units, 1); + assert_eq!(outcome.summary.imported_events, 1); + assert!(outcome.made_durable_progress()); + assert!(matches!( + error.downcast_ref::(), + Some(CaptureError::Io(source)) if source.kind() == std::io::ErrorKind::TimedOut + )); + assert!(error.to_string().contains("retry this exact file")); +} + #[test] fn manifested_sqlite_change_during_import_remains_pending() { let temp = tempfile::tempdir().unwrap(); @@ -210,18 +397,18 @@ fn manifested_sqlite_change_during_import_remains_pending() { Ok(successful_file_summary()) }; - import_manifested_source_with_importer( + let outcome = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .unwrap(); assert!(writer.is_some()); + assert_eq!(outcome.completed_units, 0); + assert_eq!(outcome.deferred_units, 0); + assert!(outcome.post_import_inventory_generation.is_some()); assert_eq!( store .list_pending_source_import_files(source.provider, source.path.to_str().unwrap()) @@ -259,17 +446,16 @@ fn optional_companion_directory_change_invalidates_the_absence_watch() { Ok(successful_file_summary()) }; - import_manifested_source_with_importer( + let outcome = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .unwrap(); + assert_eq!(outcome.completed_units, 0); + assert!(outcome.post_import_inventory_generation.is_some()); assert_eq!( store .list_pending_source_import_files(source.provider, source.path.to_str().unwrap()) @@ -296,17 +482,23 @@ fn removed_and_new_manifest_units_are_stale_and_pending() { Ok(successful_file_summary()) }; - import_manifested_source_with_importer( + let outcome = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .unwrap(); + assert_eq!(outcome.completed_units, 0); + assert!(outcome.post_import_inventory_generation.is_some()); + let pending = store + .list_pending_source_import_files(source.provider, source.path.to_str().unwrap()) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].source_path, old_path.display().to_string()); + let current_files = collect_source_import_files(&source).unwrap(); + persist_new_source_import_observation(&store, &source, ¤t_files).unwrap(); let pending = store .list_pending_source_import_files(source.provider, source.path.to_str().unwrap()) .unwrap(); @@ -338,15 +530,22 @@ fn removed_manifest_unit_drops_its_source_failure_after_reobservation() { let summary = import_manifested_source_with_importer( &mut store, &source, - new_id(), - None, - None, - false, + ManifestedImportOptions::new(None, None, false, None), &mut import_file, ) .unwrap(); - assert_eq!(summary, ProviderImportSummary::default()); + assert_eq!(summary.summary, ProviderImportSummary::default()); + assert_eq!(summary.completed_units, 0); + assert_eq!(summary.deferred_units, 0); + assert!(summary.post_import_inventory_generation.is_some()); + let counts = store.source_import_file_counts().unwrap(); + assert_eq!(counts.stale, 0); + assert_eq!(counts.failed, 0); + assert_eq!(counts.rejected, 0); + let current_files = collect_source_import_files(&source).unwrap(); + assert!(current_files.is_empty()); + persist_new_source_import_observation(&store, &source, ¤t_files).unwrap(); let counts = store.source_import_file_counts().unwrap(); assert_eq!(counts.stale, 1); assert_eq!(counts.failed, 0); @@ -500,6 +699,7 @@ fn provider_sqlite_lock_is_pending_until_the_lock_is_released() { .unwrap(), 1 ); + persist_source_material(&store, &file); mark_source_import_file_result( &store, &file, @@ -513,3 +713,76 @@ fn provider_sqlite_lock_is_pending_until_the_lock_is_released() { .unwrap() .is_empty()); } + +#[test] +fn setup_drain_imports_later_complete_files_despite_an_incomplete_earlier_file() { + let temp = tempdir(); + let source_root = temp.path().join("pi-fair-source"); + let source = write_pi_source(&source_root, "partial-first"); + let data_root = temp.path().join("data"); + let args = crate::ImportArgs { + provider: Some(NativeProviderArg::Pi), + path: Some(source.path.clone()), + history_source: None, + history_source_manifest: Vec::new(), + reset_cursor: false, + format: None, + all: false, + resume: false, + no_daemon: true, + json: false, + progress: ProgressArg::None, + }; + let run = |data_root: PathBuf| { + crate::commands::import::run_import_internal( + &args, + data_root, + &mut serde_json::Map::new(), + crate::commands::import::ImportRunOptions { + progress: ProgressArg::None, + json: false, + print_human: false, + allow_empty_sources: false, + include_history_source_plugins: false, + operation: "setup", + }, + ) + .unwrap() + }; + let baseline = run(data_root.clone()); + assert_eq!(baseline.totals.fresh_units_pending, 0, "{baseline:?}"); + fs::OpenOptions::new() + .append(true) + .open(source_root.join("session.jsonl")) + .unwrap() + .write_all(br#"{"type":"message","id":"partial""#) + .unwrap(); + fs::write( + source_root.join("later.jsonl"), + format!( + "{}{}", + jsonl(json!({ + "type": "session", + "id": "complete-later", + "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", + "id": "complete-later-message", + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "complete later foreground content"} + })) + ), + ) + .unwrap(); + + let report = run(data_root.clone()); + + assert_eq!(report.totals.fresh_units_processed, 1, "{report:?}"); + assert_eq!(report.totals.fresh_units_pending, 1, "{report:?}"); + let store = Store::open(ctx_history_core::database_path(data_root)).unwrap(); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("complete later foreground content")); + assert!(!store.has_pending_provider_file_publications().unwrap()); +} diff --git a/crates/ctx-cli/src/commands/import/native_tests/retirement.rs b/crates/ctx-cli/src/commands/import/native_tests/retirement.rs new file mode 100644 index 00000000..43ee7caa --- /dev/null +++ b/crates/ctx-cli/src/commands/import/native_tests/retirement.rs @@ -0,0 +1,430 @@ +fn create_orphaned_pi_publication( + data_root: &Path, + prior_event_count: usize, + retain_prior_events: bool, +) -> PathBuf { + fs::create_dir_all(data_root).unwrap(); + let source_root = data_root.join("pi-sessions"); + let source_path = source_root.join("session.jsonl"); + fs::create_dir_all(&source_root).unwrap(); + fs::write( + &source_path, + format!( + "{}{}", + jsonl(json!({ + "type": "session", "id": "pi-retirement", "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", "id": "pi-retirement-user", "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "retirement fixture"} + })) + ), + ) + .unwrap(); + let source = explicit_path_source(CaptureProvider::Pi, source_root.clone()); + let db_path = database_path(data_root.to_path_buf()); + let mut store = Store::open(&db_path).unwrap(); + run_single_fresh_unit(&mut store, source.clone()); + + let template = store + .export_archive() + .unwrap() + .events + .into_iter() + .next() + .expect("Pi fixture must import an event"); + let mut prior_events = vec![template.clone()]; + for index in 0..prior_event_count { + let mut event = template.clone(); + event.id = new_id(); + event.seq = 10_000 + index as u64; + event.dedupe_key = None; + event.payload = json!({"text": format!("prior retirement event {index}")}); + store.upsert_event(&event).unwrap(); + prior_events.push(event); + } + + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let (file, generation) = match &inventory.sources[0].preinventory { + SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + } => (files[0].clone(), *inventory_generation), + other => panic!("unexpected Pi inventory: {other:?}"), + }; + let material_source_format = + provider_canonical_material_source_format(file.provider, &file.source_format).unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + ProviderFileInventoryObservation::SourceImport { + source_format: &file.source_format, + update: SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + }, + material_source_format, + ProviderFilePublicationKind::Replacement, + utc_now().timestamp_millis(), + ) + .unwrap(); + loop { + let progress = store + .prepare_provider_file_publication_slice(&scope, 64) + .unwrap(); + if progress.complete { + break; + } + } + if retain_prior_events { + store + .with_provider_file_publication_writes(&scope, |store| { + for event in &prior_events { + store.upsert_event(event)?; + } + Ok(()) + }) + .unwrap(); + } + let mut crash_event = template; + crash_event.id = new_id(); + crash_event.seq = 20_000; + crash_event.dedupe_key = None; + crash_event.payload = json!({"text": "crash after mutation"}); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&crash_event)) + .unwrap(); + drop(scope); + drop(store); + + fs::remove_file(&source_path).unwrap(); + let observer = Store::open(&db_path).unwrap(); + let tombstone_generation = observer + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + observer + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + utc_now().timestamp_millis(), + tombstone_generation, + ) + .unwrap(); + assert_eq!( + observer + .provider_file_publication_retirement_work_count() + .unwrap(), + 1 + ); + assert!(observer.export_archive().unwrap().events.is_empty()); + source_root +} + +fn create_orphaned_pi_incremental_publication(data_root: &Path, owner_source_count: usize) { + assert!(owner_source_count > 0); + fs::create_dir_all(data_root).unwrap(); + let source_root = data_root.join("pi-incremental-sessions"); + let source_path = source_root.join("session.jsonl"); + fs::create_dir_all(&source_root).unwrap(); + fs::write( + &source_path, + format!( + "{}{}", + jsonl(json!({ + "type": "session", "id": "pi-preparation", "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", "id": "pi-preparation-user", "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "preparation fixture"} + })) + ), + ) + .unwrap(); + let source = explicit_path_source(CaptureProvider::Pi, source_root.clone()); + let db_path = database_path(data_root.to_path_buf()); + let mut store = Store::open(&db_path).unwrap(); + run_single_fresh_unit(&mut store, source.clone()); + + let archive = store.export_archive().unwrap(); + let template_source = archive + .capture_sources + .first() + .expect("Pi fixture must import a capture source") + .clone(); + for index in 1..owner_source_count { + let mut extra_source = template_source.clone(); + extra_source.id = new_id(); + extra_source.descriptor.external_session_id = Some(format!("pi-preparation-{index}")); + store.upsert_capture_source(&extra_source).unwrap(); + } + let template_event = archive + .events + .first() + .expect("Pi fixture must import an event") + .clone(); + + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let (file, generation) = match &inventory.sources[0].preinventory { + SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + } => (files[0].clone(), *inventory_generation), + other => panic!("unexpected Pi inventory: {other:?}"), + }; + let material_source_format = + provider_canonical_material_source_format(file.provider, &file.source_format).unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + ProviderFileInventoryObservation::SourceImport { + source_format: &file.source_format, + update: SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + }, + material_source_format, + ProviderFilePublicationKind::Incremental, + utc_now().timestamp_millis(), + ) + .unwrap(); + let mut mutation = template_event; + mutation.id = new_id(); + mutation.seq = 30_000; + mutation.dedupe_key = None; + mutation.payload = json!({"text": "mutation before bounded preparation"}); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&mutation)) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + + fs::remove_file(&source_path).unwrap(); + let tombstone_generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + utc_now().timestamp_millis(), + tombstone_generation, + ) + .unwrap(); + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 1 + ); +} + +#[test] +fn index_status_keeps_publication_retirement_in_lexical_pending_work() { + let temp = tempdir(); + create_orphaned_pi_publication(temp.path(), 0, false); + + let status = crate::commands::index::index_status_snapshot(temp.path()).unwrap(); + assert_eq!( + status["lexical"]["pending_provider_publication_retirements"], + 1 + ); + assert!(matches!( + status["lexical"]["status"].as_str(), + Some("pending" | "partial") + )); +} + +#[test] +fn setup_drains_orphaned_mutated_publication_retirement() { + let temp = tempdir(); + let data_root = temp.path().join("data"); + let source_root = create_orphaned_pi_publication(&data_root, 2, false); + let args = ImportArgs { + provider: Some(NativeProviderArg::Pi), + path: Some(source_root), + history_source: None, + history_source_manifest: Vec::new(), + reset_cursor: false, + format: None, + all: false, + resume: false, + no_daemon: true, + json: false, + progress: ProgressArg::None, + }; + let report = run_import_internal( + &args, + data_root.clone(), + &mut serde_json::Map::new(), + ImportRunOptions { + progress: ProgressArg::None, + json: false, + print_human: false, + allow_empty_sources: true, + include_history_source_plugins: false, + operation: "setup", + }, + ) + .unwrap(); + assert_eq!(report.totals.recovery_units_processed, 1); + assert_eq!(report.totals.recovery_units_pending, 0); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(store.export_archive().unwrap().events.is_empty()); +} + +#[test] +fn daemon_cycles_restart_and_finish_bounded_publication_retirement() { + let temp = tempdir(); + let data_root = temp.path().join("data"); + create_orphaned_pi_publication(&data_root, 130, false); + + let first = crate::commands::search::refresh_sources_for_search( + &data_root, + Vec::new(), + Vec::new(), + crate::commands::search::RefreshArg::Background, + false, + ImportExecutionPolicy::Daemon, + ) + .unwrap(); + assert_eq!(first.recovery_units_processed, 0); + assert_eq!(first.recovery_units_pending, 1); + + let mut pending = first.recovery_units_pending; + let mut completed = 0usize; + for _ in 0..16 { + if pending == 0 { + break; + } + let outcome = crate::commands::search::refresh_sources_for_search( + &data_root, + Vec::new(), + Vec::new(), + crate::commands::search::RefreshArg::Background, + false, + ImportExecutionPolicy::Daemon, + ) + .unwrap(); + completed = completed.saturating_add(outcome.recovery_units_processed); + pending = outcome.recovery_units_pending; + } + assert_eq!(pending, 0); + assert_eq!(completed, 1); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(store.export_archive().unwrap().events.is_empty()); +} + +#[test] +fn interrupted_incremental_retirement_converges_as_atomic_cleanup() { + let temp = tempdir(); + let data_root = temp.path().join("data"); + create_orphaned_pi_incremental_publication(&data_root, 130); + let db_path = database_path(data_root); + + let mut completed = false; + let mut attempts = 0; + for _ in 0..256 { + let store = Store::open(&db_path).unwrap(); + let work = store + .list_provider_file_publication_retirement_work(1) + .unwrap(); + assert_eq!(work.len(), 1); + let recovery = + recover_provider_file_publication_retirement(&store, &work[0], false).unwrap(); + assert!(recovery.maintenance_warnings.is_empty()); + assert!(recovery.made_durable_progress); + attempts += 1; + if recovery.completed { + completed = true; + break; + } + } + + assert!(completed, "incremental retirement did not converge"); + assert!(attempts >= 3, "owner cleanup must remain bounded"); + let store = Store::open(&db_path).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(store.export_archive().unwrap().events.is_empty()); +} + +#[test] +fn retirement_discards_staged_candidates_and_converges_across_store_reopens() { + let temp = tempdir(); + let data_root = temp.path().join("data"); + create_orphaned_pi_publication(&data_root, 130, true); + let db_path = database_path(data_root); + + let mut completed = false; + let mut attempts = 0; + for _ in 0..32 { + let store = Store::open(&db_path).unwrap(); + let work = store + .list_provider_file_publication_retirement_work(1) + .unwrap(); + assert_eq!(work.len(), 1); + let recovery = + recover_provider_file_publication_retirement(&store, &work[0], false).unwrap(); + assert!(recovery.maintenance_warnings.is_empty()); + assert!(recovery.made_durable_progress); + attempts += 1; + if recovery.completed { + completed = true; + break; + } + } + + assert!( + completed, + "bounded retained-candidate cleanup did not converge" + ); + assert!( + attempts >= 3, + "131 retained events must require multiple slices" + ); + let store = Store::open(&db_path).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(store.export_archive().unwrap().events.is_empty()); +} + +#[test] +fn retirement_without_marker_or_current_observation_is_not_durable_progress() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let outcome = recover_provider_file_publication_retirement( + &store, + &ProviderFilePublicationRetirementWork { + provider: CaptureProvider::Pi, + material_source_format: "pi_session_jsonl".to_owned(), + material_source_root: "/missing/pi-root".to_owned(), + source_path: "/missing/pi-root/session.jsonl".to_owned(), + estimated_bytes: 0, + last_attempt_at_ms: 0, + }, + false, + ) + .unwrap(); + + assert!(outcome.completed); + assert!(!outcome.made_durable_progress); + assert!(outcome.maintenance_warnings.is_empty()); +} diff --git a/crates/ctx-cli/src/commands/import/native_tests/root_and_inventory.rs b/crates/ctx-cli/src/commands/import/native_tests/root_and_inventory.rs index b62ee1e5..ad1bf552 100644 --- a/crates/ctx-cli/src/commands/import/native_tests/root_and_inventory.rs +++ b/crates/ctx-cli/src/commands/import/native_tests/root_and_inventory.rs @@ -4,6 +4,7 @@ fn persist_indexed_root(store: &Store, source: &SourceInfo) -> (SourceImportFile persist_new_source_import_observation(store, source, std::slice::from_ref(&file)).unwrap(); let inventory_generation = persisted.inventory_generation; let source_root = file.source_root.clone(); + persist_source_material(store, &file); let changed = store .mark_source_import_file_indexed( source.provider, @@ -33,6 +34,39 @@ fn persist_indexed_root(store: &Store, source: &SourceInfo) -> (SourceImportFile (file, inventory_generation) } +fn persist_source_material(store: &Store, file: &SourceImportFile) { + let material_source_format = + provider_canonical_material_source_format(file.provider, &file.source_format) + .unwrap_or(&file.source_format); + store + .upsert_capture_source(&CaptureSource { + id: new_id(), + descriptor: CaptureSourceDescriptor { + kind: CaptureSourceKind::ProviderImport, + provider: file.provider, + machine_id: "native-import-test".to_owned(), + process_id: None, + cwd: None, + raw_source_path: Some(file.source_path.clone()), + source_format: Some(material_source_format.to_owned()), + source_root: Some(file.source_root.clone()), + source_identity: None, + external_session_id: None, + }, + started_at: utc_now(), + ended_at: None, + sync: SyncMetadata { + visibility: Visibility::LocalOnly, + fidelity: Fidelity::Imported, + sync_state: SyncState::LocalOnly, + sync_version: 0, + deleted_at: None, + metadata: json!({}), + }, + }) + .unwrap(); +} + fn inventory_source_file(store: &Store, file: &SourceImportFile) -> u64 { let inventory_generation = store .allocate_source_import_inventory_generation(file.provider, &file.source_root) @@ -205,31 +239,58 @@ fn stale_root_plan_skips_after_newer_completion_wins_bulk_lock() { let source_path = temp.path().join("state.db"); fs::write(&source_path, b"not a sqlite database").unwrap(); let source = explicit_path_source(CaptureProvider::Hermes, source_path.clone()); - let (_, observation) = observe_source_root(&source).unwrap(); + let (stats, observation) = observe_source_root(&source).unwrap(); let lock_store = Store::open(&db_path).unwrap(); let old_generation = inventory_source_file(&lock_store, &observation); - let mut import_store = Store::open(&db_path).unwrap(); + let plan = ImportPlan::build( + &lock_store, + vec![PlannedImportSource { + source: source.clone(), + stats, + preinventory: SourcePreinventory::SourceRoot { + file: observation.clone(), + inventory_generation: old_generation, + }, + }], + ) + .unwrap(); + assert_eq!(plan.fresh_units, 1); let guard = lock_store.begin_event_search_bulk_mode().unwrap(); let (waiting_tx, waiting_rx) = std::sync::mpsc::channel(); - let waiting_source = source.clone(); - let waiting_observation = observation.clone(); + let waiting_db_path = db_path.clone(); let importer = std::thread::spawn(move || { - import_one_source_inner_with_pre_lock_hook( + let mut import_store = Store::open(waiting_db_path).unwrap(); + let progress = ProgressReporter::new(ProgressArg::None, false, "test-import", 0); + let mut totals = ImportTotals::default(); + let mut imported_sources = Vec::new(); + let mut native_reports = NativeSourceReports::default(); + let mut execution_state = ImportExecutionState::for_plan(&plan); + let result = execute_import_plan_class_for_report_with_pre_lock_hook( &mut import_store, - &waiting_source, - None, - false, - false, - &SourcePreinventory::SourceRoot { - file: waiting_observation, - inventory_generation: old_generation, + &plan, + &mut execution_state, + ImportWorkClass::Fresh, + plan.fresh_units, + &progress, + ImportRunOptions { + progress: ProgressArg::None, + json: false, + print_human: false, + allow_empty_sources: false, + include_history_source_plugins: false, + operation: "test-import", }, + &mut totals, + &mut native_reports, || waiting_tx.send(()).unwrap(), - ) + ); + native_reports.append_json(&plan, &mut imported_sources); + (result, totals, imported_sources) }); waiting_rx.recv().unwrap(); let new_generation = inventory_source_file(&lock_store, &observation); + persist_source_material(&lock_store, &observation); mark_source_import_file_result( &lock_store, &observation, @@ -241,58 +302,151 @@ fn stale_root_plan_skips_after_newer_completion_wins_bulk_lock() { lock_store.finish_event_search_bulk_mode(&guard).unwrap(); drop(guard); - assert_eq!( - importer.join().unwrap().unwrap(), - ProviderImportSummary::default() - ); + let (result, totals, imported_sources) = importer.join().unwrap(); + result.unwrap(); + assert_eq!(totals.fresh_units_processed, 0); + assert_eq!(totals.failed_sources, 0); + assert!(imported_sources.is_empty()); } #[test] fn waiting_root_plan_reobserves_a_change_before_skipping() { let temp = tempdir(); let db_path = temp.path().join("work.sqlite"); - let source_path = temp.path().join("state.db"); - fs::write(&source_path, b"").unwrap(); - let source = explicit_path_source(CaptureProvider::Hermes, source_path.clone()); - let lock_store = Store::open(&db_path).unwrap(); - persist_indexed_root(&lock_store, &source); - fs::write(&source_path, b"first pending change").unwrap(); - let (_, observation) = observe_source_root(&source).unwrap(); - let old_generation = persist_new_source_import_observation( - &lock_store, - &source, - std::slice::from_ref(&observation), + let source_path = temp.path().join("pi.jsonl"); + fs::write( + &source_path, + format!( + "{}{}", + jsonl(json!({ + "type": "session", + "id": "pi-waiting-change", + "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", + "id": "pi-waiting-change-user", + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "initial"} + })) + ), ) - .unwrap() - .inventory_generation; - let mut import_store = Store::open(&db_path).unwrap(); + .unwrap(); + let source = explicit_path_source(CaptureProvider::Pi, source_path.clone()); + let mut lock_store = Store::open(&db_path).unwrap(); + let (_, initial_checkpoint) = run_single_fresh_unit(&mut lock_store, source.clone()); + fs::OpenOptions::new() + .append(true) + .open(&source_path) + .unwrap() + .write_all( + jsonl(json!({ + "type": "message", + "id": "pi-waiting-change-assistant", + "timestamp": "2026-07-14T12:00:02Z", + "message": {"role": "assistant", "content": "first pending append"} + })) + .as_bytes(), + ) + .unwrap(); + let inventory = inventory_import_sources(&lock_store, vec![source.clone()], false).unwrap(); + let plan = ImportPlan::build(&lock_store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 1); let guard = lock_store.begin_event_search_bulk_mode().unwrap(); let (waiting_tx, waiting_rx) = std::sync::mpsc::channel(); - let waiting_source = source.clone(); + let waiting_db_path = db_path.clone(); let importer = std::thread::spawn(move || { - import_one_source_inner_with_pre_lock_hook( + let mut import_store = Store::open(waiting_db_path).unwrap(); + let progress = ProgressReporter::new(ProgressArg::None, false, "test-import", 0); + let mut totals = ImportTotals::default(); + let mut native_reports = NativeSourceReports::default(); + let mut execution_state = ImportExecutionState::for_plan(&plan); + let result = execute_import_plan_class_for_report_with_pre_lock_hook( &mut import_store, - &waiting_source, - None, - false, - false, - &SourcePreinventory::SourceRoot { - file: observation, - inventory_generation: old_generation, + &plan, + &mut execution_state, + ImportWorkClass::Fresh, + plan.fresh_units, + &progress, + ImportRunOptions { + progress: ProgressArg::None, + json: false, + print_human: false, + allow_empty_sources: false, + include_history_source_plugins: false, + operation: "test-import", }, + &mut totals, + &mut native_reports, || waiting_tx.send(()).unwrap(), - ) + ); + native_reports.apply_totals(&mut totals); + (result, totals) }); waiting_rx.recv().unwrap(); - fs::write(&source_path, b"second pending change with a different size").unwrap(); + fs::OpenOptions::new() + .append(true) + .open(&source_path) + .unwrap() + .write_all( + jsonl(json!({ + "type": "message", + "id": "pi-waiting-change-user-2", + "timestamp": "2026-07-14T12:00:03Z", + "message": {"role": "user", "content": "second append while waiting"} + })) + .as_bytes(), + ) + .unwrap(); + let current_len = fs::metadata(&source_path).unwrap().len(); lock_store.finish_event_search_bulk_mode(&guard).unwrap(); drop(guard); - assert!( - importer.join().unwrap().is_err(), - "a changed source must reach its provider adapter after the wait" + let (result, totals) = importer.join().unwrap(); + result.unwrap(); + let pending_after_wait = lock_store + .list_source_import_file_work( + source.provider, + &initial_checkpoint.source_root, + ImportWorkClass::Fresh, + 10, + ) + .unwrap(); + assert_eq!( + totals.fresh_units_processed, 1, + "revalidated import totals: {totals:?}; pending: {pending_after_wait:?}" ); + assert_eq!(totals.failed_sources, 0); + assert!(totals.imported_events > 0); + assert!(lock_store + .list_source_import_file_work( + source.provider, + &initial_checkpoint.source_root, + ImportWorkClass::Fresh, + 10, + ) + .unwrap() + .is_empty()); + assert!(lock_store + .list_source_import_file_work( + source.provider, + &initial_checkpoint.source_root, + ImportWorkClass::Recovery, + 10, + ) + .unwrap() + .is_empty()); + let current_checkpoint = lock_store + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider: initial_checkpoint.provider, + source_format: &initial_checkpoint.source_format, + source_root: &initial_checkpoint.source_root, + source_path: &initial_checkpoint.source_path, + }) + .unwrap() + .unwrap(); + assert_eq!(current_checkpoint.committed_byte_offset, current_len); } #[test] @@ -452,6 +606,7 @@ fn source_outcome_and_generation_commit_before_competing_inventory() { status: CatalogIndexedStatus::Indexed, error: None, }; + persist_source_material(&store, &observation); let committed = persist_source_import_observation_with_outcomes_and_hook( &store, @@ -514,6 +669,7 @@ fn stale_manifest_plan_skips_after_newer_completion_wins_bulk_lock() { .allocate_source_import_inventory_generation(source.provider, &files[0].source_root) .unwrap(); persist_source_import_files(&lock_store, &source, new_generation, &files).unwrap(); + persist_source_material(&lock_store, &files[0]); mark_source_import_file_result( &lock_store, &files[0], diff --git a/crates/ctx-cli/src/commands/import/native_tests/selection_and_append.rs b/crates/ctx-cli/src/commands/import/native_tests/selection_and_append.rs index bf053c5a..ffeecaa5 100644 --- a/crates/ctx-cli/src/commands/import/native_tests/selection_and_append.rs +++ b/crates/ctx-cli/src/commands/import/native_tests/selection_and_append.rs @@ -1,3 +1,51 @@ +fn tempdir() -> tempfile::TempDir { + let temp_root = fs::canonicalize(std::env::temp_dir()) + .expect("system temporary directory should be canonicalizable"); + tempfile::Builder::new() + .prefix("ctx-native-import-") + .tempdir_in(temp_root) + .unwrap() +} + +#[test] +fn append_completeness_probe_rejects_oversized_unterminated_line_at_limit() { + let temp = tempdir(); + let source_path = temp.path().join("oversized.jsonl"); + let file = fs::File::create(&source_path).unwrap(); + file.set_len((MAX_PROVIDER_JSONL_LINE_BYTES as u64).saturating_mul(4)) + .unwrap(); + + let error = + provider_jsonl_range_has_complete_line(&source_path, 0, file.metadata().unwrap().len()) + .unwrap_err(); + + assert!(error + .to_string() + .contains("provider JSONL line exceeds max bytes")); +} + +#[test] +fn staged_append_completion_preserves_accepted_content_status() { + let mut summary = ProviderImportSummary::default(); + summary.failed = 1; + summary.mark_retained_existing_content(); + assert_eq!( + provider_summary_import_status(&summary), + CatalogIndexedStatus::CompletedWithRejections + ); + + let completion = + encode_staged_append_completion(summary, None, Some("a".repeat(64)), 123).unwrap(); + let staged = decode_staged_append_completion(completion).unwrap(); + assert_eq!(staged.source_prefix_sha256, Some("a".repeat(64))); + let (restored, _, _) = staged.into_restored(); + + assert_eq!( + provider_summary_import_status(&restored), + CatalogIndexedStatus::CompletedWithRejections + ); +} + #[test] fn codex_preinventory_failures_survive_when_catalog_has_no_pending_sessions() { let temp = tempdir(); @@ -25,17 +73,644 @@ fn codex_preinventory_failures_survive_when_catalog_has_no_pending_sessions() { ..CatalogSummary::default() }; + let record = import_record_for_source(&source); let summary = import_incremental_codex_session_tree( &mut store, &source, - new_id(), + &record, None, Some(&catalog), Some(inventory_generation), false, + None, ) .unwrap(); assert_eq!(summary.failed, 1); assert_eq!(summary.failures, catalog.failures); + assert_eq!(summary.completed_units, 0); + assert_eq!(summary.deferred_units, 0); + assert_eq!(summary.post_import_inventory_generation, None); +} + +fn write_valid_codex_session(root: &Path, session_id: &str) { + fs::create_dir_all(root).unwrap(); + fs::write( + root.join(format!("{session_id}.jsonl")), + format!( + "{{\"timestamp\":\"2026-07-13T12:00:00Z\",\"type\":\"session_meta\",\"payload\":{{\"id\":\"{session_id}\",\"timestamp\":\"2026-07-13T12:00:00Z\",\"cwd\":\"/repo\",\"originator\":\"codex-cli\",\"source\":\"cli\"}}}}\n\ + {{\"timestamp\":\"2026-07-13T12:00:01Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"message\",\"role\":\"user\",\"content\":[{{\"type\":\"input_text\",\"text\":\"supersession retry oracle\"}}]}}}}\n" + ), + ) + .unwrap(); +} + +#[test] +fn catalog_batch_commits_one_sibling_before_reporting_later_error() { + let temp = tempdir(); + let source_path = temp.path().join("sessions"); + write_valid_codex_session(&source_path, "a-good"); + write_valid_codex_session(&source_path, "z-retry"); + let source = explicit_path_source(CaptureProvider::Codex, source_path.clone()); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 2); + let slice = plan + .select_slice(&store, ImportWorkClass::Fresh, 2) + .unwrap(); + let selected = &slice.sources[0]; + fs::remove_file(source_path.join("z-retry.jsonl")).unwrap(); + let source_plan = &plan.sources[selected.source_index]; + + let mut completed = None; + for _ in 0..32 { + let result = import_selected_source( + &mut store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + let remaining_error = result.remaining_error; + if result.outcome.completed_units == 1 { + completed = Some((result.outcome, remaining_error)); + break; + } + assert_eq!(result.outcome.deferred_units, 1); + assert!(result.outcome.made_durable_progress()); + assert!(remaining_error.is_none()); + } + let (outcome, remaining_error) = + completed.expect("healthy sibling must publish after bounded phases"); + + assert_eq!(outcome.completed_units, 1); + assert!(outcome.summary.imported_events > 0); + assert!(outcome.made_durable_progress()); + let _error = remaining_error.expect("the missing sibling must remain an error"); + + let inventory = inventory_import_sources( + &store, + vec![explicit_path_source(CaptureProvider::Codex, source_path)], + false, + ) + .unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 0); + assert_eq!(plan.recovery_units, 0); +} + +fn jsonl(value: serde_json::Value) -> String { + format!("{}\n", serde_json::to_string(&value).unwrap()) +} + +fn run_single_fresh_unit( + store: &mut Store, + source: SourceInfo, +) -> (ProviderImportSummary, ProviderFileCheckpoint) { + let inventory = inventory_import_sources(store, vec![source], false).unwrap(); + assert!(inventory.failures.is_empty()); + let plan = ImportPlan::build(store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 1); + assert_eq!(plan.recovery_units, 0); + let slice = plan + .select_slice(store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + assert_eq!(slice.units, 1); + let selected = &slice.sources[0]; + let selected_bytes = selected.stats.bytes; + let source_file_work = matches!(&selected.work, SelectedImportWork::SourceFiles(_)); + let pre_import_inventory_generation = selected.preinventory.inventory_generation(); + let (provider, source_format, source_root, source_path) = match &selected.work { + SelectedImportWork::Catalog(work) => { + let unit = &work[0].session; + ( + unit.provider, + unit.source_format.clone(), + unit.source_root.clone(), + unit.source_path.clone(), + ) + } + SelectedImportWork::SourceFiles(work) => { + let unit = &work[0].file; + ( + unit.provider, + unit.source_format.clone(), + unit.source_root.clone(), + unit.source_path.clone(), + ) + } + }; + let source_plan = &plan.sources[selected.source_index]; + let mut completed = None; + for _ in 0..32 { + let result = import_selected_source( + store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap_or_else(|error| { + panic!( + "bounded import for {} failed: {error:#}", + source_plan.source.path.display() + ) + }); + assert!(result.remaining_error.is_none()); + if result.completed_units == 1 { + completed = Some(result.outcome); + break; + } + assert_eq!(result.deferred_units, 1); + } + let summary = completed.expect("bounded publication phases must converge"); + let checkpoint = store + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider, + source_format: &source_format, + source_root: &source_root, + source_path: &source_path, + }) + .unwrap() + .expect("completed append-capable unit must persist a checkpoint"); + assert_eq!(summary.completed_units, 1); + assert_eq!(summary.completed_bytes, selected_bytes); + assert_eq!(summary.deferred_units, 0); + if source_file_work { + if source_uses_import_file_manifest(&source_plan.source) { + assert_eq!( + summary.post_import_inventory_generation, pre_import_inventory_generation, + "manifested completion must stay on its selected generation" + ); + } else { + assert!( + summary.post_import_inventory_generation > pre_import_inventory_generation, + "whole-source completion must return its committed post-import generation" + ); + } + assert_eq!( + summary + .post_import_preinventory + .as_ref() + .and_then(SourcePreinventory::inventory_generation), + summary.post_import_inventory_generation, + "the scheduler cache must receive the complete committed observation" + ); + } else { + assert_eq!(summary.post_import_inventory_generation, None); + assert!(summary.post_import_preinventory.is_none()); + } + (summary.summary, checkpoint) +} + +fn leave_mutated_pi_publication(store: &mut Store, source: &SourceInfo) -> (SourceImportFile, u64) { + run_single_fresh_unit(store, source.clone()); + let template = store + .export_archive() + .unwrap() + .events + .into_iter() + .next() + .expect("Pi fixture must import an event"); + let inventory = inventory_import_sources(store, vec![source.clone()], false).unwrap(); + let (file, generation) = match &inventory.sources[0].preinventory { + SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + } => (files[0].clone(), *inventory_generation), + other => panic!("unexpected Pi inventory: {other:?}"), + }; + assert_eq!( + store + .schedule_source_import_explicit_rescan(file.provider, &file.source_root, generation,) + .unwrap(), + 1 + ); + let scope = store + .begin_provider_file_publication( + file.provider, + ProviderFileInventoryObservation::SourceImport { + source_format: &file.source_format, + update: SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + }, + provider_canonical_material_source_format(file.provider, &file.source_format).unwrap(), + ProviderFilePublicationKind::Replacement, + utc_now().timestamp_millis(), + ) + .unwrap(); + while !store + .prepare_provider_file_publication_slice(&scope, 64) + .unwrap() + .complete + {} + let mut mutation = template; + mutation.id = new_id(); + mutation.seq = 40_000; + mutation.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&mutation)) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + (file, generation) +} + +fn write_pi_source(root: &Path, id: &str) -> SourceInfo { + fs::create_dir_all(root).unwrap(); + fs::write( + root.join("session.jsonl"), + format!( + "{}{}", + jsonl(json!({"type": "session", "id": id, "timestamp": "2026-07-14T12:00:00Z"})), + jsonl(json!({ + "type": "message", "id": format!("{id}-user"), + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": id} + })) + ), + ) + .unwrap(); + explicit_path_source(CaptureProvider::Pi, root.to_path_buf()) +} + +fn finish_all_retirement_work(store: &Store) { + for _ in 0..64 { + let work = store + .list_provider_file_publication_retirement_work(1) + .unwrap(); + if work.is_empty() { + return; + } + recover_provider_file_publication_retirement(store, &work[0], true).unwrap(); + } + panic!("bounded publication retirement did not converge"); +} + +fn finish_selected_source(store: &mut Store, plan: &ImportPlan, selected: &SelectedImportSource) { + let source = &plan.sources[selected.source_index]; + for _ in 0..32 { + let result = import_selected_source( + store, + &source.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + if result.completed_units == selected.work.unit_count() { + return; + } + assert!(result.deferred_units > 0); + } + panic!("bounded selected source did not converge"); +} + +#[test] +fn publication_owner_omitted_by_filter_resumes_before_requested_source() { + let temp = tempdir(); + let source_a = write_pi_source(&temp.path().join("source-a"), "owner-a"); + let source_b = write_pi_source(&temp.path().join("source-b"), "requested-b"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + leave_mutated_pi_publication(&mut store, &source_a); + + let inventory = inventory_import_sources(&store, vec![source_b], false).unwrap(); + assert_eq!(inventory.sources.len(), 2); + assert_eq!(inventory.sources[0].source.path, source_a.path); + assert!(has_provider_file_publication_work(&store).unwrap()); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + let executable = plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Recovery, + 1, + &mut state, + || {}, + ) + .unwrap() + .unwrap(); + assert_eq!(executable.slice.sources[0].source_index, 0); + store + .finish_event_search_bulk_mode(&executable.bulk_guard) + .unwrap(); + finish_selected_source(&mut store, &plan, &executable.slice.sources[0]); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + let fresh = plan + .select_slice(&store, ImportWorkClass::Fresh, 1) + .unwrap(); + assert_eq!(fresh.sources[0].source_index, 1); + finish_selected_source(&mut store, &plan, &fresh.sources[0]); +} + +#[test] +fn unchanged_owner_omitted_by_filter_is_not_retired() { + let temp = tempdir(); + let source_a = write_pi_source(&temp.path().join("source-a"), "owner-a"); + let source_b = write_pi_source(&temp.path().join("source-b"), "requested-b"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + leave_mutated_pi_publication(&mut store, &source_a); + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 0 + ); + + let inventory = inventory_import_sources(&store, vec![source_b], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + let executable = plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Recovery, + 1, + &mut state, + || {}, + ) + .unwrap() + .unwrap(); + assert!(executable.slice.retirements.is_empty()); + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 0 + ); + store + .finish_event_search_bulk_mode(&executable.bulk_guard) + .unwrap(); +} + +#[test] +fn changed_owner_with_advanced_generation_tombstones_then_reinventories() { + let temp = tempdir(); + let source = write_pi_source(&temp.path().join("source-a"), "owner-a"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (owner_file, owner_generation) = leave_mutated_pi_publication(&mut store, &source); + let advanced_generation = store + .allocate_source_import_inventory_generation(source.provider, &owner_file.source_root) + .unwrap(); + assert!(advanced_generation > owner_generation); + fs::OpenOptions::new() + .append(true) + .open(&owner_file.source_path) + .unwrap() + .write_all( + jsonl(json!({ + "type": "message", "id": "changed-after-marker", + "timestamp": "2026-07-14T12:00:02Z", + "message": {"role": "assistant", "content": "changed"} + })) + .as_bytes(), + ) + .unwrap(); + + let inventory = inventory_import_sources(&store, Vec::new(), false).unwrap(); + assert_eq!( + inventory.sources[0].preinventory.inventory_generation(), + Some(owner_generation) + ); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + let executable = plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Recovery, + 1, + &mut state, + || {}, + ) + .unwrap() + .unwrap(); + assert_eq!(executable.slice.retirements.len(), 1); + store + .finish_event_search_bulk_mode(&executable.bulk_guard) + .unwrap(); + finish_all_retirement_work(&store); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + let pending_after_retirement = store + .list_pending_source_import_files(source.provider, &owner_file.source_root) + .unwrap(); + assert_eq!( + pending_after_retirement.len(), + 1, + "{:?}", + store.source_import_file_counts().unwrap() + ); + let fresh = plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Fresh, + 1, + &mut state, + || {}, + ) + .unwrap(); + let next = match fresh { + Some(next) => next, + None => plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Recovery, + 1, + &mut state, + || {}, + ) + .unwrap() + .unwrap(), + }; + assert_eq!(next.slice.sources.len(), 1); + assert_eq!(next.slice.sources[0].work.unit_count(), 1); + store + .finish_event_search_bulk_mode(&next.bulk_guard) + .unwrap(); + + let later = inventory_import_sources(&store, vec![source], false).unwrap(); + let later_plan = ImportPlan::build(&store, later.sources).unwrap(); + assert_eq!( + later_plan + .fresh_units + .saturating_add(later_plan.recovery_units), + 1 + ); +} + +#[test] +fn missing_owner_root_tombstones_and_retires() { + let temp = tempdir(); + let source = write_pi_source(&temp.path().join("source-a"), "owner-a"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + leave_mutated_pi_publication(&mut store, &source); + fs::remove_dir_all(&source.path).unwrap(); + + let inventory = inventory_import_sources(&store, Vec::new(), false).unwrap(); + assert_eq!(inventory.sources.len(), 1); + assert!(matches!( + inventory.sources[0].preinventory, + SourcePreinventory::SourceImportFiles { .. } + )); + assert_eq!( + inventory.sources[0].source.source_format, + "pi_session_jsonl" + ); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + let executable = plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Recovery, + 1, + &mut state, + || {}, + ) + .unwrap() + .unwrap(); + assert_eq!(executable.slice.retirements.len(), 1); + store + .finish_event_search_bulk_mode(&executable.bulk_guard) + .unwrap(); + finish_all_retirement_work(&store); + assert!(!store.has_pending_provider_file_publications().unwrap()); +} + +fn assert_unchanged_source_has_no_work(store: &Store, source: SourceInfo) { + let inventory = inventory_import_sources(store, vec![source], false).unwrap(); + assert!(inventory.failures.is_empty()); + let plan = ImportPlan::build(store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 0); + assert_eq!(plan.recovery_units, 0); +} + +#[test] +fn resume_does_not_reschedule_healthy_codex_and_pi_units() { + let temp = tempdir(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let codex_root = temp.path().join("codex-sessions"); + write_valid_codex_session(&codex_root, "resume-codex"); + let codex = explicit_path_source(CaptureProvider::Codex, codex_root); + run_single_fresh_unit(&mut store, codex.clone()); + + let pi_root = temp.path().join("pi-sessions"); + fs::create_dir_all(&pi_root).unwrap(); + fs::write( + pi_root.join("resume-pi.jsonl"), + format!( + "{}{}", + jsonl(json!({ + "type": "session", "id": "resume-pi", "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", "id": "resume-pi-user", "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "resume pi"} + })) + ), + ) + .unwrap(); + let pi = explicit_path_source(CaptureProvider::Pi, pi_root); + run_single_fresh_unit(&mut store, pi.clone()); + + let inventory = inventory_import_sources(&store, vec![codex, pi], true).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 0); + assert_eq!(plan.recovery_units, 0); +} + +#[test] +fn append_allowlist_native_paths_publish_checkpoints_and_noop_when_unchanged() { + let cases = [ + (CaptureProvider::Codex, "codex-file"), + (CaptureProvider::Codex, "codex-tree"), + (CaptureProvider::Pi, "pi"), + (CaptureProvider::Claude, "claude"), + (CaptureProvider::Tabnine, "tabnine"), + ]; + for (index, (provider, label)) in cases.into_iter().enumerate() { + let temp = tempdir(); + let root = temp.path().join(label); + let file = match (provider, label) { + (CaptureProvider::Codex, "codex-file") => root.join("session.jsonl"), + (CaptureProvider::Codex, _) => root.join("session.jsonl"), + (CaptureProvider::Pi, _) => root.join("session.jsonl"), + (CaptureProvider::Claude, _) => root.join("project/session.jsonl"), + (CaptureProvider::Tabnine, _) => root.join("chats/session.jsonl"), + _ => unreachable!(), + }; + fs::create_dir_all(file.parent().unwrap()).unwrap(); + let contents = match provider { + CaptureProvider::Codex => format!( + "{}{}", + jsonl(json!({ + "timestamp": "2026-07-14T12:00:00Z", + "type": "session_meta", + "payload": {"id": format!("codex-{index}"), "timestamp": "2026-07-14T12:00:00Z", "cwd": "/repo"} + })), + jsonl(json!({ + "timestamp": "2026-07-14T12:00:01Z", + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "codex checkpoint"}]} + })) + ), + CaptureProvider::Pi => format!( + "{}{}", + jsonl( + json!({"type": "session", "id": "pi-checkpoint", "timestamp": "2026-07-14T12:00:00Z"}) + ), + jsonl(json!({ + "type": "message", "id": "pi-user", "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "pi checkpoint"} + })) + ), + CaptureProvider::Claude => jsonl(json!({ + "sessionId": "claude-checkpoint", "timestamp": "2026-07-14T12:00:00Z", + "type": "user", "uuid": "claude-user", + "message": {"role": "user", "content": "claude checkpoint"} + })), + CaptureProvider::Tabnine => format!( + "{}{}", + jsonl( + json!({"sessionId": "tabnine-checkpoint", "startTime": "2026-07-14T12:00:00Z"}) + ), + jsonl(json!({ + "id": "tabnine-user", "timestamp": "2026-07-14T12:00:01Z", + "type": "user", "content": "tabnine checkpoint" + })) + ), + _ => unreachable!(), + }; + fs::write(&file, contents).unwrap(); + let source_path = if label == "codex-file" { + file.clone() + } else { + root + }; + let source = explicit_path_source(provider, source_path); + let db_path = temp.path().join("work.sqlite"); + let mut store = Store::open(&db_path).unwrap(); + + let (summary, checkpoint) = run_single_fresh_unit(&mut store, source.clone()); + assert!(summary.imported_events > 0, "{label}"); + assert_eq!( + checkpoint.committed_byte_offset, + fs::metadata(&file).unwrap().len() + ); + drop(store); + let store = Store::open(&db_path).unwrap(); + assert_unchanged_source_has_no_work(&store, source); + } } diff --git a/crates/ctx-cli/src/commands/import/report.rs b/crates/ctx-cli/src/commands/import/report.rs index eda12b7a..09594bb9 100644 --- a/crates/ctx-cli/src/commands/import/report.rs +++ b/crates/ctx-cli/src/commands/import/report.rs @@ -54,6 +54,13 @@ pub(crate) fn import_report_json(report: &ImportReport) -> Value { pub(crate) fn import_totals_json(totals: &ImportTotals) -> Value { json!({ + "durable_progress": totals.durable_progress, + "fresh_units_processed": totals.fresh_units_processed, + "recovery_units_processed": totals.recovery_units_processed, + "fresh_units_pending": totals.fresh_units_pending, + "fresh_units_pending_exact": pending_count_is_exact(totals.fresh_units_pending), + "recovery_units_pending": totals.recovery_units_pending, + "recovery_units_pending_exact": pending_count_is_exact(totals.recovery_units_pending), "source_files": totals.source_files, "source_bytes": totals.source_bytes, "imported_sources": totals.imported_sources, @@ -86,6 +93,14 @@ pub(crate) fn print_import_report_human(report: &ImportReport) { report.totals.sources_completed_with_rejections ); println!("failed_sources: {}", report.totals.failed_sources); + println!( + "fresh_units_processed: {}", + report.totals.fresh_units_processed + ); + println!( + "recovery_units_processed: {}", + report.totals.recovery_units_processed + ); println!("imported_sessions: {}", report.totals.imported_sessions); println!("imported_events: {}", report.totals.imported_events); println!("imported_edges: {}", report.totals.imported_edges); @@ -94,10 +109,44 @@ pub(crate) fn print_import_report_human(report: &ImportReport) { println!("skipped_edges: {}", report.totals.skipped_edges); println!("skipped: {}", report.totals.skipped); println!("rejected_records: {}", report.totals.failed); + for line in import_pending_backlog_lines(&report.totals) { + println!("{line}"); + } println!("resume: {}", report.resume); println!("resume_mode: {}", report.resume_mode()); } +fn import_pending_backlog_lines(totals: &ImportTotals) -> Vec { + let mut lines = Vec::new(); + if totals.fresh_units_pending > 0 { + lines.push(format!( + "fresh_units_pending: {}{}", + if pending_count_is_exact(totals.fresh_units_pending) { + "" + } else { + "at least " + }, + totals.fresh_units_pending, + )); + } + if totals.recovery_units_pending > 0 { + lines.push(format!( + "recovery_units_pending: {}{}", + if pending_count_is_exact(totals.recovery_units_pending) { + "" + } else { + "at least " + }, + totals.recovery_units_pending, + )); + } + lines +} + +fn pending_count_is_exact(count: usize) -> bool { + count < IMPORT_PENDING_REPORT_LIMIT +} + pub(crate) fn source_import_json( source: &SourceInfo, stats: &SourceStats, @@ -892,4 +941,20 @@ mod tests { assert_eq!(import_report_failure_type(&totals), expected_type); } } + + #[test] + fn human_report_lines_expose_nonzero_pending_backlogs() { + assert!(import_pending_backlog_lines(&ImportTotals::default()).is_empty()); + assert_eq!( + import_pending_backlog_lines(&ImportTotals { + fresh_units_pending: 3, + recovery_units_pending: 7, + ..ImportTotals::default() + }), + vec![ + "fresh_units_pending: 3".to_owned(), + "recovery_units_pending: 7".to_owned(), + ] + ); + } } diff --git a/crates/ctx-cli/src/commands/import/scheduler.rs b/crates/ctx-cli/src/commands/import/scheduler.rs new file mode 100644 index 00000000..58d89271 --- /dev/null +++ b/crates/ctx-cli/src/commands/import/scheduler.rs @@ -0,0 +1,27 @@ +use std::cmp::Ordering; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use ctx_history_core::canonical_provider_material_source_format; +use ctx_history_store::{ + CatalogImportWork, CatalogIndexedStatus, EventSearchBulkGuard, ImportWorkClass, + ProviderFileInventoryFamily, ProviderFilePublicationRetirementWork, SourceImportFileWork, + Store, +}; + +use super::inventory::observe_source_root; +use super::manifest::{ + collect_source_import_files, observe_selected_source_import_file, + persist_new_source_import_observation, persisted_import_identity, + same_source_import_observation, source_uses_import_file_manifest, +}; +use super::{ + import_error_scope, import_failure_type, provider_publication_blocks_attempt, + ImportFailureScope, ImportFailureType, PlannedImportSource, SourcePreinventory, SourceStats, +}; + +include!("scheduler/types.rs"); +include!("scheduler/plan.rs"); +include!("scheduler/selection.rs"); +include!("scheduler/tests.rs"); diff --git a/crates/ctx-cli/src/commands/import/scheduler/plan.rs b/crates/ctx-cli/src/commands/import/scheduler/plan.rs new file mode 100644 index 00000000..bfabc7b0 --- /dev/null +++ b/crates/ctx-cli/src/commands/import/scheduler/plan.rs @@ -0,0 +1,533 @@ +impl ImportPlan { + pub(crate) fn build(store: &Store, sources: Vec) -> Result { + let (fresh_units, recovery_units) = import_work_counts(store, &sources)?; + Ok(Self { + sources, + fresh_units: execution_budget(fresh_units), + recovery_units: execution_budget(recovery_units), + }) + } + + #[cfg(test)] + pub(crate) fn select_slice( + &self, + store: &Store, + class: ImportWorkClass, + max_units: usize, + ) -> Result { + let state = ImportExecutionState::for_plan(self); + self.select_slice_with_state(store, class, max_units, &state, None) + } + + pub(crate) fn select_slice_for_execution_with_pre_lock_hook( + &self, + store: &Store, + class: ImportWorkClass, + max_units: usize, + state: &mut ImportExecutionState, + before_bulk_lock: impl FnOnce(), + ) -> Result> { + let provisional = self + .select_slice_with_state(store, class, max_units, state, None) + .context("select provisional import slice")?; + if provisional.is_empty() { + return Ok(None); + } + before_bulk_lock(); + let validation_failures = self + .observe_provisional_sources(store, state, &provisional) + .context("observe import slice before bulk lock")?; + let bulk_guard = store + .begin_event_search_bulk_mode() + .context("begin import search bulk mode")?; + match self + .revalidate_slice( + store, + class, + max_units, + state, + &provisional, + validation_failures, + ) + .context("revalidate locked import slice") + { + Ok((slice, validation_failures)) => Ok(Some(ExecutableImportSlice { + slice, + bulk_guard, + validation_failures, + })), + Err(error) => { + if let Err(finish_error) = store.finish_event_search_bulk_mode(&bulk_guard) { + return Err(finish_error.into()); + } + Err(error) + } + } + } + + pub(crate) fn pending_counts(&self, store: &Store) -> Result<(usize, usize)> { + import_work_counts(store, &self.sources) + } + + pub(crate) fn pending_count(&self, store: &Store, class: ImportWorkClass) -> Result { + Ok(execution_budget(import_work_count( + store, + &self.sources, + class, + )?)) + } + + fn select_slice_with_state( + &self, + store: &Store, + class: ImportWorkClass, + max_units: usize, + state: &ImportExecutionState, + eligible_sources: Option<&BTreeSet>, + ) -> Result { + let slice_limit = IMPORT_SLICE_MAX_UNITS.min(max_units); + if slice_limit == 0 { + return Ok(ImportSlice::empty()); + } + + let mut candidates = Vec::new(); + let fetch_limit = slice_limit; + if class == ImportWorkClass::Recovery { + for work in store.list_provider_file_publication_retirement_work(fetch_limit)? { + candidates.push(ImportCandidate::Retirement(work)); + if candidates.len() >= slice_limit { + break; + } + } + } + for (source_index, plan) in self.sources.iter().enumerate() { + if eligible_sources.is_some_and(|eligible| !eligible.contains(&source_index)) { + continue; + } + let preinventory = state + .observed_preinventories + .get(source_index) + .and_then(Option::as_ref) + .unwrap_or(&plan.preinventory); + let Some(work) = list_source_work(store, plan, preinventory, class, fetch_limit)? + else { + continue; + }; + match work { + SelectedImportWork::Catalog(work) => { + candidates.extend( + work.into_iter() + .map(|work| ImportCandidate::Catalog { source_index, work }) + .take(slice_limit), + ); + } + SelectedImportWork::SourceFiles(work) => { + candidates.extend( + work.into_iter() + .map(|work| ImportCandidate::SourceFile { source_index, work }) + .take(slice_limit), + ); + } + } + } + candidates.sort_by(|left, right| self.compare_candidates(left, right)); + if store.has_pending_provider_file_publications()? { + if !candidates + .first() + .is_some_and(ImportCandidate::has_active_publication) + { + return Ok(ImportSlice::empty()); + } + if state.has_attempted(&candidates[0].identity()) { + return Ok(ImportSlice::empty()); + } + candidates.truncate(1); + } else { + candidates.retain(|candidate| !state.has_attempted(&candidate.identity())); + } + + let mut slice = ImportSlice::empty(); + for candidate in candidates { + if slice.units >= slice_limit { + break; + } + let bytes = candidate.estimated_bytes(); + let exceeds_target = + slice.units > 0 && slice.bytes.saturating_add(bytes) > IMPORT_SLICE_TARGET_BYTES; + if exceeds_target { + break; + } + slice.units += 1; + slice.bytes = slice.bytes.saturating_add(bytes); + match candidate { + ImportCandidate::Retirement(work) => slice.retirements.push(work), + ImportCandidate::Catalog { source_index, work } => push_source_candidate( + &mut slice, + source_index, + self.selected_preinventory(state, source_index), + SelectedCandidate::Catalog(work), + ), + ImportCandidate::SourceFile { source_index, work } => push_source_candidate( + &mut slice, + source_index, + self.selected_preinventory(state, source_index), + SelectedCandidate::SourceFile(work), + ), + } + } + Ok(slice) + } + + fn selected_preinventory( + &self, + state: &ImportExecutionState, + source_index: usize, + ) -> SourcePreinventory { + state + .observed_preinventories + .get(source_index) + .and_then(Option::as_ref) + .unwrap_or(&self.sources[source_index].preinventory) + .clone() + } + + fn compare_candidates(&self, left: &ImportCandidate, right: &ImportCandidate) -> Ordering { + let publication_order = right + .has_active_publication() + .cmp(&left.has_active_publication()); + if publication_order != Ordering::Equal { + return publication_order; + } + let attempt_order = left + .last_attempt_at_ms() + .is_some() + .cmp(&right.last_attempt_at_ms().is_some()) + .then_with(|| left.last_attempt_at_ms().cmp(&right.last_attempt_at_ms())); + if attempt_order != Ordering::Equal { + return attempt_order; + } + left.stable_identity(&self.sources) + .cmp(&right.stable_identity(&self.sources)) + } + + fn revalidate_slice( + &self, + store: &Store, + class: ImportWorkClass, + max_units: usize, + state: &mut ImportExecutionState, + provisional: &ImportSlice, + mut validation_failures: Vec, + ) -> Result<(ImportSlice, Vec)> { + let mut eligible_sources = BTreeSet::new(); + for selected in &provisional.sources { + if self.selection_needs_prelock_observation(selected) + && state.observed_preinventories[selected.source_index].is_none() + { + continue; + } + eligible_sources.insert(selected.source_index); + } + validation_failures.extend(self.refresh_changed_manifest_units( + store, + provisional, + state, + )?); + let mut slice = + self.select_slice_with_state(store, class, max_units, state, Some(&eligible_sources))?; + validation_failures + .extend(self.retain_current_file_observations(store, &mut slice, state)?); + retain_current_generations(store, &mut slice, state)?; + Ok((slice, validation_failures)) + } + + fn observe_provisional_sources( + &self, + store: &Store, + state: &mut ImportExecutionState, + provisional: &ImportSlice, + ) -> Result> { + let mut failures = Vec::new(); + for selected in &provisional.sources { + if !self.selection_needs_prelock_observation(selected) + || state.observed_preinventories[selected.source_index].is_some() + { + continue; + } + let plan = &self.sources[selected.source_index]; + match observe_current_preinventory(store, plan, selected) { + Ok(preinventory) => { + state.observed_preinventories[selected.source_index] = Some(preinventory); + } + Err(error) + if import_error_scope(&error) == ImportFailureScope::Source + && import_failure_type(&error) == ImportFailureType::NotFound + && matches!( + &selected.work, + SelectedImportWork::SourceFiles(work) + if work.iter().any(|candidate| candidate.has_active_publication) + ) => + { + let inventory_generation = selected + .preinventory + .inventory_generation() + .unwrap_or_default(); + if let SelectedImportWork::SourceFiles(work) = &selected.work { + for candidate in work + .iter() + .filter(|candidate| candidate.has_active_publication) + { + invalidate_source_file_publication_observation( + store, + candidate, + inventory_generation, + )?; + state.mark_validation_skip(source_file_work_identity(candidate)); + } + } + } + Err(error) if import_error_scope(&error) == ImportFailureScope::Source => { + state.record_source_attempt(&selected.work); + failures.push(SourceValidationFailure { + source_index: selected.source_index, + stats: selected.stats, + error, + }); + } + Err(error) => return Err(error), + } + } + Ok(failures) + } + + fn selection_needs_prelock_observation(&self, selected: &SelectedImportSource) -> bool { + let SelectedImportWork::SourceFiles(work) = &selected.work else { + return false; + }; + !source_uses_import_file_manifest(&self.sources[selected.source_index].source) + || work + .iter() + .any(|candidate| candidate.has_active_publication) + } + + fn refresh_changed_manifest_units( + &self, + store: &Store, + provisional: &ImportSlice, + state: &mut ImportExecutionState, + ) -> Result> { + let mut failures = Vec::new(); + for selected in &provisional.sources { + let SourcePreinventory::SourceImportFiles { + inventory_generation, + .. + } = &selected.preinventory + else { + continue; + }; + let SelectedImportWork::SourceFiles(work) = &selected.work else { + continue; + }; + let source = &self.sources[selected.source_index].source; + for candidate in work { + let (current, current_generation, persist_current) = + if let Some(SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + }) = state + .observed_preinventories + .get(selected.source_index) + .and_then(Option::as_ref) + { + let current = files + .iter() + .find(|file| file.source_path == candidate.file.source_path) + .cloned(); + ( + Ok(current), + *inventory_generation, + *inventory_generation + == selected.preinventory.inventory_generation().unwrap_or(0), + ) + } else if source_uses_import_file_manifest(source) { + ( + observe_selected_source_import_file( + source, + &candidate.file.source_path, + ), + *inventory_generation, + true, + ) + } else { + let Some(SourcePreinventory::SourceRoot { + file, + inventory_generation, + }) = state + .observed_preinventories + .get(selected.source_index) + .and_then(Option::as_ref) + else { + continue; + }; + (Ok(Some(file.clone())), *inventory_generation, false) + }; + match current { + Ok(Some(current)) => { + let compatible_append = active_append_observation_is_compatible( + store, source, candidate, ¤t, + )?; + if !compatible_append + && !same_source_import_observation(&candidate.file, ¤t) + { + let invalidated_publication = + invalidate_source_file_publication_observation( + store, + candidate, + *inventory_generation, + )?; + if persist_current { + store.upsert_source_import_files( + current_generation, + std::slice::from_ref(¤t), + )?; + } + if invalidated_publication { + state.mark_validation_skip(source_file_work_identity(candidate)); + } + } + } + Ok(None) => { + invalidate_source_file_publication_observation( + store, + candidate, + *inventory_generation, + )?; + state.mark_validation_skip(source_file_work_identity(candidate)); + if !candidate.has_active_publication { + failures.push(SourceValidationFailure { + source_index: selected.source_index, + stats: SourceStats { + files: 1, + bytes: candidate.estimated_bytes, + change_token: None, + }, + error: anyhow::Error::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!( + "import unit disappeared before indexing: {}", + candidate.file.source_path + ), + )), + }); + } + } + Err(error) if import_error_scope(&error) == ImportFailureScope::Source => { + state.mark_validation_skip(source_file_work_identity(candidate)); + failures.push(SourceValidationFailure { + source_index: selected.source_index, + stats: SourceStats { + files: 1, + bytes: candidate.estimated_bytes, + change_token: None, + }, + error, + }); + } + Err(error) => return Err(error), + } + } + } + Ok(failures) + } + + fn retain_current_file_observations( + &self, + store: &Store, + slice: &mut ImportSlice, + state: &mut ImportExecutionState, + ) -> Result> { + let mut failures = Vec::new(); + for selected in &mut slice.sources { + let source = &self.sources[selected.source_index].source; + match &mut selected.work { + SelectedImportWork::Catalog(work) => { + let mut retained = Vec::with_capacity(work.len()); + for candidate in std::mem::take(work) { + let allow_append = candidate.has_active_publication + && source.mutation_contract + == ctx_history_capture::ProviderFileMutationContract::AppendOnlyNewlineDelimited + && store.effective_provider_file_publication_has_staged_completion()?; + if file_observation_is_current_or_compatible_append( + Path::new(&candidate.session.source_path), + candidate.session.file_size_bytes, + candidate.session.file_modified_at_ms, + &candidate.session.metadata, + allow_append, + ) { + retained.push(candidate); + } else { + invalidate_catalog_publication_observation( + store, + &candidate, + selected.preinventory.inventory_generation().unwrap_or(0), + )?; + state.mark_validation_skip(catalog_work_identity(&candidate)); + } + } + *work = retained; + } + SelectedImportWork::SourceFiles(_) + if matches!( + &selected.preinventory, + SourcePreinventory::SourceRoot { .. } + ) || !source_uses_import_file_manifest(source) => {} + SelectedImportWork::SourceFiles(work) => { + let mut retained = Vec::with_capacity(work.len()); + for candidate in std::mem::take(work) { + match observe_selected_source_import_file( + source, + &candidate.file.source_path, + ) { + Ok(Some(current)) => { + if same_source_import_observation(&candidate.file, ¤t) + || active_append_observation_is_compatible( + store, source, &candidate, ¤t, + )? + { + retained.push(candidate); + } else { + state.mark_validation_skip(source_file_work_identity( + &candidate, + )); + } + } + Ok(None) => { + state.mark_validation_skip(source_file_work_identity(&candidate)); + } + Err(error) + if import_error_scope(&error) == ImportFailureScope::Source => + { + state.mark_validation_skip(source_file_work_identity(&candidate)); + failures.push(SourceValidationFailure { + source_index: selected.source_index, + stats: SourceStats { + files: 1, + bytes: candidate.estimated_bytes, + change_token: None, + }, + error, + }); + } + Err(error) => return Err(error), + } + } + *work = retained; + } + } + selected.stats.files = selected.work.unit_count(); + selected.stats.bytes = selected_work_bytes(&selected.work); + } + recompute_slice_totals(slice); + Ok(failures) + } +} diff --git a/crates/ctx-cli/src/commands/import/scheduler/selection.rs b/crates/ctx-cli/src/commands/import/scheduler/selection.rs new file mode 100644 index 00000000..6262fef2 --- /dev/null +++ b/crates/ctx-cli/src/commands/import/scheduler/selection.rs @@ -0,0 +1,688 @@ +enum ImportCandidate { + Retirement(ProviderFilePublicationRetirementWork), + Catalog { + source_index: usize, + work: CatalogImportWork, + }, + SourceFile { + source_index: usize, + work: SourceImportFileWork, + }, +} + +impl ImportCandidate { + fn has_active_publication(&self) -> bool { + match self { + Self::Retirement(_) => true, + Self::Catalog { work, .. } => work.has_active_publication, + Self::SourceFile { work, .. } => work.has_active_publication, + } + } + + fn estimated_bytes(&self) -> u64 { + match self { + Self::Retirement(work) => work.estimated_bytes, + Self::Catalog { work, .. } => work.estimated_bytes, + Self::SourceFile { work, .. } => work.estimated_bytes, + } + } + + fn last_attempt_at_ms(&self) -> Option { + match self { + Self::Retirement(work) => Some(work.last_attempt_at_ms), + Self::Catalog { work, .. } => work.last_attempt_at_ms, + Self::SourceFile { work, .. } => work.last_attempt_at_ms, + } + } + + fn identity(&self) -> String { + match self { + Self::Retirement(work) => retirement_work_identity(work), + Self::Catalog { work, .. } => catalog_work_identity(work), + Self::SourceFile { work, .. } => source_file_work_identity(work), + } + } + + fn stable_identity(&self, sources: &[PlannedImportSource]) -> String { + match self { + Self::Retirement(work) => format!( + "{}\u{0}{}\u{0}{}\u{0}{}", + work.provider.as_str(), + work.material_source_format, + work.material_source_root, + work.source_path + ), + Self::Catalog { source_index, work } => { + stable_source_work_identity(&sources[*source_index], &work.session.source_path) + } + Self::SourceFile { source_index, work } => { + stable_source_work_identity(&sources[*source_index], &work.file.source_path) + } + } + } +} + +enum SelectedCandidate { + Catalog(CatalogImportWork), + SourceFile(SourceImportFileWork), +} + +fn push_source_candidate( + slice: &mut ImportSlice, + source_index: usize, + preinventory: SourcePreinventory, + candidate: SelectedCandidate, +) { + let bytes = match &candidate { + SelectedCandidate::Catalog(work) => work.estimated_bytes, + SelectedCandidate::SourceFile(work) => work.estimated_bytes, + }; + if let Some(selected) = slice + .sources + .iter_mut() + .find(|selected| selected.source_index == source_index) + { + match (&mut selected.work, candidate) { + (SelectedImportWork::Catalog(work), SelectedCandidate::Catalog(candidate)) => { + work.push(candidate); + } + (SelectedImportWork::SourceFiles(work), SelectedCandidate::SourceFile(candidate)) => { + work.push(candidate); + } + _ => unreachable!("one source cannot mix catalog and source-file work"), + } + selected.stats.files = selected.stats.files.saturating_add(1); + selected.stats.bytes = selected.stats.bytes.saturating_add(bytes); + return; + } + + let work = match candidate { + SelectedCandidate::Catalog(work) => SelectedImportWork::Catalog(vec![work]), + SelectedCandidate::SourceFile(work) => SelectedImportWork::SourceFiles(vec![work]), + }; + slice.sources.push(SelectedImportSource { + source_index, + preinventory, + work, + stats: SourceStats { + files: 1, + bytes, + change_token: None, + }, + attempts_persisted: false, + }); +} + +fn retain_current_generations( + store: &Store, + slice: &mut ImportSlice, + state: &mut ImportExecutionState, +) -> Result<()> { + for selected in &mut slice.sources { + match (&selected.preinventory, &mut selected.work) { + ( + SourcePreinventory::CodexSessionCatalog { + inventory_generation, + .. + }, + SelectedImportWork::Catalog(work), + ) => { + let mut retained = Vec::with_capacity(work.len()); + for candidate in std::mem::take(work) { + match persist_catalog_attempt_started(store, &candidate, *inventory_generation) + { + Ok(1) => retained.push(candidate), + Ok(_) => state.mark_validation_skip(catalog_work_identity(&candidate)), + Err(error) if provider_publication_blocks_attempt(&error) => { + if catalog_candidate_matches_publication( + store, + &candidate, + *inventory_generation, + )? { + retained.push(candidate); + } else { + state.mark_validation_skip(catalog_work_identity(&candidate)); + } + } + Err(error) => return Err(error), + } + } + *work = retained; + } + ( + SourcePreinventory::SourceRoot { + inventory_generation, + .. + } + | SourcePreinventory::SourceImportFiles { + inventory_generation, + .. + }, + SelectedImportWork::SourceFiles(work), + ) => { + let mut retained = Vec::with_capacity(work.len()); + for candidate in std::mem::take(work) { + match persist_source_file_attempt_started( + store, + &candidate, + *inventory_generation, + ) { + Ok(1) => retained.push(candidate), + Ok(_) => state.mark_validation_skip(source_file_work_identity(&candidate)), + Err(error) if provider_publication_blocks_attempt(&error) => { + if source_file_candidate_matches_publication( + store, + &candidate, + *inventory_generation, + )? { + retained.push(candidate); + } else { + state.mark_validation_skip(source_file_work_identity(&candidate)); + } + } + Err(error) => return Err(error), + } + } + *work = retained; + } + _ => {} + } + selected.stats.files = selected.work.unit_count(); + selected.stats.bytes = selected_work_bytes(&selected.work); + selected.attempts_persisted = true; + } + recompute_slice_totals(slice); + Ok(()) +} + +fn catalog_candidate_matches_publication( + store: &Store, + candidate: &CatalogImportWork, + inventory_generation: u64, +) -> Result { + let material_source_format = canonical_provider_material_source_format( + candidate.session.provider, + &candidate.session.source_format, + ) + .unwrap_or(&candidate.session.source_format); + store + .provider_file_publication_matches_candidate( + candidate.session.provider, + ctx_history_store::ProviderFileInventoryObservation::ObservedCatalog { + source_format: &candidate.session.source_format, + update: ctx_history_store::CatalogSourceIndexUpdate { + source_root: &candidate.session.source_root, + source_path: &candidate.session.source_path, + file_size_bytes: candidate.session.file_size_bytes, + file_modified_at_ms: candidate.session.file_modified_at_ms, + import_revision: candidate.session.import_revision, + inventory_generation, + file_sha256: None, + event_count: None, + indexed_at_ms: 0, + }, + metadata: &candidate.session.metadata, + }, + material_source_format, + &candidate.session.source_root, + ) + .map_err(Into::into) +} + +fn source_file_candidate_matches_publication( + store: &Store, + candidate: &SourceImportFileWork, + inventory_generation: u64, +) -> Result { + let material_source_format = canonical_provider_material_source_format( + candidate.file.provider, + &candidate.file.source_format, + ) + .unwrap_or(&candidate.file.source_format); + store + .provider_file_publication_matches_candidate( + candidate.file.provider, + ctx_history_store::ProviderFileInventoryObservation::SourceImport { + source_format: &candidate.file.source_format, + update: ctx_history_store::SourceImportFileIndexUpdate { + source_root: &candidate.file.source_root, + source_path: &candidate.file.source_path, + file_size_bytes: candidate.file.file_size_bytes, + file_modified_at_ms: candidate.file.file_modified_at_ms, + import_revision: candidate.file.import_revision, + inventory_generation, + metadata: &candidate.file.metadata, + indexed_at_ms: 0, + }, + }, + material_source_format, + &candidate.file.source_root, + ) + .map_err(Into::into) +} + +fn invalidate_catalog_publication_observation( + store: &Store, + candidate: &CatalogImportWork, + inventory_generation: u64, +) -> Result { + let Some(owner) = store.effective_provider_file_publication_inventory_owner()? else { + return Ok(false); + }; + let metadata_matches = owner.metadata_json.as_deref().is_none_or(|metadata| { + serde_json::from_str::(metadata) + .is_ok_and(|metadata| metadata == candidate.session.metadata) + }); + if owner.inventory_family != ProviderFileInventoryFamily::Catalog + || owner.provider != candidate.session.provider + || owner.source_format != candidate.session.source_format + || owner.source_root != candidate.session.source_root + || owner.source_path != candidate.session.source_path + || owner.inventory_generation != inventory_generation + || owner.file_size_bytes != candidate.session.file_size_bytes + || owner.file_modified_at_ms != candidate.session.file_modified_at_ms + || owner.import_revision != candidate.session.import_revision + || !metadata_matches + { + return Ok(false); + } + store + .invalidate_effective_provider_file_publication_observation( + &owner, + ctx_history_core::utc_now().timestamp_millis(), + ) + .map_err(Into::into) +} + +fn invalidate_source_file_publication_observation( + store: &Store, + candidate: &SourceImportFileWork, + inventory_generation: u64, +) -> Result { + let Some(owner) = store.effective_provider_file_publication_inventory_owner()? else { + return Ok(false); + }; + let metadata_matches = owner + .metadata_json + .as_deref() + .and_then(|metadata| serde_json::from_str::(metadata).ok()) + .is_some_and(|metadata| metadata == candidate.file.metadata); + if owner.inventory_family != ProviderFileInventoryFamily::SourceImport + || owner.provider != candidate.file.provider + || owner.source_format != candidate.file.source_format + || owner.source_root != candidate.file.source_root + || owner.source_path != candidate.file.source_path + || owner.inventory_generation != inventory_generation + || owner.file_size_bytes != candidate.file.file_size_bytes + || owner.file_modified_at_ms != candidate.file.file_modified_at_ms + || owner.import_revision != candidate.file.import_revision + || !metadata_matches + { + return Ok(false); + } + store + .invalidate_effective_provider_file_publication_observation( + &owner, + ctx_history_core::utc_now().timestamp_millis(), + ) + .map_err(Into::into) +} + +fn recompute_slice_totals(slice: &mut ImportSlice) { + slice + .sources + .retain(|selected| selected.work.unit_count() > 0); + slice.units = slice + .retirements + .len() + .saturating_add(slice.sources.iter().map(|source| source.stats.files).sum()); + slice.bytes = slice + .retirements + .iter() + .fold(0_u64, |total, work| { + total.saturating_add(work.estimated_bytes) + }) + .saturating_add(slice.sources.iter().fold(0_u64, |total, source| { + total.saturating_add(source.stats.bytes) + })); +} + +fn file_observation_is_current_or_compatible_append( + path: &Path, + expected_bytes: u64, + expected_modified_ms: i64, + expected_metadata: &serde_json::Value, + allow_append: bool, +) -> bool { + let Ok(observation) = ctx_history_capture::observe_ordinary_file(path) else { + return false; + }; + let current_bytes = observation.len(); + let token_matches = expected_metadata + .get("file_observation_token_v1") + .and_then(serde_json::Value::as_str) + .is_some_and(|expected| expected == observation.token_hex()); + (current_bytes == expected_bytes + && super::catalog::system_time_ms(observation.modified_at()) == expected_modified_ms + && token_matches) + || (allow_append && current_bytes > expected_bytes) +} + +fn active_append_observation_is_compatible( + store: &Store, + source: &crate::provider_sources::SourceInfo, + candidate: &SourceImportFileWork, + current: &ctx_history_store::SourceImportFile, +) -> Result { + Ok(candidate.has_active_publication + && source.mutation_contract + == ctx_history_capture::ProviderFileMutationContract::AppendOnlyNewlineDelimited + && store.effective_provider_file_publication_has_staged_completion()? + && candidate.file.provider == current.provider + && candidate.file.source_format == current.source_format + && candidate.file.source_root == current.source_root + && candidate.file.source_path == current.source_path + && candidate.file.import_revision == current.import_revision + && append_observation_metadata_is_compatible(&candidate.file.metadata, ¤t.metadata) + && current.file_size_bytes > candidate.file.file_size_bytes) +} + +fn append_observation_metadata_is_compatible( + previous: &serde_json::Value, + current: &serde_json::Value, +) -> bool { + let (Some(mut previous), Some(mut current)) = + (previous.as_object().cloned(), current.as_object().cloned()) + else { + return false; + }; + if !previous + .get("dependencies") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) + || !current + .get("dependencies") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) + { + return false; + } + previous.remove("change_token_v1"); + current.remove("change_token_v1"); + previous == current +} + +fn selected_work_bytes(work: &SelectedImportWork) -> u64 { + match work { + SelectedImportWork::Catalog(work) => work.iter().fold(0_u64, |total, work| { + total.saturating_add(work.estimated_bytes) + }), + SelectedImportWork::SourceFiles(work) => work.iter().fold(0_u64, |total, work| { + total.saturating_add(work.estimated_bytes) + }), + } +} + +fn stable_source_work_identity(plan: &PlannedImportSource, source_path: &str) -> String { + format!( + "{}\u{0}{}\u{0}{}\u{0}{}", + plan.source.provider.as_str(), + plan.source.source_format, + plan.source.path.display(), + source_path + ) +} + +fn retirement_work_identity(work: &ProviderFilePublicationRetirementWork) -> String { + format!( + "retirement\u{0}{}\u{0}{}\u{0}{}\u{0}{}", + work.provider.as_str(), + work.material_source_format, + work.material_source_root, + work.source_path + ) +} + +fn catalog_work_identity(work: &CatalogImportWork) -> String { + format!( + "catalog\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}", + work.session.provider.as_str(), + work.session.source_format, + work.session.source_root, + work.session.source_path, + work.session.file_size_bytes, + work.session.file_modified_at_ms, + work.session.import_revision, + ) +} + +fn source_file_work_identity(work: &SourceImportFileWork) -> String { + format!( + "source-file\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}\u{0}{}", + work.file.provider.as_str(), + work.file.source_format, + work.file.source_root, + work.file.source_path, + work.file.file_size_bytes, + work.file.file_modified_at_ms, + work.file.import_revision, + work.file.metadata, + ) +} + +fn observe_current_preinventory( + store: &Store, + plan: &PlannedImportSource, + selected: &SelectedImportSource, +) -> Result { + if source_uses_import_file_manifest(&plan.source) { + let (files, inventory_generation) = match (&selected.preinventory, &selected.work) { + ( + SourcePreinventory::SourceImportFiles { + inventory_generation, + .. + }, + SelectedImportWork::SourceFiles(work), + ) => { + let candidate = work + .iter() + .find(|candidate| candidate.has_active_publication) + .ok_or_else(|| { + anyhow::Error::new(ctx_history_capture::CaptureError::SystemInvariant( + "pre-lock manifest observation has no active publication candidate", + )) + })?; + let current = + observe_selected_source_import_file(&plan.source, &candidate.file.source_path)?; + let observation_changed = current.as_ref().is_none_or(|current| { + !same_source_import_observation(&candidate.file, current) + }); + let generation_is_current = store.current_source_import_inventory_generation( + plan.source.provider, + &candidate.file.source_root, + )? == Some(*inventory_generation); + if observation_changed && !generation_is_current { + let files = collect_source_import_files(&plan.source)?; + let persisted = + persist_new_source_import_observation(store, &plan.source, &files)?; + (files, persisted.inventory_generation) + } else { + let mut files = match &selected.preinventory { + SourcePreinventory::SourceImportFiles { files, .. } => files.clone(), + _ => unreachable!("matched manifested source-file preinventory"), + }; + let owner_was_cached = files + .iter() + .any(|file| file.source_path == candidate.file.source_path); + if observation_changed || !owner_was_cached { + files.retain(|file| file.source_path != candidate.file.source_path); + files.extend(current); + files.sort_by(|left, right| left.source_path.cmp(&right.source_path)); + } + (files, *inventory_generation) + } + } + _ => { + return Err(anyhow::Error::new( + ctx_history_capture::CaptureError::SystemInvariant( + "manifest publication selection has no source-file inventory", + ), + )); + } + }; + return Ok(SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + }); + } + let (_, file) = observe_source_root(&plan.source)?; + let persisted = + persist_new_source_import_observation(store, &plan.source, std::slice::from_ref(&file))?; + Ok(SourcePreinventory::SourceRoot { + file, + inventory_generation: persisted.inventory_generation, + }) +} + +fn list_source_work( + store: &Store, + plan: &PlannedImportSource, + preinventory: &SourcePreinventory, + class: ImportWorkClass, + limit: usize, +) -> Result> { + match preinventory { + SourcePreinventory::CodexSessionCatalog { .. } => { + let source_root = super::catalog::codex_catalog_root_identity(&plan.source.path)?; + Ok(Some(SelectedImportWork::Catalog( + store.list_catalog_import_work(plan.source.provider, source_root, class, limit)?, + ))) + } + SourcePreinventory::SourceImportFiles { files, .. } => { + let source_root = files + .first() + .map(|file| file.source_root.as_str()) + .map(Ok) + .unwrap_or_else(|| persisted_import_identity(&plan.source.path, "source root"))?; + Ok(Some(SelectedImportWork::SourceFiles( + store.list_source_import_file_work( + plan.source.provider, + source_root, + class, + limit, + )?, + ))) + } + SourcePreinventory::SourceRoot { file, .. } => Ok(Some(SelectedImportWork::SourceFiles( + store.list_source_import_file_work( + plan.source.provider, + &file.source_root, + class, + limit, + )?, + ))), + SourcePreinventory::None => Ok(None), + } +} + +fn import_work_counts(store: &Store, sources: &[PlannedImportSource]) -> Result<(usize, usize)> { + Ok(( + import_work_count(store, sources, ImportWorkClass::Fresh)?, + import_work_count(store, sources, ImportWorkClass::Recovery)?, + )) +} + +fn import_work_count( + store: &Store, + sources: &[PlannedImportSource], + class: ImportWorkClass, +) -> Result { + let mut count = 0usize; + if class == ImportWorkClass::Recovery { + count = store + .list_provider_file_publication_retirement_work(IMPORT_PENDING_REPORT_LIMIT)? + .len(); + } + for plan in sources { + let remaining = IMPORT_PENDING_REPORT_LIMIT.saturating_sub(count); + if remaining == 0 { + break; + } + let source_count = match &plan.preinventory { + SourcePreinventory::CodexSessionCatalog { .. } => { + let source_root = super::catalog::codex_catalog_root_identity(&plan.source.path)?; + store + .list_catalog_import_work(plan.source.provider, source_root, class, remaining)? + .len() + } + SourcePreinventory::SourceImportFiles { files, .. } => { + let source_root = files + .first() + .map(|file| file.source_root.as_str()) + .map(Ok) + .unwrap_or_else(|| { + persisted_import_identity(&plan.source.path, "source root") + })?; + store + .list_source_import_file_work( + plan.source.provider, + source_root, + class, + remaining, + )? + .len() + } + SourcePreinventory::SourceRoot { file, .. } => store + .list_source_import_file_work( + plan.source.provider, + &file.source_root, + class, + remaining, + )? + .len(), + SourcePreinventory::None => continue, + }; + count = count.saturating_add(source_count); + } + Ok(count.min(IMPORT_PENDING_REPORT_LIMIT)) +} + +pub(crate) fn bounded_unplanned_root_work_counts( + store: &Store, + provider: ctx_history_core::CaptureProvider, + source_root: &str, +) -> Result<(usize, usize)> { + Ok(( + bounded_unplanned_root_work_count(store, provider, source_root, ImportWorkClass::Fresh)?, + bounded_unplanned_root_work_count(store, provider, source_root, ImportWorkClass::Recovery)?, + )) +} + +fn bounded_unplanned_root_work_count( + store: &Store, + provider: ctx_history_core::CaptureProvider, + source_root: &str, + class: ImportWorkClass, +) -> Result { + let catalog = store.list_catalog_import_work( + provider, + source_root, + class, + IMPORT_PENDING_REPORT_LIMIT, + )?; + let remaining = IMPORT_PENDING_REPORT_LIMIT.saturating_sub(catalog.len()); + let source_files = + store.list_source_import_file_work(provider, source_root, class, remaining)?; + Ok(catalog + .len() + .saturating_add(source_files.len()) + .min(IMPORT_PENDING_REPORT_LIMIT)) +} + +fn execution_budget(reported_units: usize) -> usize { + if reported_units >= IMPORT_PENDING_REPORT_LIMIT { + usize::MAX + } else { + reported_units + } +} diff --git a/crates/ctx-cli/src/commands/import/scheduler/tests.rs b/crates/ctx-cli/src/commands/import/scheduler/tests.rs new file mode 100644 index 00000000..2accb2f5 --- /dev/null +++ b/crates/ctx-cli/src/commands/import/scheduler/tests.rs @@ -0,0 +1,775 @@ +#[cfg(test)] +fn admit_catalog_work( + slice: &mut ImportSlice, + work: Vec, +) -> Vec { + admit_work(slice, work, |work| work.estimated_bytes) +} + +#[cfg(test)] +fn admit_work( + slice: &mut ImportSlice, + work: Vec, + estimated_bytes: impl Fn(&T) -> u64, +) -> Vec { + let mut admitted = Vec::new(); + for unit in work { + let bytes = estimated_bytes(&unit); + let exceeds_target = + slice.units > 0 && slice.bytes.saturating_add(bytes) > IMPORT_SLICE_TARGET_BYTES; + if slice.units >= IMPORT_SLICE_MAX_UNITS || exceeds_target { + break; + } + slice.units += 1; + slice.bytes = slice.bytes.saturating_add(bytes); + admitted.push(unit); + if slice.units >= IMPORT_SLICE_MAX_UNITS || slice.bytes >= IMPORT_SLICE_TARGET_BYTES { + break; + } + } + admitted +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::import::{ + import_totals_json, import_work_progress_done, import_work_progress_message, ImportTotals, + }; + use crate::provider_sources::explicit_path_source; + use ctx_history_core::{AgentType, CaptureProvider}; + use ctx_history_store::{ + CatalogIndexedStatus, CatalogSession, ImportPendingReason, SourceImportFile, + SourceImportFileIndexUpdate, + }; + use serde_json::json; + + #[test] + fn append_observation_metadata_only_allows_change_token_updates() { + let previous = json!({ + "change_token_v1": {"size": 64}, + "dependencies": [], + "session_id": "session-1", + }); + let current = json!({ + "change_token_v1": {"size": 128}, + "dependencies": [], + "session_id": "session-1", + }); + assert!(append_observation_metadata_is_compatible( + &previous, ¤t + )); + + let changed_dependencies = json!({ + "change_token_v1": {"size": 128}, + "dependencies": ["metadata.json"], + "session_id": "session-1", + }); + assert!(!append_observation_metadata_is_compatible( + &previous, + &changed_dependencies, + )); + + let missing_dependencies = json!({ + "change_token_v1": {"size": 128}, + "session_id": "session-1", + }); + assert!(!append_observation_metadata_is_compatible( + &previous, + &missing_dependencies, + )); + + let changed_identity = json!({ + "change_token_v1": {"size": 128}, + "dependencies": [], + "session_id": "session-2", + }); + assert!(!append_observation_metadata_is_compatible( + &previous, + &changed_identity, + )); + } + + fn catalog_work(path: &str, bytes: u64) -> CatalogImportWork { + CatalogImportWork { + session: CatalogSession { + provider: CaptureProvider::Codex, + source_format: "codex_session_jsonl_tree".to_owned(), + source_root: "/sessions".to_owned(), + source_path: path.to_owned(), + external_session_id: Some(path.to_owned()), + parent_external_session_id: None, + agent_type: AgentType::Primary, + role_hint: None, + external_agent_id: None, + cwd: None, + session_started_at_ms: None, + file_size_bytes: bytes, + file_modified_at_ms: 1, + import_revision: 1, + cataloged_at_ms: 1, + metadata: json!({}), + }, + reason: ImportPendingReason::FreshNew, + estimated_bytes: bytes, + last_attempt_at_ms: None, + has_active_publication: false, + } + } + + fn recovery_source( + store: &Store, + root: &str, + attempted_at_ms: Option, + ) -> (PlannedImportSource, SourceImportFile, u64) { + let source = explicit_path_source(CaptureProvider::Pi, root.into()); + let file = SourceImportFile { + provider: CaptureProvider::Pi, + source_format: source.source_format.to_owned(), + source_root: root.to_owned(), + source_path: format!("{root}/session.jsonl"), + file_size_bytes: 64, + file_modified_at_ms: 100, + import_revision: 1, + observed_at_ms: 100, + metadata: json!({}), + }; + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Pi, root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + match attempted_at_ms { + Some(indexed_at_ms) => { + store + .record_source_import_file_result( + CaptureProvider::Pi, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms, + }, + CatalogIndexedStatus::Failed, + Some("deterministic failure"), + ) + .unwrap(); + } + None => { + store + .record_source_import_file_result( + CaptureProvider::Pi, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: 100, + }, + CatalogIndexedStatus::Indexed, + None, + ) + .unwrap(); + store + .schedule_source_import_explicit_rescan(CaptureProvider::Pi, root, generation) + .unwrap(); + } + } + ( + PlannedImportSource { + source, + stats: SourceStats::default(), + preinventory: SourcePreinventory::SourceImportFiles { + files: vec![file.clone()], + inventory_generation: generation, + }, + }, + file, + generation, + ) + } + + #[test] + fn slice_admits_one_oversized_unit() { + let mut slice = ImportSlice::empty(); + let admitted = admit_catalog_work( + &mut slice, + vec![ + catalog_work("oversized", IMPORT_SLICE_TARGET_BYTES + 1), + catalog_work("later", 1), + ], + ); + assert_eq!(admitted.len(), 1); + assert_eq!(slice.units, 1); + assert_eq!(slice.bytes, IMPORT_SLICE_TARGET_BYTES + 1); + } + + #[test] + fn slice_caps_units_and_bytes() { + let mut slice = ImportSlice::empty(); + let admitted = admit_catalog_work( + &mut slice, + (0..100) + .map(|index| catalog_work(&format!("unit-{index:03}"), 1)) + .collect(), + ); + assert_eq!(admitted.len(), IMPORT_SLICE_MAX_UNITS); + + let mut slice = ImportSlice::empty(); + let admitted = admit_catalog_work( + &mut slice, + vec![ + catalog_work("first", IMPORT_SLICE_TARGET_BYTES - 1), + catalog_work("second", 2), + ], + ); + assert_eq!(admitted.len(), 1); + } + + #[test] + fn locked_revalidation_preserves_missing_material_recovery_work() { + let temp = tempfile::tempdir().unwrap(); + let source_path = temp.path().join("state.json"); + std::fs::write(&source_path, b"{}").unwrap(); + let source = explicit_path_source(CaptureProvider::CodeBuddy, source_path); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (stats, file) = observe_source_root(&source).unwrap(); + let first = + persist_new_source_import_observation(&store, &source, std::slice::from_ref(&file)) + .unwrap(); + store + .record_source_import_file_result( + source.provider, + SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: first.inventory_generation, + metadata: &file.metadata, + indexed_at_ms: 1, + }, + CatalogIndexedStatus::Indexed, + None, + ) + .unwrap(); + let current = + persist_new_source_import_observation(&store, &source, std::slice::from_ref(&file)) + .unwrap(); + let plan = ImportPlan::build( + &store, + vec![PlannedImportSource { + source, + stats, + preinventory: SourcePreinventory::SourceRoot { + file, + inventory_generation: current.inventory_generation, + }, + }], + ) + .unwrap(); + assert_eq!(plan.recovery_units, 1); + let mut execution_state = ImportExecutionState::for_plan(&plan); + + let executable = plan + .select_slice_for_execution_with_pre_lock_hook( + &store, + ImportWorkClass::Recovery, + plan.recovery_units, + &mut execution_state, + || {}, + ) + .unwrap() + .unwrap(); + assert_eq!(executable.slice.units, 1); + let SelectedImportWork::SourceFiles(work) = &executable.slice.sources[0].work else { + panic!("source-root recovery must select source-file work"); + }; + assert_eq!(work.len(), 1); + assert_eq!(work[0].reason, ImportPendingReason::MissingMaterial); + store + .finish_event_search_bulk_mode(&executable.bulk_guard) + .unwrap(); + } + + #[test] + fn execution_policies_bound_only_the_intended_phases() { + assert_eq!(ImportExecutionPolicy::Drain.fresh_slice_limit(), None); + assert_eq!(ImportExecutionPolicy::Drain.recovery_slice_limit(), None); + assert_eq!( + ImportExecutionPolicy::Interactive.fresh_slice_limit(), + Some(1) + ); + assert_eq!( + ImportExecutionPolicy::Interactive.recovery_slice_limit(), + Some(1) + ); + assert_eq!(ImportExecutionPolicy::Daemon.fresh_slice_limit(), Some(1)); + assert_eq!( + ImportExecutionPolicy::Daemon.recovery_slice_limit(), + Some(1) + ); + assert_eq!( + ImportExecutionPolicy::Drain.disk_io_limits(), + (64 * MEBIBYTE, 8 * MEBIBYTE) + ); + assert_eq!( + ImportExecutionPolicy::Interactive.disk_io_limits(), + (32 * MEBIBYTE, 4 * MEBIBYTE) + ); + assert_eq!( + ImportExecutionPolicy::Daemon.disk_io_limits(), + (8 * MEBIBYTE, MEBIBYTE) + ); + } + + #[test] + fn progress_and_json_distinguish_fresh_from_recovery() { + assert_eq!( + import_work_progress_message(ImportWorkClass::Fresh, CaptureProvider::Pi), + ("indexing", "indexing new/changed pi history".to_owned()) + ); + assert_eq!( + import_work_progress_message(ImportWorkClass::Recovery, CaptureProvider::Pi), + ("repairing", "repairing prior pi history".to_owned()) + ); + let source = explicit_path_source(CaptureProvider::Pi, "/fixture/pi".into()); + assert_eq!( + import_work_progress_done(ImportWorkClass::Fresh, &source), + ("indexing", "Indexed new/changed Pi history.".to_owned()) + ); + assert_eq!( + import_work_progress_done(ImportWorkClass::Recovery, &source), + ("repairing", "Repaired prior Pi history.".to_owned()) + ); + + let totals = ImportTotals { + fresh_units_processed: 3, + recovery_units_processed: 2, + fresh_units_pending: 1, + recovery_units_pending: 4, + ..ImportTotals::default() + }; + let snapshot = import_totals_json(&totals); + assert_eq!(snapshot["fresh_units_processed"], 3); + assert_eq!(snapshot["recovery_units_processed"], 2); + assert_eq!(snapshot["fresh_units_pending"], 1); + assert_eq!(snapshot["fresh_units_pending_exact"], true); + assert_eq!(snapshot["recovery_units_pending"], 4); + assert_eq!(snapshot["recovery_units_pending_exact"], true); + } + + #[test] + fn large_backlog_reporting_is_a_bounded_lower_bound() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let root = "/fixture/large-backlog"; + let source = explicit_path_source(CaptureProvider::Pi, root.into()); + let files = (0..10_000) + .map(|index| SourceImportFile { + provider: CaptureProvider::Pi, + source_format: source.source_format.to_owned(), + source_root: root.to_owned(), + source_path: format!("{root}/{index:05}.jsonl"), + file_size_bytes: 1, + file_modified_at_ms: 1, + import_revision: 1, + observed_at_ms: 1, + metadata: json!({}), + }) + .collect::>(); + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Pi, root) + .unwrap(); + store + .upsert_source_import_files(generation, &files) + .unwrap(); + let plan = ImportPlan::build( + &store, + vec![PlannedImportSource { + source, + stats: SourceStats::default(), + preinventory: SourcePreinventory::SourceImportFiles { + files, + inventory_generation: generation, + }, + }], + ) + .unwrap(); + + assert_eq!(plan.fresh_units, usize::MAX); + assert_eq!( + plan.pending_counts(&store).unwrap(), + (IMPORT_PENDING_REPORT_LIMIT, 0) + ); + let snapshot = import_totals_json(&ImportTotals { + fresh_units_pending: IMPORT_PENDING_REPORT_LIMIT, + ..ImportTotals::default() + }); + assert_eq!(snapshot["fresh_units_pending"], IMPORT_PENDING_REPORT_LIMIT); + assert_eq!(snapshot["fresh_units_pending_exact"], false); + } + + #[test] + fn fresh_work_is_selected_before_a_global_failed_and_revision_backlog() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let backlog_root = "/fixture/backlog"; + let mut backlog = (0..100) + .map(|index| SourceImportFile { + provider: CaptureProvider::Pi, + source_format: "pi_session_jsonl".to_owned(), + source_root: backlog_root.to_owned(), + source_path: format!("{backlog_root}/{index:03}.jsonl"), + file_size_bytes: 128, + file_modified_at_ms: 1000 + index, + import_revision: 1, + observed_at_ms: 2000, + metadata: json!({}), + }) + .collect::>(); + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Pi, backlog_root) + .unwrap(); + store + .upsert_source_import_files(generation, &backlog) + .unwrap(); + for file in &backlog[..50] { + store + .record_source_import_file_result( + CaptureProvider::Pi, + SourceImportFileIndexUpdate { + source_root: backlog_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: 3000, + }, + CatalogIndexedStatus::Failed, + Some("retry"), + ) + .unwrap(); + } + for file in &backlog[50..] { + store + .record_source_import_file_result( + CaptureProvider::Pi, + SourceImportFileIndexUpdate { + source_root: backlog_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: 3000, + }, + CatalogIndexedStatus::Indexed, + None, + ) + .unwrap(); + } + for file in &mut backlog[50..] { + file.import_revision = 2; + } + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Pi, backlog_root) + .unwrap(); + store + .upsert_source_import_files(generation, &backlog) + .unwrap(); + + let fresh_root = "/fixture/fresh"; + let fresh = SourceImportFile { + provider: CaptureProvider::Pi, + source_format: "pi_session_jsonl".to_owned(), + source_root: fresh_root.to_owned(), + source_path: format!("{fresh_root}/new.jsonl"), + file_size_bytes: 64, + file_modified_at_ms: 5000, + import_revision: 1, + observed_at_ms: 5000, + metadata: json!({}), + }; + let fresh_generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Pi, fresh_root) + .unwrap(); + store + .upsert_source_import_files(fresh_generation, std::slice::from_ref(&fresh)) + .unwrap(); + + let plan = ImportPlan::build( + &store, + vec![ + PlannedImportSource { + source: explicit_path_source(CaptureProvider::Pi, backlog_root.into()), + stats: SourceStats::default(), + preinventory: SourcePreinventory::SourceImportFiles { + files: backlog, + inventory_generation: generation, + }, + }, + PlannedImportSource { + source: explicit_path_source(CaptureProvider::Pi, fresh_root.into()), + stats: SourceStats::default(), + preinventory: SourcePreinventory::SourceImportFiles { + files: vec![fresh], + inventory_generation: fresh_generation, + }, + }, + ], + ) + .unwrap(); + assert_eq!(plan.fresh_units, 1); + assert_eq!(plan.recovery_units, 100); + + let fresh_slice = plan + .select_slice(&store, ImportWorkClass::Fresh, plan.fresh_units) + .unwrap(); + assert_eq!(fresh_slice.units, 1); + assert_eq!(fresh_slice.sources[0].source_index, 1); + let recovery_slice = plan + .select_slice(&store, ImportWorkClass::Recovery, plan.recovery_units) + .unwrap(); + assert_eq!(recovery_slice.units, IMPORT_SLICE_MAX_UNITS); + assert_eq!(recovery_slice.sources[0].source_index, 0); + } + + #[test] + fn global_recovery_prefers_unattempted_work_from_a_later_source() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("work.sqlite"); + let store = Store::open(&db_path).unwrap(); + let (first, _, _) = recovery_source(&store, "/fixture/first", Some(100)); + let (later, _, _) = recovery_source(&store, "/fixture/later", Some(200)); + rusqlite::Connection::open(db_path) + .unwrap() + .execute( + "UPDATE source_import_files SET indexed_at_ms = NULL WHERE source_root = ?1", + ["/fixture/later"], + ) + .unwrap(); + let plan = ImportPlan::build(&store, vec![first, later]).unwrap(); + + let slice = plan + .select_slice(&store, ImportWorkClass::Recovery, 1) + .unwrap(); + assert_eq!(slice.units, 1); + assert_eq!(slice.sources[0].source_index, 1); + } + + #[test] + fn failed_recovery_source_rotates_behind_the_older_other_source() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (first, first_file, first_generation) = + recovery_source(&store, "/fixture/first", Some(100)); + let (later, _, _) = recovery_source(&store, "/fixture/later", Some(200)); + let plan = ImportPlan::build(&store, vec![first, later]).unwrap(); + + let first_slice = plan + .select_slice(&store, ImportWorkClass::Recovery, 1) + .unwrap(); + assert_eq!(first_slice.sources[0].source_index, 0); + store + .record_source_import_file_result( + CaptureProvider::Pi, + SourceImportFileIndexUpdate { + source_root: &first_file.source_root, + source_path: &first_file.source_path, + file_size_bytes: first_file.file_size_bytes, + file_modified_at_ms: first_file.file_modified_at_ms, + import_revision: first_file.import_revision, + inventory_generation: first_generation, + metadata: &first_file.metadata, + indexed_at_ms: 300, + }, + CatalogIndexedStatus::Failed, + Some("still failing"), + ) + .unwrap(); + + let second_slice = plan + .select_slice(&store, ImportWorkClass::Recovery, 1) + .unwrap(); + assert_eq!(second_slice.sources[0].source_index, 1); + } + + #[test] + fn one_execution_does_not_select_the_same_pending_unit_twice() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (source, _, _) = recovery_source(&store, "/fixture/deferred", None); + let plan = ImportPlan::build(&store, vec![source]).unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + + let first = plan + .select_slice_with_state(&store, ImportWorkClass::Recovery, 1, &state, None) + .unwrap(); + assert_eq!(first.units, 1); + first.sources[0].persist_attempt_started(&store).unwrap(); + state.record_source_attempt(&first.sources[0].work); + let second = plan + .select_slice_with_state(&store, ImportWorkClass::Recovery, 1, &state, None) + .unwrap(); + assert!(second.is_empty()); + } + + #[test] + fn post_import_cache_replaces_file_metadata_with_the_new_observation() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (source, file, _) = recovery_source(&store, "/fixture/reobserved", None); + let plan = ImportPlan::build(&store, vec![source]).unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + let selected = plan + .select_slice_with_state(&store, ImportWorkClass::Recovery, 1, &state, None) + .unwrap(); + let selected_work = &selected.sources[0].work; + + let mut reobserved = file; + reobserved.file_size_bytes = 128; + reobserved.file_modified_at_ms = 200; + state.record_source_outcome( + 0, + selected_work, + Some(SourcePreinventory::SourceImportFiles { + files: vec![reobserved.clone()], + inventory_generation: 99, + }), + ); + + let SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + } = plan.selected_preinventory(&state, 0) + else { + panic!("manifest source must cache its post-import observation"); + }; + assert_eq!(inventory_generation, 99); + assert_eq!(files, vec![reobserved]); + } + + #[test] + fn rebased_execution_state_preserves_only_unaffected_source_observations() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (dirty_source, _, _) = recovery_source(&store, "/fixture/dirty", None); + let (stable_source, _, stable_generation) = + recovery_source(&store, "/fixture/stable", None); + let old_plan = ImportPlan { + sources: vec![dirty_source.clone(), stable_source.clone()], + fresh_units: 0, + recovery_units: 0, + }; + let mut state = ImportExecutionState::for_plan(&old_plan); + state.observed_preinventories = vec![ + Some(dirty_source.preinventory.clone()), + Some(stable_source.preinventory.clone()), + ]; + state.attempted_work.insert("attempted-work".to_owned()); + + let new_plan = ImportPlan { + sources: vec![stable_source, dirty_source], + fresh_units: 0, + recovery_units: 0, + }; + let rebased = state.rebase_for_plan( + &old_plan, + &new_plan, + &BTreeSet::from([PathBuf::from("/fixture/dirty")]), + ); + + assert_eq!( + rebased.observed_preinventories[0] + .as_ref() + .and_then(SourcePreinventory::inventory_generation), + Some(stable_generation) + ); + assert!(rebased.observed_preinventories[1].is_none()); + assert!(rebased.attempted_work.contains("attempted-work")); + } + + #[test] + fn selected_but_still_pending_work_is_not_completed_progress() { + let mut result = ImportExecutionResult::default(); + result.add_slice(1, 0, 1, false); + assert_eq!(result.selected_units, 1); + assert_eq!(result.completed_units, 0); + assert_eq!(result.deferred_units, 1); + assert!(!result.made_durable_progress()); + } + + #[test] + fn attempted_fresh_window_does_not_hide_later_work_in_the_same_source() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let root = "/fixture/fresh-window"; + let source = explicit_path_source(CaptureProvider::Pi, root.into()); + let files = (0..65) + .map(|index| SourceImportFile { + provider: CaptureProvider::Pi, + source_format: source.source_format.to_owned(), + source_root: root.to_owned(), + source_path: format!("{root}/{index:03}.jsonl"), + file_size_bytes: 1, + file_modified_at_ms: 1, + import_revision: 1, + observed_at_ms: 1, + metadata: json!({}), + }) + .collect::>(); + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Pi, root) + .unwrap(); + store + .upsert_source_import_files(generation, &files) + .unwrap(); + let plan = ImportPlan::build( + &store, + vec![PlannedImportSource { + source, + stats: SourceStats::default(), + preinventory: SourcePreinventory::SourceImportFiles { + files, + inventory_generation: generation, + }, + }], + ) + .unwrap(); + let mut state = ImportExecutionState::for_plan(&plan); + + let first = plan + .select_slice_with_state( + &store, + ImportWorkClass::Fresh, + IMPORT_SLICE_MAX_UNITS, + &state, + None, + ) + .unwrap(); + assert_eq!(first.units, IMPORT_SLICE_MAX_UNITS); + first.sources[0].persist_attempt_started(&store).unwrap(); + state.record_source_attempt(&first.sources[0].work); + let second = plan + .select_slice_with_state(&store, ImportWorkClass::Fresh, 1, &state, None) + .unwrap(); + assert_eq!(second.units, 1); + let SelectedImportWork::SourceFiles(work) = &second.sources[0].work else { + panic!("fresh manifest work must use source-file selection"); + }; + assert!(work[0].file.source_path.ends_with("/064.jsonl")); + } +} diff --git a/crates/ctx-cli/src/commands/import/scheduler/types.rs b/crates/ctx-cli/src/commands/import/scheduler/types.rs new file mode 100644 index 00000000..7a8ecdf0 --- /dev/null +++ b/crates/ctx-cli/src/commands/import/scheduler/types.rs @@ -0,0 +1,333 @@ +pub(crate) const IMPORT_SLICE_MAX_UNITS: usize = 64; +pub(crate) const IMPORT_SLICE_TARGET_BYTES: u64 = 8 * 1024 * 1024; +pub(crate) const IMPORT_PENDING_REPORT_LIMIT: usize = 256; +const MEBIBYTE: u64 = 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ImportExecutionPolicy { + Drain, + Interactive, + Daemon, +} + +impl ImportExecutionPolicy { + pub(crate) fn fresh_slice_limit(self) -> Option { + match self { + Self::Interactive | Self::Daemon => Some(1), + Self::Drain => None, + } + } + + pub(crate) fn recovery_slice_limit(self) -> Option { + match self { + Self::Drain => None, + Self::Interactive | Self::Daemon => Some(1), + } + } + + pub(crate) fn disk_io_pacer(self) -> ctx_history_capture::DiskIoPacer { + let (bytes_per_second, burst_bytes) = self.disk_io_limits(); + ctx_history_capture::DiskIoPacer::new(bytes_per_second, burst_bytes) + } + + pub(crate) const fn disk_io_limits(self) -> (u64, u64) { + match self { + Self::Drain => (64 * MEBIBYTE, 8 * MEBIBYTE), + Self::Interactive => (32 * MEBIBYTE, 4 * MEBIBYTE), + Self::Daemon => (8 * MEBIBYTE, MEBIBYTE), + } + } +} + +#[derive(Debug)] +pub(crate) struct ImportPlan { + pub(crate) sources: Vec, + pub(crate) fresh_units: usize, + pub(crate) recovery_units: usize, +} + +#[derive(Debug)] +pub(crate) struct ImportSlice { + pub(crate) sources: Vec, + pub(crate) retirements: Vec, + pub(crate) units: usize, + pub(crate) bytes: u64, +} + +pub(crate) struct ExecutableImportSlice { + pub(crate) slice: ImportSlice, + pub(crate) bulk_guard: EventSearchBulkGuard, + pub(crate) validation_failures: Vec, +} + +#[derive(Debug)] +pub(crate) struct SourceValidationFailure { + pub(crate) source_index: usize, + pub(crate) stats: SourceStats, + pub(crate) error: anyhow::Error, +} + +#[derive(Debug, Default)] +pub(crate) struct ImportExecutionState { + observed_preinventories: Vec>, + attempted_work: BTreeSet, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct ImportExecutionResult { + pub(crate) selected_units: usize, + pub(crate) completed_units: usize, + pub(crate) deferred_units: usize, + durable_progress: bool, +} + +impl ImportExecutionResult { + pub(crate) fn add_slice( + &mut self, + selected_units: usize, + completed_units: usize, + deferred_units: usize, + maintenance_progress: bool, + ) { + self.selected_units = self.selected_units.saturating_add(selected_units); + self.completed_units = self.completed_units.saturating_add(completed_units); + self.deferred_units = self.deferred_units.saturating_add(deferred_units); + self.durable_progress |= completed_units > 0 || maintenance_progress; + } + + pub(crate) fn made_durable_progress(&self) -> bool { + self.durable_progress + } +} + +impl ImportSlice { + fn empty() -> Self { + Self { + sources: Vec::new(), + retirements: Vec::new(), + units: 0, + bytes: 0, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.units == 0 + } +} + +#[derive(Debug)] +pub(crate) struct SelectedImportSource { + pub(crate) source_index: usize, + pub(crate) preinventory: SourcePreinventory, + pub(crate) work: SelectedImportWork, + pub(crate) stats: SourceStats, + attempts_persisted: bool, +} + +#[derive(Debug)] +pub(crate) enum SelectedImportWork { + Catalog(Vec), + SourceFiles(Vec), +} + +impl SelectedImportWork { + pub(crate) fn unit_count(&self) -> usize { + match self { + Self::Catalog(work) => work.len(), + Self::SourceFiles(work) => work.len(), + } + } +} + +impl ImportExecutionState { + pub(crate) fn for_plan(plan: &ImportPlan) -> Self { + Self { + observed_preinventories: vec![None; plan.sources.len()], + attempted_work: BTreeSet::new(), + } + } + + pub(crate) fn begin_new_pass(&mut self) { + self.attempted_work.clear(); + } + + pub(crate) fn rebase_for_plan( + &self, + old_plan: &ImportPlan, + new_plan: &ImportPlan, + invalidated_paths: &BTreeSet, + ) -> Self { + let mut rebased = Self::for_plan(new_plan); + rebased.attempted_work = self.attempted_work.clone(); + for (new_index, new_source) in new_plan.sources.iter().enumerate() { + if invalidated_paths.contains(&new_source.source.path) { + continue; + } + let Some(old_index) = old_plan + .sources + .iter() + .position(|old_source| old_source.source == new_source.source) + else { + continue; + }; + rebased.observed_preinventories[new_index] = self + .observed_preinventories + .get(old_index) + .cloned() + .flatten(); + } + rebased + } + + pub(crate) fn record_retirement_attempt( + &mut self, + work: &ProviderFilePublicationRetirementWork, + ) { + self.attempted_work.insert(retirement_work_identity(work)); + } + + pub(crate) fn record_source_attempt(&mut self, work: &SelectedImportWork) { + match work { + SelectedImportWork::Catalog(work) => { + self.attempted_work + .extend(work.iter().map(catalog_work_identity)); + } + SelectedImportWork::SourceFiles(work) => { + self.attempted_work + .extend(work.iter().map(source_file_work_identity)); + } + } + } + + pub(crate) fn record_source_outcome( + &mut self, + source_index: usize, + work: &SelectedImportWork, + post_import_preinventory: Option, + ) { + // Native source-file batches reobserve the complete source state after import. + // Cache only that post-import observation: advancing the old generation alone + // would pair fresh generation metadata with stale file metadata. + if !matches!(work, SelectedImportWork::SourceFiles(_)) { + return; + } + self.observed_preinventories[source_index] = post_import_preinventory; + } + + fn has_attempted(&self, identity: &str) -> bool { + self.attempted_work.contains(identity) + } + + fn mark_validation_skip(&mut self, identity: String) { + self.attempted_work.insert(identity); + } +} + +impl SelectedImportSource { + pub(crate) fn persist_attempt_started(&self, store: &Store) -> Result { + if self.attempts_persisted { + return Ok(self.work.unit_count()); + } + match &self.work { + SelectedImportWork::Catalog(selected) => { + let SourcePreinventory::CodexSessionCatalog { + inventory_generation, + .. + } = &self.preinventory + else { + return Ok(0); + }; + selected.iter().try_fold(0usize, |persisted, work| { + Ok(persisted.saturating_add(persist_catalog_attempt_started( + store, + work, + *inventory_generation, + )?)) + }) + } + SelectedImportWork::SourceFiles(selected) => { + let inventory_generation = match &self.preinventory { + SourcePreinventory::SourceRoot { + inventory_generation, + .. + } + | SourcePreinventory::SourceImportFiles { + inventory_generation, + .. + } => *inventory_generation, + SourcePreinventory::CodexSessionCatalog { .. } | SourcePreinventory::None => { + return Ok(0) + } + }; + selected.iter().try_fold(0usize, |persisted, work| { + Ok( + persisted.saturating_add(persist_source_file_attempt_started( + store, + work, + inventory_generation, + )?), + ) + }) + } + } + } +} + +fn persist_catalog_attempt_started( + store: &Store, + work: &CatalogImportWork, + inventory_generation: u64, +) -> Result { + let state = store.catalog_source_index_state( + work.session.provider, + &work.session.source_root, + &work.session.source_path, + )?; + store + .record_observed_catalog_source_import_result( + work.session.provider, + ctx_history_store::CatalogSourceIndexUpdate { + source_root: &work.session.source_root, + source_path: &work.session.source_path, + file_size_bytes: work.session.file_size_bytes, + file_modified_at_ms: work.session.file_modified_at_ms, + import_revision: work.session.import_revision, + inventory_generation, + file_sha256: state + .as_ref() + .and_then(|state| state.last_imported_file_sha256.as_deref()), + event_count: state + .as_ref() + .and_then(|state| state.last_imported_event_count), + indexed_at_ms: ctx_history_core::utc_now().timestamp_millis(), + }, + &work.session.metadata, + CatalogIndexedStatus::Pending, + None, + ) + .map_err(Into::into) +} + +fn persist_source_file_attempt_started( + store: &Store, + work: &SourceImportFileWork, + inventory_generation: u64, +) -> Result { + store + .record_source_import_file_result( + work.file.provider, + ctx_history_store::SourceImportFileIndexUpdate { + source_root: &work.file.source_root, + source_path: &work.file.source_path, + file_size_bytes: work.file.file_size_bytes, + file_modified_at_ms: work.file.file_modified_at_ms, + import_revision: work.file.import_revision, + inventory_generation, + metadata: &work.file.metadata, + indexed_at_ms: ctx_history_core::utc_now().timestamp_millis(), + }, + CatalogIndexedStatus::Pending, + None, + ) + .map_err(Into::into) +} diff --git a/crates/ctx-cli/src/commands/import/state.rs b/crates/ctx-cli/src/commands/import/state.rs index bd593cf0..f2feaec5 100644 --- a/crates/ctx-cli/src/commands/import/state.rs +++ b/crates/ctx-cli/src/commands/import/state.rs @@ -1,5 +1,92 @@ +const PENDING_REASON_REPAIR_BATCH_ROWS: usize = 512; + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct ImportMaintenanceProgress { + pub(crate) processed_rows: usize, + pub(crate) complete: bool, +} + +pub(crate) fn repair_import_maintenance( + store: &Store, + policy: ImportExecutionPolicy, +) -> Result { + let mut aggregate = ImportMaintenanceProgress::default(); + loop { + let progress = store.repair_import_pending_reasons(PENDING_REASON_REPAIR_BATCH_ROWS)?; + let mut processed_rows = progress.processed_rows; + let bulk_complete = if progress.complete + && !store.has_pending_provider_file_publications()? + && !store.event_search_bulk_maintenance_outcome()?.is_complete() + { + processed_rows = processed_rows.saturating_add(1); + store.advance_event_search_bulk_maintenance()?.is_complete() + } else { + true + }; + aggregate.processed_rows = aggregate.processed_rows.saturating_add(processed_rows); + aggregate.complete = progress.complete && bulk_complete; + if aggregate.complete || policy != ImportExecutionPolicy::Drain { + return Ok(aggregate); + } + if processed_rows == 0 { + return Err(anyhow::Error::new(CaptureError::SystemInvariant( + "import maintenance made no progress", + ))); + } + } +} + +pub(crate) fn provider_publication_blocks_attempt(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + matches!( + cause.downcast_ref::(), + Some(StoreError::ProviderFileReplacementBusy { .. }) + ) + }) +} + +pub(crate) fn import_work_progress_message( + class: ImportWorkClass, + provider: CaptureProvider, +) -> (&'static str, String) { + match class { + ImportWorkClass::Fresh => ( + "indexing", + format!("indexing new/changed {} history", provider.as_str()), + ), + ImportWorkClass::Recovery => ( + "repairing", + format!("repairing prior {} history", provider.as_str()), + ), + } +} + +pub(crate) fn import_work_progress_done( + class: ImportWorkClass, + source: &SourceInfo, +) -> (&'static str, String) { + match class { + ImportWorkClass::Fresh => ( + "indexing", + format!( + "Indexed new/changed {} history.", + source_provider_label(source) + ), + ), + ImportWorkClass::Recovery => ( + "repairing", + format!("Repaired prior {} history.", source_provider_label(source)), + ), + } +} + #[derive(Debug, Clone, Default)] pub(crate) struct ImportTotals { + pub(crate) durable_progress: bool, + pub(crate) fresh_units_processed: usize, + pub(crate) recovery_units_processed: usize, + pub(crate) fresh_units_pending: usize, + pub(crate) recovery_units_pending: usize, pub(crate) source_files: usize, pub(crate) source_bytes: u64, pub(crate) imported_sources: usize, @@ -98,6 +185,140 @@ impl ImportTotals { } } +#[derive(Debug, Default)] +pub(crate) struct NativeSourceReports { + sources: BTreeMap, +} + +#[derive(Debug, Default)] +struct NativeSourceReport { + summary: ProviderImportSummary, + stats: SourceStats, + failure: Option, + reportable: bool, +} + +#[derive(Debug)] +struct NativeSourceFailure { + error: String, + failure_type: ImportFailureType, + rejected: bool, +} + +impl NativeSourceReports { + pub(crate) fn record_outcome( + &mut self, + source_index: usize, + summary: &ProviderImportSummary, + completed_stats: SourceStats, + no_op_stats: Option, + ) { + let report = self.sources.entry(source_index).or_default(); + report.summary.merge_from(summary.clone()); + report.stats.files = report.stats.files.saturating_add(completed_stats.files); + report.stats.bytes = report.stats.bytes.saturating_add(completed_stats.bytes); + report.stats.change_token = report.stats.change_token.or(completed_stats.change_token); + let reportable_no_op = no_op_stats.is_some(); + if report.stats.files == 0 { + if let Some(stats) = no_op_stats { + report.stats = stats; + } + } + report.reportable |= completed_stats.files > 0 + || reportable_no_op + || summary != &ProviderImportSummary::default(); + } + + pub(crate) fn record_failure( + &mut self, + source_index: usize, + stats: SourceStats, + error: &anyhow::Error, + ) { + let report = self.sources.entry(source_index).or_default(); + report.stats.files = report.stats.files.saturating_add(stats.files); + report.stats.bytes = report.stats.bytes.saturating_add(stats.bytes); + report.stats.change_token = report.stats.change_token.or(stats.change_token); + let rejected_summary = rejected_source_summary(error); + if let Some(summary) = rejected_summary.as_ref() { + report.summary.merge_from(summary.clone()); + } + report.failure.get_or_insert_with(|| NativeSourceFailure { + error: error_summary(error), + failure_type: import_failure_type(error), + rejected: rejected_summary.is_some(), + }); + report.reportable = true; + } + + pub(crate) fn apply_totals(&self, totals: &mut ImportTotals) { + for report in self.sources.values().filter(|report| report.reportable) { + let rejected_without_content = report.summary.failed > 0 + && !provider_summary_has_imported_content(&report.summary); + let failed_without_content = + report.failure.is_some() && !provider_summary_has_imported_content(&report.summary); + if rejected_without_content || failed_without_content { + if report.summary.failed > 0 { + totals.add_rejected_source(&report.summary, &report.stats); + } else { + totals.add_source_failure(&report.stats); + } + continue; + } + + totals.add(&report.summary, &report.stats); + if report + .failure + .as_ref() + .is_some_and(|failure| !failure.rejected) + { + totals.failed_sources = totals.failed_sources.saturating_add(1); + } + } + } + + fn append_json(self, plan: &ImportPlan, imported_sources: &mut Vec) { + for (source_index, report) in self + .sources + .into_iter() + .filter(|(_, report)| report.reportable) + { + let source = &plan.sources[source_index].source; + let rejected_without_content = report.summary.failed > 0 + && !provider_summary_has_imported_content(&report.summary); + let failed_without_content = + report.failure.is_some() && !provider_summary_has_imported_content(&report.summary); + if rejected_without_content || failed_without_content { + let failure = report.failure.unwrap_or_else(|| NativeSourceFailure { + error: format!( + "provider import reported {} failure(s)", + report.summary.failed + ), + failure_type: ImportFailureType::RecordRejection, + rejected: true, + }); + imported_sources.push(source_failure_json(&ImportSourceFailure { + source: source.clone(), + stats: report.stats, + error: failure.error, + failure_type: failure.failure_type, + rejected_summary: (report.summary.failed > 0).then_some(report.summary), + })); + continue; + } + + let mut value = source_import_json(source, &report.stats, &report.summary); + if let Some(failure) = report.failure.filter(|failure| !failure.rejected) { + value["status"] = json!("completed_with_source_failure"); + value["failure_scope"] = json!("source"); + value["failure_type"] = json!(failure.failure_type.as_str()); + value["error"] = json!(source_error_reason(source, &failure.error)); + } + imported_sources.push(value); + } + } +} + pub(crate) fn provider_summary_has_imported_content(summary: &ProviderImportSummary) -> bool { summary.has_accepted_content() } @@ -271,6 +492,28 @@ pub(crate) struct SourceStats { pub(crate) change_token: Option<[u8; 32]>, } +pub(crate) fn failed_inventory_pending_counts( + store: &Store, + failures: &[ImportSourceFailure], +) -> Result<(usize, usize)> { + let mut fresh = 0usize; + let mut recovery = 0usize; + for failure in failures { + let Some(source_root) = failure.source.path.to_str() else { + continue; + }; + let counts = + bounded_unplanned_root_work_counts(store, failure.source.provider, source_root)?; + fresh = capped_pending_add(fresh, counts.0); + recovery = capped_pending_add(recovery, counts.1); + } + Ok((fresh, recovery)) +} + +pub(crate) fn capped_pending_add(left: usize, right: usize) -> usize { + left.saturating_add(right).min(IMPORT_PENDING_REPORT_LIMIT) +} + #[derive(Debug, Clone)] pub(crate) struct PlannedImportSource { pub(crate) source: SourceInfo, diff --git a/crates/ctx-cli/src/commands/index.rs b/crates/ctx-cli/src/commands/index.rs index a155886b..df5f53f8 100644 --- a/crates/ctx-cli/src/commands/index.rs +++ b/crates/ctx-cli/src/commands/index.rs @@ -153,7 +153,7 @@ fn run_index_wait(args: IndexWaitArgs, data_root: &Path, quiet: bool) -> Result< } } -fn index_status_snapshot(data_root: &Path) -> Result { +pub(crate) fn index_status_snapshot(data_root: &Path) -> Result { let db_path = database_path(data_root.to_path_buf()); let initialized = db_path.exists(); let config_path = data_root.join(CONFIG_FILE); @@ -166,6 +166,7 @@ fn index_status_snapshot(data_root: &Path) -> Result { pending_inventory_units, failed_inventory_units, stale_inventory_units, + pending_provider_publication_retirements, semantic, daemon, ) = if initialized { @@ -173,12 +174,16 @@ fn index_status_snapshot(data_root: &Path) -> Result { let indexed_counts = store.indexed_history_counts()?; let catalog_counts = store.catalog_session_counts()?; let source_import_file_counts = store.source_import_file_counts()?; + let pending_provider_publication_retirements = + store.provider_file_publication_retirement_work_count()?; let inventory_units = catalog_counts .total - .saturating_add(source_import_file_counts.total); + .saturating_add(source_import_file_counts.total) + .saturating_add(pending_provider_publication_retirements); let pending_inventory_units = catalog_counts .pending - .saturating_add(source_import_file_counts.pending); + .saturating_add(source_import_file_counts.pending) + .saturating_add(pending_provider_publication_retirements); let failed_inventory_units = catalog_counts .failed .saturating_add(source_import_file_counts.failed); @@ -195,6 +200,7 @@ fn index_status_snapshot(data_root: &Path) -> Result { pending_inventory_units, failed_inventory_units, stale_inventory_units, + pending_provider_publication_retirements, semantic_worker_report_configured_json(&config, &semantic_report), daemon, ) @@ -209,6 +215,7 @@ fn index_status_snapshot(data_root: &Path) -> Result { 0, 0, 0, + 0, semantic_worker_report_configured_json(&config, &semantic_report), daemon, ) @@ -234,6 +241,7 @@ fn index_status_snapshot(data_root: &Path) -> Result { "pending_inventory_units": pending_inventory_units, "failed_inventory_units": failed_inventory_units, "stale_inventory_units": stale_inventory_units, + "pending_provider_publication_retirements": pending_provider_publication_retirements, }, "semantic": semantic, "daemon": daemon, @@ -278,6 +286,13 @@ fn print_index_status_human(status: &Value) { "lexical_pending_units: {}", usize_at(status, &["lexical", "pending_inventory_units"]) ); + println!( + "lexical_pending_provider_publication_retirements: {}", + usize_at( + status, + &["lexical", "pending_provider_publication_retirements"] + ) + ); println!("semantic_status: {}", semantic_job_status(status)); println!( "semantic_embedded_items: {}", diff --git a/crates/ctx-cli/src/commands/search.rs b/crates/ctx-cli/src/commands/search.rs index ad7b4d58..d6ef8432 100644 --- a/crates/ctx-cli/src/commands/search.rs +++ b/crates/ctx-cli/src/commands/search.rs @@ -1,28 +1,36 @@ use std::{ + collections::BTreeSet, fs, path::{Path, PathBuf}, - sync::{Arc, Mutex}, - thread, - time::{Duration as StdDuration, Instant}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::{Duration as StdDuration, Instant, SystemTime}, }; use anyhow::{anyhow, Context, Result}; use clap::ValueEnum; +use notify::{Config as NotifyConfig, RecommendedWatcher, RecursiveMode, Watcher}; use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; use ctx_history_capture::{ - discover_provider_sources_for_provider, CaptureError, ProviderImportSummary, + discover_provider_sources_for_provider, DiskIoPacer, ProviderImportSummary, ProviderSourceStatus, }; use ctx_history_core::database_path; -use ctx_history_store::Store; +use ctx_history_store::{ImportWorkClass, ProviderFilePublicationInventoryOwner, Store}; use crate::analytics::AnalyticsProperties; use crate::commands::import::{ - error_summary, import_error_scope, import_history_source_plugin, - import_one_source_for_search_refresh, import_totals_json, inventory_import_sources, - one_line_error, rejected_source_summary, should_parallelize_import, ImportFailureScope, - ImportSourceOutcome, ImportTotals, SourceStats, + error_summary, failed_inventory_pending_counts, import_error_scope, + import_history_source_plugin, import_selected_source, import_totals_json, + import_work_progress_done, import_work_progress_message, inventory_import_sources, + one_line_error, publication_recovery_maintenance_warning, + recover_provider_file_publication_retirement, rejected_source_summary, + repair_import_maintenance, ExecutableImportSlice, ImportExecutionPolicy, ImportFailureScope, + ImportPlan, ImportSourceFailure, ImportTotals, SourceStats, }; use crate::commands::setup::{ indexed_history_item_count, insert_db_size_bucket, insert_store_analytics_counts, @@ -31,7 +39,7 @@ use crate::history_source_plugins::{ discover_history_source_plugins, HistorySourcePluginRefresh, HistorySourcePluginSource, }; use crate::output::{compact_json, print_json}; -use crate::progress::{ProgressArg, ProgressReporter, SourceProgressSnapshot}; +use crate::progress::{ProgressArg, ProgressReporter}; use crate::provider_args::ProviderArg; use crate::provider_sources::{discovered_sources, home_dir, SourceInfo}; use crate::search_filters::{ @@ -45,746 +53,7 @@ use crate::store_util::open_existing_store_read_only; use crate::transcript::shell_quote_arg; use crate::{analytics, config, semantic, SearchArgs, SearchBackendArg, WAL_TRUNCATE_MIN_BYTES}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] -pub(crate) enum RefreshArg { - Background, - Off, - Wait, -} - -impl RefreshArg { - pub(crate) fn as_str(self) -> &'static str { - match self { - Self::Background => "background", - Self::Off => "off", - Self::Wait => "wait", - } - } -} -#[derive(Debug, Clone)] -pub(crate) struct SearchRefreshReport { - mode: RefreshArg, - status: &'static str, - source_count: usize, - totals: ImportTotals, - error: Option, -} - -impl SearchRefreshReport { - pub(crate) fn skipped(mode: RefreshArg, status: &'static str) -> Self { - Self { - mode, - status, - source_count: 0, - totals: ImportTotals::default(), - error: None, - } - } - - fn completed(mode: RefreshArg, source_count: usize, totals: ImportTotals) -> Self { - Self { - mode, - status: "completed", - source_count, - totals, - error: None, - } - } - - fn failed(mode: RefreshArg, source_count: usize, error: String) -> Self { - Self { - mode, - status: "failed", - source_count, - totals: ImportTotals::default(), - error: Some(error), - } - } - - pub(crate) fn to_json(&self) -> Value { - compact_json(json!({ - "mode": self.mode.as_str(), - "status": self.status, - "source_count": self.source_count, - "totals": import_totals_json(&self.totals), - "error": self.error, - })) - } -} - -pub(crate) fn run_search( - args: SearchArgs, - data_root: PathBuf, - analytics_properties: &mut AnalyticsProperties, - config: &config::AppConfig, -) -> Result<()> { - if !search_has_intent(SearchIntentInput { - query: args.query.as_deref(), - terms: &args.term, - file: args.file.as_deref(), - }) { - return Err(missing_search_intent_error()); - } - - let db_path = database_path(data_root.clone()); - let had_existing_store = db_path.exists(); - let indexed_content_before_search = if had_existing_store { - existing_store_indexed_content(&db_path) - } else { - Some(false) - }; - analytics::insert_bool( - analytics_properties, - "had_existing_store_before_search", - had_existing_store, - ); - analytics::insert_bool( - analytics_properties, - "indexed_content_before_search_known", - indexed_content_before_search.is_some(), - ); - analytics::insert_bool( - analytics_properties, - "had_indexed_content_before_search", - indexed_content_before_search.unwrap_or(false), - ); - let refresh_started = Instant::now(); - let refresh = refresh_before_search(&args, &data_root, config.daemon.enabled)?; - analytics::insert_duration( - analytics_properties, - "refresh_duration", - refresh_started.elapsed(), - ); - analytics::insert_str( - analytics_properties, - "search_refresh_mode", - refresh.mode.as_str(), - ); - analytics::insert_str( - analytics_properties, - "search_refresh_status", - refresh.status, - ); - analytics::insert_count_bucket( - analytics_properties, - "search_refresh_source_count_bucket", - refresh.source_count as u64, - ); - let backend_override = args.backend; - let requested_backend = resolve_search_backend(backend_override, config)?; - let semantic_enabled = config.semantic_search_enabled(); - if args.refresh == RefreshArg::Background - && semantic_enabled - && semantic::semantic_query_service_supported() - && matches!( - requested_backend, - SearchBackendArg::Semantic | SearchBackendArg::Hybrid - ) - { - semantic::maybe_autostart_daemon_for_search(&data_root, config); - semantic::wait_for_daemon_query_service(&data_root, StdDuration::from_secs(3)); - } - insert_db_size_bucket(analytics_properties, &db_path); - if refresh.status == "failed" && args.refresh == RefreshArg::Background && !had_existing_store { - return Err(anyhow!( - "search refresh failed and no existing ctx index is available; run `ctx import` first or retry with `--refresh wait`: {}", - refresh.error.as_deref().unwrap_or("unknown refresh error") - )); - } - let store = if args.refresh == RefreshArg::Off - || refresh.status == "failed" - || refresh.status == "completed" - || had_existing_store - { - open_existing_store_read_only(&db_path, "ctx search")? - } else { - Store::open(&db_path)? - }; - analytics::insert_bool( - analytics_properties, - "store_created_by_search", - !had_existing_store && db_path.exists(), - ); - insert_store_analytics_counts(analytics_properties, &store)?; - analytics::insert_bool( - analytics_properties, - "has_indexed_content_after_search", - indexed_history_item_count(&store)? > 0, - ); - let source_identity = SourceIdentityFilterArgs::from(&args); - let query = args.query.unwrap_or_default(); - let query_term_count = query - .split_whitespace() - .filter(|term| !term.trim().is_empty()) - .count() - .saturating_add( - args.term - .iter() - .filter(|term| !term.trim().is_empty()) - .count(), - ); - analytics::insert_text_length_bucket( - analytics_properties, - "query_length_bucket", - query.chars().count(), - ); - analytics::insert_count_bucket( - analytics_properties, - "query_term_count_bucket", - query_term_count as u64, - ); - let event_results = args.events || args.session.is_some(); - let options = ctx_history_search::PacketOptions { - limit: args.limit, - filters: search_filters( - SearchFilterInput { - session: args.session, - provider: args.provider, - source_identity, - workspace: args.workspace.clone(), - since: args.since.clone(), - primary_only: args.primary_only, - include_subagents: args.include_subagents, - event_type: args.event_type.clone(), - file: args.file.clone(), - include_current_session: args.include_current_session, - }, - Some(&store), - )?, - result_mode: if event_results { - ctx_history_search::SearchResultMode::Events - } else { - ctx_history_search::SearchResultMode::Sessions - }, - ..ctx_history_search::PacketOptions::default() - }; - let uses_composed_terms = args.term.iter().any(|term| !term.trim().is_empty()); - let query_started = Instant::now(); - let (packet, retrieval) = search_packet_with_backend( - &store, - &data_root, - &query, - &args.term, - &options, - requested_backend, - semantic_enabled, - args.semantic_weight, - args.refresh, - !args.json, - )?; - analytics::insert_duration( - analytics_properties, - "query_duration", - query_started.elapsed(), - ); - analytics::insert_str( - analytics_properties, - "search_backend_requested", - requested_backend.as_str(), - ); - analytics::insert_str( - analytics_properties, - "search_backend_effective", - retrieval.effective_mode().as_str(), - ); - let result_count = packet.results.len(); - let citation_count = packet - .results - .iter() - .map(|result| result.citations.len()) - .sum::(); - analytics::insert_count_bucket( - analytics_properties, - "result_count_bucket", - result_count as u64, - ); - analytics::insert_count_bucket( - analytics_properties, - "citation_count_bucket", - citation_count as u64, - ); - analytics::insert_bool(analytics_properties, "zero_result", result_count == 0); - let render_started = Instant::now(); - if args.json { - let suggested_next_query = (!uses_composed_terms).then_some(query.as_str()); - print_json(SearchDto::packet( - &store, - &packet, - &refresh, - &retrieval, - suggested_next_query, - ))?; - } else { - if refresh.status == "failed" && args.refresh == RefreshArg::Background { - if let Some(error) = &refresh.error { - eprintln!( - "warning: search refresh failed; serving existing index; use --refresh wait to fail instead: {error}" - ); - } - } - if packet.results.is_empty() { - if let Some(file) = args - .file - .as_deref() - .filter(|_| query.trim().is_empty() && !uses_composed_terms) - { - println!("no indexed events touched {}", file.display()); - let indexed_items = indexed_history_item_count(&store)?; - if indexed_items == 0 { - println!("next: ctx import --all"); - } else { - println!( - "next: ctx search {}", - shell_quote_arg(&file.display().to_string()) - ); - } - } else { - println!( - "no results for {}", - search_no_results_target(&query, &args.term) - ); - let indexed_items = indexed_history_item_count(&store)?; - if indexed_items == 0 { - println!("next: ctx import --all"); - } else { - println!("next: try broader terms with ctx search --term \"\""); - } - } - } - let suggested_next_query = (!uses_composed_terms).then_some(query.as_str()); - for (index, result) in packet.results.iter().enumerate() { - if args.verbose { - print_search_result_verbose(result, suggested_next_query); - } else { - print_search_result_compact(index + 1, result); - } - } - } - analytics::insert_duration( - analytics_properties, - "render_duration", - render_started.elapsed(), - ); - Ok(()) -} - -pub(crate) fn resolve_search_backend( - backend: Option, - config: &config::AppConfig, -) -> Result { - let semantic_enabled = config.semantic_search_enabled(); - match backend { - Some(SearchBackendArg::Semantic) if !semantic_enabled => Err(anyhow!( - "semantic search is disabled. Set [search] semantic = true in ctx config to enable the local semantic preview" - )), - Some(SearchBackendArg::Semantic) if !semantic::semantic_query_service_supported() => Err( - anyhow!( - "local semantic search is not supported on this platform yet. Set [search] semantic = false or use --backend lexical" - ), - ), - Some(SearchBackendArg::Semantic) if !config.daemon.enabled => Err(anyhow!( - "local semantic search requires the ctx daemon. Set [daemon] enabled = true, set [search] semantic = false, or use --backend lexical" - )), - value - if semantic_enabled - && semantic::semantic_query_service_supported() - && !config.daemon.enabled - && !matches!(value, Some(SearchBackendArg::Lexical)) => - { - Err(anyhow!( - "local semantic search requires the ctx daemon. Set [daemon] enabled = true, set [search] semantic = false, or use --backend lexical" - )) - } - Some(value) => Ok(value), - None if semantic_enabled => Ok(SearchBackendArg::Hybrid), - None => Ok(SearchBackendArg::Lexical), - } -} - -fn existing_store_indexed_content(db_path: &Path) -> Option { - open_existing_store_read_only(db_path, "ctx search analytics preflight") - .and_then(|store| indexed_history_item_count(&store)) - .ok() - .map(|indexed_items| indexed_items > 0) -} - -pub(crate) fn refresh_before_search( - args: &SearchArgs, - data_root: &Path, - daemon_enabled: bool, -) -> Result { - if args.refresh == RefreshArg::Off { - return Ok(SearchRefreshReport::skipped(RefreshArg::Off, "skipped")); - } - if args.refresh == RefreshArg::Background && daemon_enabled { - return Ok(SearchRefreshReport::skipped( - RefreshArg::Background, - "daemon_background", - )); - } - let source_identity = normalize_source_identity_filters(SourceIdentityFilterArgs::from(args))?; - if !source_identity.is_empty() - && args - .provider - .is_some_and(|provider| !matches!(provider, ProviderArg::Custom)) - { - return Err(anyhow!( - "custom history source filters can only be combined with --provider custom" - )); - } - let sources = if source_identity.is_empty() { - search_refresh_sources(args.provider) - } else { - Vec::new() - }; - let plugin_sources = - match search_refresh_plugin_sources(data_root, args.provider, &source_identity) { - Ok(sources) => sources, - Err(err) if args.refresh == RefreshArg::Background => { - return Ok(SearchRefreshReport::failed( - RefreshArg::Background, - sources.len(), - error_summary(&err), - )); - } - Err(err) => return Err(err.context("search refresh failed")), - }; - if sources.is_empty() && plugin_sources.is_empty() { - if args.refresh == RefreshArg::Wait { - return Err(anyhow!( - "wait search refresh found no supported discovered native provider or enabled auto history-source plugin sources; rerun the search with --refresh off to use the existing index" - )); - } - return Ok(SearchRefreshReport::skipped(args.refresh, "no_sources")); - } - let source_count = sources.len().saturating_add(plugin_sources.len()); - match refresh_sources_for_search(data_root, sources, plugin_sources, args.refresh, args.json) { - Ok(totals) => Ok(SearchRefreshReport::completed( - args.refresh, - source_count, - totals, - )), - Err(err) if args.refresh == RefreshArg::Background => Ok(SearchRefreshReport::failed( - RefreshArg::Background, - source_count, - error_summary(&err), - )), - Err(err) => Err(err.context("search refresh failed")), - } -} - -pub(crate) fn search_refresh_sources(provider: Option) -> Vec { - let Some(home) = home_dir() else { - return Vec::new(); - }; - let mut sources = if let Some(provider) = provider { - discover_provider_sources_for_provider(&home, provider.capture_provider()) - } else { - discovered_sources() - }; - sources - .drain(..) - .filter(|source| { - source.exists - && source.import_support.is_auto_importable() - && source.status == ProviderSourceStatus::Available - && source.source_format != "codex_history_jsonl" - }) - .collect() -} - -pub(crate) fn search_refresh_plugin_sources( - data_root: &Path, - provider: Option, - source_identity: &SourceIdentityFilters, -) -> Result> { - if !matches!(provider, None | Some(ProviderArg::Custom)) { - return Ok(Vec::new()); - } - Ok(discover_history_source_plugins(data_root, &[])? - .into_iter() - .filter(|source| { - source.enabled - && source.refresh == HistorySourcePluginRefresh::Auto - && source_identity.matches_plugin_source(source) - }) - .collect()) -} - -pub(crate) fn refresh_sources_for_search( - data_root: &Path, - sources: Vec, - plugin_sources: Vec, - refresh: RefreshArg, - json_output: bool, -) -> Result { - fs::create_dir_all(data_root)?; - config::write_default_config(data_root)?; - let db_path = database_path(data_root.to_path_buf()); - let store = Store::open(&db_path)?; - let had_indexed_content = store.indexed_history_item_count()? > 0; - let inventory = inventory_import_sources(&store, sources, false)?; - let planned_sources = inventory.sources; - let inventory_failures = inventory.failures; - let planned_total_bytes = inventory.totals.source_bytes; - drop(store); - if planned_sources.is_empty() && inventory_failures.is_empty() && plugin_sources.is_empty() { - return Ok(ImportTotals::default()); - } - - let progress_arg = match refresh { - RefreshArg::Wait if json_output => ProgressArg::Json, - RefreshArg::Wait => ProgressArg::Auto, - RefreshArg::Background | RefreshArg::Off => ProgressArg::None, - }; - let progress = ProgressReporter::new( - progress_arg, - json_output, - "search-refresh", - planned_total_bytes, - ); - let mut totals = ImportTotals::default(); - let mut first_refresh_failure = None::; - for failure in inventory_failures { - first_refresh_failure.get_or_insert_with(|| failure.error.clone()); - totals.add_source_failure(&failure.stats); - progress.warning(format!( - "skipped {} during inventory: {}", - failure.source.provider.as_str(), - one_line_error(&failure.error) - )); - } - if should_parallelize_import(&planned_sources) { - let source_states = Arc::new(Mutex::new( - planned_sources - .iter() - .map(|plan| SourceProgressSnapshot { - completed_bytes: 0, - total_bytes: plan.stats.bytes, - }) - .collect::>(), - )); - let handles = planned_sources - .into_iter() - .enumerate() - .map(|(index, plan)| { - let db_path = db_path.clone(); - let progress_callback = progress.parallel_codex_import_callback( - &plan.source, - index, - Arc::clone(&source_states), - ); - thread::spawn(move || -> Result { - let mut store = Store::open(&db_path)?; - let summary = import_one_source_for_search_refresh( - &mut store, - &plan.source, - progress_callback, - &plan.preinventory, - )?; - Ok(ImportSourceOutcome { - index, - source: plan.source, - stats: plan.stats, - summary, - }) - }) - }) - .collect::>(); - - let mut outcomes = Vec::with_capacity(handles.len()); - for handle in handles { - let result = handle - .join() - .map_err(|_| CaptureError::WorkerPanicked("provider import"))?; - match result { - Ok(outcome) => outcomes.push(outcome), - Err(err) - if refresh == RefreshArg::Background - && import_error_scope(&err) == ImportFailureScope::Source => - { - first_refresh_failure.get_or_insert_with(|| error_summary(&err)); - add_refresh_source_failure(&mut totals, &SourceStats::default(), &err); - } - Err(err) => return Err(err), - } - } - outcomes.sort_by_key(|outcome| outcome.index); - for outcome in outcomes { - warn_on_rejected_records( - &progress, - json_output, - outcome.source.provider.as_str(), - &outcome.summary, - ); - totals.add(&outcome.summary, &outcome.stats); - } - } else { - let mut store = Store::open(&db_path)?; - let mut completed_source_bytes = 0u64; - for plan in planned_sources { - progress.message( - "refreshing", - format!("importing {}", plan.source.provider.as_str()), - ); - let source_progress = - progress.codex_import_callback(&plan.source, completed_source_bytes); - completed_source_bytes = completed_source_bytes.saturating_add(plan.stats.bytes); - let import_result = import_one_source_for_search_refresh( - &mut store, - &plan.source, - source_progress, - &plan.preinventory, - ); - match import_result { - Ok(summary) => { - warn_on_rejected_records( - &progress, - json_output, - plan.source.provider.as_str(), - &summary, - ); - totals.add(&summary, &plan.stats); - progress.done( - "refreshing", - format!("refreshed {}", plan.source.provider.as_str()), - completed_source_bytes, - ); - } - Err(err) - if refresh == RefreshArg::Background - && import_error_scope(&err) == ImportFailureScope::Source => - { - let error = error_summary(&err); - first_refresh_failure.get_or_insert_with(|| error.clone()); - add_refresh_source_failure(&mut totals, &plan.stats, &err); - progress.done( - "refreshing", - format!( - "skipped {}: {}", - plan.source.provider.as_str(), - one_line_error(&error) - ), - completed_source_bytes, - ); - } - Err(err) => return Err(err), - } - } - } - - if !plugin_sources.is_empty() { - let mut store = Store::open(&db_path)?; - for plugin_source in plugin_sources { - progress.message( - "refreshing", - format!("running history source plugin {}", plugin_source.label()), - ); - let import_result = - import_history_source_plugin(&mut store, &plugin_source, data_root, false) - .with_context(|| { - format!("refresh history source plugin {}", plugin_source.label()) - }); - match import_result { - Ok((summary, stats)) => { - warn_on_rejected_records( - &progress, - json_output, - &plugin_source.label(), - &summary, - ); - totals.add(&summary, &stats); - progress.done( - "refreshing", - format!("refreshed history source plugin {}", plugin_source.label()), - 0, - ); - } - Err(err) - if refresh == RefreshArg::Background - && import_error_scope(&err) == ImportFailureScope::Source => - { - let error = error_summary(&err); - first_refresh_failure.get_or_insert_with(|| error.clone()); - add_refresh_source_failure(&mut totals, &SourceStats::default(), &err); - progress.done( - "refreshing", - format!( - "skipped history source plugin {}: {}", - plugin_source.label(), - one_line_error(&error) - ), - 0, - ); - } - Err(err) => return Err(err), - } - } - } - - let all_sources_failed = totals.imported_sources == 0 && totals.failed_sources > 0; - let all_rejected_without_prior_index = !had_indexed_content - && totals.imported_sessions == 0 - && totals.imported_events == 0 - && totals.failed > 0; - if refresh == RefreshArg::Background && (all_sources_failed || all_rejected_without_prior_index) - { - let detail = first_refresh_failure - .map(|error| format!("; first failure: {error}")) - .or_else(|| { - (totals.failed > 0).then(|| { - format!( - "; background refresh imported no content and reported {} failure(s)", - totals.failed - ) - }) - }) - .unwrap_or_default(); - return Err(anyhow!("all search refresh sources failed{detail}")); - } - - Store::open(&db_path)?.checkpoint_wal_truncate_if_larger_than(WAL_TRUNCATE_MIN_BYTES)?; - Ok(totals) -} - -fn add_refresh_source_failure( - totals: &mut ImportTotals, - stats: &SourceStats, - error: &anyhow::Error, -) { - if let Some(summary) = rejected_source_summary(error) { - totals.add_rejected_source(&summary, stats); - } else { - totals.add_source_failure(stats); - } -} - -fn warn_on_rejected_records( - progress: &ProgressReporter, - json_output: bool, - source: &str, - summary: &ProviderImportSummary, -) { - if summary.failed == 0 { - return; - } - let first_failure = summary - .failures - .first() - .map(|failure| { - format!( - "; first failure at line {}: {}", - failure.line, failure.error - ) - }) - .unwrap_or_default(); - let warning = format!( - "refreshed {source} with {} rejected history record(s){first_failure}", - summary.failed - ); - if progress.is_enabled() { - progress.warning(warning); - } else if !json_output { - eprintln!("warning: {warning}"); - } -} +include!("search/query.rs"); +include!("search/refresh_plan.rs"); +include!("search/refresh_execute.rs"); +include!("search/freshness_tests.rs"); diff --git a/crates/ctx-cli/src/commands/search/freshness_tests.rs b/crates/ctx-cli/src/commands/search/freshness_tests.rs new file mode 100644 index 00000000..2818c3f2 --- /dev/null +++ b/crates/ctx-cli/src/commands/search/freshness_tests.rs @@ -0,0 +1,1497 @@ +#[cfg(test)] +mod freshness_tests { + use super::*; + use crate::commands::import::{run_import_internal, ImportRunOptions}; + use crate::provider_args::NativeProviderArg; + use crate::provider_sources::explicit_path_source; + use crate::ImportArgs; + use ctx_history_core::{canonical_provider_material_source_format, utc_now, CaptureProvider}; + use ctx_history_store::{ + CatalogIndexedStatus, ProviderFileInventoryObservation, ProviderFilePublicationKind, + SourceImportFileIndexUpdate, + }; + + fn write_pi_source(root: &Path, count: usize, label: &str) -> SourceInfo { + fs::create_dir_all(root).unwrap(); + for index in 0..count { + fs::write( + root.join(format!("{index:03}.jsonl")), + format!( + "{}\n{}\n", + json!({ + "type": "session", + "id": format!("{label}-{index}"), + "timestamp": "2026-07-14T12:00:00Z" + }), + json!({ + "type": "message", + "id": format!("{label}-message-{index}"), + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": format!("{label} {index}")} + }) + ), + ) + .unwrap(); + } + explicit_path_source(CaptureProvider::Pi, root.to_path_buf()) + } + + fn seed_failed_pi_backlog(data_root: &Path, count: usize) -> SourceInfo { + fs::create_dir_all(data_root).unwrap(); + let source = write_pi_source(&data_root.join("pi-backlog"), count, "recovery"); + let store = Store::open(database_path(data_root.to_path_buf())).unwrap(); + let inventory = inventory_import_sources(&store, vec![source.clone()], false).unwrap(); + let (files, inventory_generation) = match &inventory.sources[0].preinventory { + crate::commands::import::SourcePreinventory::SourceImportFiles { + files, + inventory_generation, + } => (files, *inventory_generation), + other => panic!("unexpected Pi inventory: {other:?}"), + }; + for file in files { + assert_eq!( + store + .record_source_import_file_result( + file.provider, + SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation, + metadata: &file.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + CatalogIndexedStatus::Failed, + Some("deterministic recovery fixture"), + ) + .unwrap(), + 1 + ); + } + source + } + + fn refresh( + data_root: &Path, + sources: Vec, + policy: ImportExecutionPolicy, + ) -> ImportTotals { + refresh_sources_for_search( + data_root, + sources, + Vec::new(), + RefreshArg::Background, + false, + policy, + ) + .unwrap() + } + + fn refresh_with_runtime( + data_root: &Path, + runtime: &mut SearchRefreshRuntime, + sources: Vec, + ) -> ImportTotals { + refresh_sources_for_search_with_runtime( + data_root, + sources, + Vec::new(), + RefreshArg::Background, + false, + ImportExecutionPolicy::Daemon, + runtime, + ) + .unwrap() + } + + fn cached_inventory_generation(runtime: &SearchRefreshRuntime, source: &SourceInfo) -> u64 { + runtime + .cached_work + .as_ref() + .unwrap() + .plan + .sources + .iter() + .find(|planned| planned.source.path == source.path) + .and_then(|planned| planned.preinventory.inventory_generation()) + .unwrap() + } + + fn leave_unmutated_pi_publication(store: &Store, source: &SourceInfo) { + let inventory = inventory_import_sources(store, vec![source.clone()], false).unwrap(); + let inventory_generation = match &inventory.sources[0].preinventory { + crate::commands::import::SourcePreinventory::SourceImportFiles { + inventory_generation, + .. + } => *inventory_generation, + other => panic!("unexpected Pi inventory: {other:?}"), + }; + let source_root = source.path.to_str().unwrap(); + store + .schedule_source_import_explicit_rescan( + source.provider, + source_root, + inventory_generation, + ) + .unwrap(); + let file = store + .list_pending_source_import_files(source.provider, source_root) + .unwrap() + .into_iter() + .next() + .unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + ProviderFileInventoryObservation::SourceImport { + source_format: &file.source_format, + update: SourceImportFileIndexUpdate { + source_root: file.source_root.as_str(), + source_path: file.source_path.as_str(), + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation, + metadata: &file.metadata, + indexed_at_ms: utc_now().timestamp_millis(), + }, + }, + canonical_provider_material_source_format(file.provider, &file.source_format) + .unwrap(), + ProviderFilePublicationKind::Replacement, + utc_now().timestamp_millis(), + ) + .unwrap(); + drop(scope); + } + + fn stage_pi_recovery_publication(data_root: &Path, source: &SourceInfo) { + let baseline = refresh( + data_root, + vec![source.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(baseline.fresh_units_pending, 0, "{baseline:?}"); + + let store = Store::open(database_path(data_root.to_path_buf())).unwrap(); + let inventory = inventory_import_sources(&store, vec![source.clone()], false).unwrap(); + let inventory_generation = match &inventory.sources[0].preinventory { + crate::commands::import::SourcePreinventory::SourceImportFiles { + inventory_generation, + .. + } => *inventory_generation, + other => panic!("unexpected Pi inventory: {other:?}"), + }; + store + .schedule_source_import_explicit_rescan( + source.provider, + source.path.to_str().unwrap(), + inventory_generation, + ) + .unwrap(); + drop(store); + + let mut runtime = SearchRefreshRuntime::default(); + let staged = refresh_with_runtime(data_root, &mut runtime, vec![source.clone()]); + assert!(staged.recovery_units_pending > 0, "{staged:?}"); + let store = Store::open(database_path(data_root.to_path_buf())).unwrap(); + assert!(store + .effective_provider_file_publication_has_staged_completion() + .unwrap()); + } + + #[test] + fn daemon_cached_refresh_watcher_tracks_source_changes() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source = write_pi_source(&data_root.join("pi-watch"), 1, "watch"); + let mut runtime = SearchRefreshRuntime::default(); + + let initial = runtime.watcher_changes(std::slice::from_ref(&source)); + assert_eq!(initial, Some(SearchRefreshSourceChanges::default())); + runtime.force_source_change_for_test(&source.path); + let changed = runtime + .watcher_changes(std::slice::from_ref(&source)) + .unwrap(); + assert!(!changed.full_rebuild); + assert_eq!(changed.dirty_paths, BTreeSet::from([source.path])); + } + + #[test] + fn daemon_cached_refresh_watcher_errors_invalidate_cached_work() { + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + + note_search_refresh_source_event( + &changes, + &healthy, + &[], + Err(notify::Error::generic("deterministic watcher failure")), + ); + + assert!(changes.into_inner().unwrap().full_rebuild); + assert!(!healthy.load(Ordering::Acquire)); + } + + #[test] + fn daemon_cached_refresh_watcher_ignores_non_mutating_access() { + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + + note_search_refresh_source_event( + &changes, + &healthy, + &[], + Ok(notify::Event::new(notify::EventKind::Access( + notify::event::AccessKind::Read, + ))), + ); + note_search_refresh_source_event( + &changes, + &healthy, + &[], + Ok(notify::Event::new(notify::EventKind::Access( + notify::event::AccessKind::Open(notify::event::AccessMode::Any), + ))), + ); + assert_eq!( + changes.into_inner().unwrap(), + SearchRefreshSourceChanges::default() + ); + assert!(healthy.load(Ordering::Acquire)); + } + + #[test] + fn daemon_cached_refresh_watcher_scopes_mutations_to_matching_roots() { + let temp = tempfile::tempdir().unwrap(); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + fs::create_dir(&first).unwrap(); + fs::create_dir(&second).unwrap(); + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + let watches = search_refresh_watch_specs(&[first.clone(), second]); + + note_search_refresh_source_event( + &changes, + &healthy, + &watches, + Ok( + notify::Event::new(notify::EventKind::Modify(notify::event::ModifyKind::Any)) + .add_path(first.join("session.jsonl")), + ), + ); + + let changes = changes.into_inner().unwrap(); + assert!(!changes.full_rebuild); + assert_eq!(changes.dirty_paths, BTreeSet::from([first])); + assert!(healthy.load(Ordering::Acquire)); + } + + #[test] + fn daemon_cached_refresh_watcher_maps_sqlite_sidecars_to_the_database_source() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("history.sqlite"); + fs::write(&database, []).unwrap(); + let watches = search_refresh_watch_specs(std::slice::from_ref(&database)); + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + + note_search_refresh_source_event( + &changes, + &healthy, + &watches, + Ok( + notify::Event::new(notify::EventKind::Modify(notify::event::ModifyKind::Any)) + .add_path(temp.path().join("history.sqlite-wal")), + ), + ); + + let changes = changes.into_inner().unwrap(); + assert!(!changes.full_rebuild); + assert_eq!(changes.dirty_paths, BTreeSet::from([database])); + } + + #[test] + fn daemon_cached_refresh_watcher_ignores_unrelated_file_siblings() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("history.sqlite"); + fs::write(&database, []).unwrap(); + let watches = search_refresh_watch_specs(std::slice::from_ref(&database)); + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + + note_search_refresh_source_event( + &changes, + &healthy, + &watches, + Ok( + notify::Event::new(notify::EventKind::Modify(notify::event::ModifyKind::Any)) + .add_path(temp.path().join("unrelated.sqlite")), + ), + ); + + assert_eq!( + changes.into_inner().unwrap(), + SearchRefreshSourceChanges::default() + ); + assert!(healthy.load(Ordering::Acquire)); + } + + #[test] + fn daemon_cached_refresh_watcher_invalidates_coalesced_parent_events() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("history.sqlite"); + fs::write(&database, []).unwrap(); + let watches = search_refresh_watch_specs(std::slice::from_ref(&database)); + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + + note_search_refresh_source_event( + &changes, + &healthy, + &watches, + Ok( + notify::Event::new(notify::EventKind::Modify(notify::event::ModifyKind::Any)) + .add_path(temp.path().to_path_buf()), + ), + ); + + let changes = changes.into_inner().unwrap(); + assert!(changes.full_rebuild); + assert!(changes.dirty_paths.is_empty()); + assert!(healthy.load(Ordering::Acquire)); + } + + #[test] + fn daemon_cached_refresh_watcher_invalidates_out_of_scope_events() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("history.sqlite"); + fs::write(&database, []).unwrap(); + let watches = search_refresh_watch_specs(std::slice::from_ref(&database)); + let changes = Mutex::new(SearchRefreshSourceChanges::default()); + let healthy = AtomicBool::new(true); + + note_search_refresh_source_event( + &changes, + &healthy, + &watches, + Ok( + notify::Event::new(notify::EventKind::Modify(notify::event::ModifyKind::Any)) + .add_path(temp.path().join("other").join("history.sqlite")), + ), + ); + + let changes = changes.into_inner().unwrap(); + assert!(changes.full_rebuild); + assert!(changes.dirty_paths.is_empty()); + assert!(healthy.load(Ordering::Acquire)); + } + + #[test] + fn daemon_cached_refresh_watcher_invalidates_ambiguous_access() { + use notify::event::{AccessKind, AccessMode}; + + for access in [ + AccessKind::Any, + AccessKind::Other, + AccessKind::Close(AccessMode::Any), + AccessKind::Close(AccessMode::Write), + AccessKind::Close(AccessMode::Other), + ] { + assert!(!search_refresh_event_is_non_mutating_access( + notify::EventKind::Access(access) + )); + } + for access in [ + AccessKind::Read, + AccessKind::Open(AccessMode::Any), + AccessKind::Open(AccessMode::Read), + AccessKind::Open(AccessMode::Write), + AccessKind::Close(AccessMode::Read), + AccessKind::Close(AccessMode::Execute), + ] { + assert!(search_refresh_event_is_non_mutating_access( + notify::EventKind::Access(access) + )); + } + } + + #[test] + fn daemon_watcher_root_identity_changes_when_path_is_replaced() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("watched-root"); + fs::create_dir(&root).unwrap(); + let initial = watched_source_path_identities(std::slice::from_ref(&root)); + + fs::rename(&root, temp.path().join("old-root")).unwrap(); + fs::create_dir(&root).unwrap(); + let replaced = watched_source_path_identities(std::slice::from_ref(&root)); + + assert_ne!(initial, replaced); + } + + #[cfg(any(unix, windows))] + #[test] + fn daemon_watcher_root_identity_ignores_directory_content_changes() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("watched-root"); + fs::create_dir(&root).unwrap(); + let initial = watched_source_path_identities(std::slice::from_ref(&root)); + + fs::write(root.join("new-session.jsonl"), b"new session").unwrap(); + let changed = watched_source_path_identities(std::slice::from_ref(&root)); + + assert_eq!(initial, changed); + assert!(initial[0].stable_id.is_some()); + } + + #[test] + fn daemon_periodic_reinventory_uses_elapsed_time_not_backlog_state() { + assert!(!daemon_search_refresh_reinventory_due( + DAEMON_SEARCH_REFRESH_REINVENTORY_INTERVAL - StdDuration::from_secs(1), + false, + )); + assert!(daemon_search_refresh_reinventory_due( + DAEMON_SEARCH_REFRESH_REINVENTORY_INTERVAL, + false, + )); + assert!(!daemon_search_refresh_reinventory_due( + DAEMON_SEARCH_REFRESH_REINVENTORY_INTERVAL, + true, + )); + } + + #[test] + fn daemon_failed_watcher_retries_on_each_daemon_pass() { + assert!(daemon_search_refresh_watcher_retry_due(false)); + assert!(!daemon_search_refresh_watcher_retry_due(true)); + assert!(daemon_search_refresh_watcher_fallback_due(None)); + assert!(!daemon_search_refresh_watcher_fallback_due(Some( + Instant::now() + ))); + assert!(daemon_search_refresh_watcher_fallback_due(Some( + Instant::now() - DAEMON_SEARCH_REFRESH_WATCHER_FALLBACK_INTERVAL + ))); + } + + #[test] + fn daemon_scoped_refresh_preserves_unaffected_plan_and_full_sweep_age() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let dirty = write_pi_source(&data_root.join("pi-dirty"), 1, "dirty"); + let unchanged = write_pi_source(&data_root.join("pi-unchanged"), 1, "unchanged"); + let baseline = refresh( + &data_root, + vec![dirty.clone(), unchanged.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(baseline.fresh_units_pending, 0, "{baseline:?}"); + + let mut runtime = SearchRefreshRuntime::default(); + let cached = refresh_with_runtime( + &data_root, + &mut runtime, + vec![dirty.clone(), unchanged.clone()], + ); + assert_eq!(cached.fresh_units_pending, 0, "{cached:?}"); + let dirty_generation = cached_inventory_generation(&runtime, &dirty); + let unchanged_generation = cached_inventory_generation(&runtime, &unchanged); + runtime + .cached_work + .as_mut() + .unwrap() + .passes_since_reinventory = 17; + + write_pi_source(&dirty.path, 2, "dirtychanged"); + runtime.force_source_change_for_test(&dirty.path); + let mut refreshed = refresh_with_runtime( + &data_root, + &mut runtime, + vec![dirty.clone(), unchanged.clone()], + ); + + assert!(refreshed.durable_progress, "{refreshed:?}"); + assert_ne!( + cached_inventory_generation(&runtime, &dirty), + dirty_generation + ); + assert_eq!( + cached_inventory_generation(&runtime, &unchanged), + unchanged_generation + ); + assert_eq!( + runtime + .cached_work + .as_ref() + .unwrap() + .passes_since_reinventory, + 18 + ); + for _ in 0..16 { + if refreshed.fresh_units_pending == 0 { + break; + } + refreshed = refresh_with_runtime( + &data_root, + &mut runtime, + vec![dirty.clone(), unchanged.clone()], + ); + } + assert_eq!(refreshed.fresh_units_pending, 0, "{refreshed:?}"); + let store = Store::open(database_path(data_root.clone())).unwrap(); + assert!(!store + .search_event_hits("dirtychanged 1", 10) + .unwrap() + .is_empty()); + assert_eq!( + cached_inventory_generation(&runtime, &unchanged), + unchanged_generation + ); + } + + #[test] + fn daemon_elapsed_safety_sweep_runs_while_recovery_is_pending() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source = seed_failed_pi_backlog(&data_root, 65); + let mut runtime = SearchRefreshRuntime::default(); + let mut pending_without_publication = false; + for _ in 0..32 { + let totals = refresh_with_runtime(&data_root, &mut runtime, vec![source.clone()]); + let publication_pending = Store::open(database_path(data_root.clone())) + .unwrap() + .has_pending_provider_file_publications() + .unwrap(); + if totals.recovery_units_pending > 0 && !publication_pending { + pending_without_publication = true; + break; + } + } + assert!(pending_without_publication); + // A real next pass would immediately start the remaining recovery unit. + // Align only the cached owner so this call isolates the elapsed sweep. + runtime.cached_work.as_mut().unwrap().publication_owner = None; + let first_generation = cached_inventory_generation(&runtime, &source); + runtime.cached_work.as_mut().unwrap().last_reinventory_at = + Instant::now() - DAEMON_SEARCH_REFRESH_REINVENTORY_INTERVAL; + + let _ = refresh_with_runtime(&data_root, &mut runtime, vec![source.clone()]); + + assert_ne!( + cached_inventory_generation(&runtime, &source), + first_generation + ); + } + + #[test] + fn daemon_cached_refresh_handles_deferred_recovery_across_passes() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source = seed_failed_pi_backlog(&data_root, 1); + let mut runtime = SearchRefreshRuntime::default(); + let mut completed = 0usize; + let mut pending = usize::MAX; + + for _ in 0..8 { + let totals = refresh_with_runtime(&data_root, &mut runtime, vec![source.clone()]); + assert!(runtime.cached_work.is_some()); + completed = completed.saturating_add(totals.recovery_units_processed); + pending = totals.recovery_units_pending; + if pending == 0 { + break; + } + } + + assert_eq!(completed, 1); + assert_eq!(pending, 0); + let store = Store::open(database_path(data_root.clone())).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + let no_op = refresh_with_runtime(&data_root, &mut runtime, vec![source]); + assert_eq!(no_op.fresh_units_processed, 0); + assert_eq!(no_op.recovery_units_processed, 0); + assert!(runtime.cached_work.is_some()); + } + + #[test] + fn daemon_cached_refresh_rebuilds_for_a_new_global_publication_owner() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let owner = write_pi_source(&data_root.join("pi-explicit-owner"), 1, "owner"); + let cached = write_pi_source(&data_root.join("pi-cached"), 1, "cached"); + let imported = refresh( + &data_root, + vec![owner.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(imported.fresh_units_processed, 1); + + let mut runtime = SearchRefreshRuntime::default(); + for _ in 0..32 { + refresh_with_runtime(&data_root, &mut runtime, vec![cached.clone()]); + if !Store::open(database_path(data_root.clone())) + .unwrap() + .has_pending_provider_file_publications() + .unwrap() + { + break; + } + } + refresh_with_runtime(&data_root, &mut runtime, vec![cached.clone()]); + assert!(runtime + .cached_work + .as_ref() + .is_some_and(|work| work.publication_owner.is_none())); + + let store = Store::open(database_path(data_root.clone())).unwrap(); + leave_unmutated_pi_publication(&store, &owner); + assert!(store.has_pending_provider_file_publications().unwrap()); + + let mut completed = 0usize; + for _ in 0..32 { + let totals = refresh_with_runtime(&data_root, &mut runtime, vec![cached.clone()]); + completed = completed.saturating_add(totals.recovery_units_processed); + if !Store::open(database_path(data_root.clone())) + .unwrap() + .has_pending_provider_file_publications() + .unwrap() + { + break; + } + } + + assert_eq!(completed, 1); + assert!(!Store::open(database_path(data_root)) + .unwrap() + .has_pending_provider_file_publications() + .unwrap()); + } + + #[test] + fn pending_report_includes_publication_created_after_plan_inventory() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let owner = write_pi_source(&data_root.join("pi-late-owner"), 1, "late-owner"); + let imported = refresh( + &data_root, + vec![owner.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(imported.fresh_units_processed, 1); + + let store = Store::open(database_path(data_root)).unwrap(); + let stale_plan = ImportPlan::build(&store, Vec::new()).unwrap(); + leave_unmutated_pi_publication(&store, &owner); + + assert_eq!( + reported_pending_counts(&store, &stale_plan).unwrap(), + (0, 1) + ); + } + + #[test] + fn synthetic_publication_owner_counts_toward_all_source_failure() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let owner = write_pi_source(&data_root.join("pi-failing-owner"), 1, "failing-owner"); + let imported = refresh( + &data_root, + vec![owner.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(imported.fresh_units_processed, 1); + let store = Store::open(database_path(data_root.clone())).unwrap(); + leave_unmutated_pi_publication(&store, &owner); + drop(store); + fs::remove_dir_all(&owner.path).unwrap(); + fs::write(&owner.path, b"publication owner root became a file").unwrap(); + + let error = refresh_sources_for_search( + &data_root, + Vec::new(), + Vec::new(), + RefreshArg::Background, + false, + ImportExecutionPolicy::Daemon, + ) + .expect_err("the only synthetic source failed"); + + assert!( + format!("{error:#}").contains("all search refresh sources failed"), + "{error:#}" + ); + } + + #[test] + fn healthy_no_op_source_prevents_all_sources_failed() { + let totals = ImportTotals { + failed_sources: 1, + ..ImportTotals::default() + }; + + assert!(!all_refresh_sources_failed(2, &totals)); + assert!(all_refresh_sources_failed(1, &totals)); + } + + #[test] + fn partially_imported_source_does_not_count_as_all_sources_failed() { + let totals = ImportTotals { + imported_sources: 1, + failed_sources: 1, + ..ImportTotals::default() + }; + + assert!(!all_refresh_sources_failed(1, &totals)); + } + + #[test] + fn failed_background_refresh_requires_known_indexed_content_for_fallback() { + assert!(has_usable_search_fallback(Some(true))); + assert!(!has_usable_search_fallback(Some(false))); + assert!(!has_usable_search_fallback(None)); + } + + #[test] + fn incomplete_fresh_tail_does_not_starve_recovery_work() { + use std::io::Write as _; + + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let incomplete_root = data_root.join("pi-incomplete"); + let incomplete = write_pi_source(&incomplete_root, 1, "still-writing"); + let baseline = refresh( + &data_root, + vec![incomplete.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(baseline.fresh_units_pending, 0); + fs::OpenOptions::new() + .append(true) + .open(incomplete_root.join("000.jsonl")) + .unwrap() + .write_all(br#"{"type":"message","id":"partial""#) + .unwrap(); + let backlog = seed_failed_pi_backlog(&data_root, 1); + + let mut recovery_completed = 0usize; + let mut last = ImportTotals::default(); + for _ in 0..8 { + last = refresh( + &data_root, + vec![backlog.clone(), incomplete.clone()], + ImportExecutionPolicy::Interactive, + ); + recovery_completed = recovery_completed.saturating_add(last.recovery_units_processed); + if recovery_completed > 0 { + break; + } + } + + assert_eq!(last.fresh_units_processed, 0, "{last:?}"); + assert_eq!(last.fresh_units_pending, 1, "{last:?}"); + assert_eq!(recovery_completed, 1, "{last:?}"); + assert_eq!(last.recovery_units_pending, 0, "{last:?}"); + } + + #[test] + fn incomplete_file_does_not_block_a_later_complete_file_in_the_same_source() { + use std::io::Write as _; + + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source_root = data_root.join("pi-fair-source"); + let source = write_pi_source(&source_root, 1, "partial-first"); + let baseline = refresh( + &data_root, + vec![source.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(baseline.fresh_units_pending, 0, "{baseline:?}"); + fs::OpenOptions::new() + .append(true) + .open(source_root.join("000.jsonl")) + .unwrap() + .write_all(br#"{"type":"message","id":"partial""#) + .unwrap(); + fs::write( + source_root.join("001.jsonl"), + format!( + "{}\n{}\n", + json!({ + "type": "session", + "id": "complete-later", + "timestamp": "2026-07-14T12:00:00Z" + }), + json!({ + "type": "message", + "id": "complete-later-message", + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "complete later content"} + }) + ), + ) + .unwrap(); + + let totals = refresh(&data_root, vec![source], ImportExecutionPolicy::Drain); + + assert_eq!(totals.fresh_units_processed, 1, "{totals:?}"); + assert_eq!(totals.fresh_units_pending, 1, "{totals:?}"); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("complete later content")); + assert!(!store.has_pending_provider_file_publications().unwrap()); + } + + #[test] + fn growing_append_source_finishes_its_snapshot_then_imports_the_tail() { + use std::io::Write as _; + + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source_root = data_root.join("pi-growing"); + let source = write_pi_source(&source_root, 1, "growing"); + let baseline = refresh( + &data_root, + vec![source.clone()], + ImportExecutionPolicy::Drain, + ); + assert_eq!(baseline.fresh_units_pending, 0, "{baseline:?}"); + let store = Store::open(database_path(data_root.clone())).unwrap(); + leave_unmutated_pi_publication(&store, &source); + fs::OpenOptions::new() + .append(true) + .open(source_root.join("000.jsonl")) + .unwrap() + .write_all( + format!( + "{}\n", + json!({ + "type": "message", + "id": "growing-tail-message", + "timestamp": "2026-07-14T12:00:02Z", + "message": {"role": "assistant", "content": "growth after inventory"} + }) + ) + .as_bytes(), + ) + .unwrap(); + drop(store); + let mut runtime = SearchRefreshRuntime::default(); + + let mut pending = usize::MAX; + for _ in 0..8 { + let totals = refresh_with_runtime(&data_root, &mut runtime, vec![source.clone()]); + pending = totals + .fresh_units_pending + .saturating_add(totals.recovery_units_pending); + } + + assert_eq!(pending, 0); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("growth after inventory")); + } + + #[test] + fn drain_revisits_fresh_tail_created_by_recovery() { + use std::io::Write as _; + + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source_root = data_root.join("pi-recovery-tail"); + let source = write_pi_source(&source_root, 1, "recovery-tail"); + stage_pi_recovery_publication(&data_root, &source); + + fs::OpenOptions::new() + .append(true) + .open(source_root.join("000.jsonl")) + .unwrap() + .write_all( + format!( + "{}\n", + json!({ + "type": "message", + "id": "recovery-created-fresh-tail", + "timestamp": "2026-07-14T12:00:02Z", + "message": {"role": "assistant", "content": "fresh tail after recovery"} + }) + ) + .as_bytes(), + ) + .unwrap(); + + let totals = refresh(&data_root, vec![source], ImportExecutionPolicy::Drain); + assert_eq!(totals.fresh_units_pending, 0, "{totals:?}"); + assert_eq!(totals.recovery_units_pending, 0, "{totals:?}"); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("fresh tail after recovery")); + } + + #[test] + fn rewritten_growing_source_invalidates_staged_snapshot_and_converges() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source_root = data_root.join("pi-rewritten-growth"); + let source = write_pi_source(&source_root, 1, "stale-snapshot"); + let mut runtime = SearchRefreshRuntime::default(); + + let first = refresh_with_runtime(&data_root, &mut runtime, vec![source.clone()]); + assert!( + first.fresh_units_pending + first.recovery_units_pending > 0, + "{first:?}" + ); + let store = Store::open(database_path(data_root.clone())).unwrap(); + assert!(store + .effective_provider_file_publication_has_staged_completion() + .unwrap()); + drop(store); + + fs::write( + source_root.join("000.jsonl"), + format!( + "{}\n{}\n{}\n", + json!({ + "type": "session", + "id": "replacement-session-with-longer-identity", + "timestamp": "2026-07-14T12:00:00Z" + }), + json!({ + "type": "message", + "id": "replacement-user-message", + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "replacement growth oracle"} + }), + json!({ + "type": "message", + "id": "replacement-assistant-message", + "timestamp": "2026-07-14T12:00:02Z", + "message": {"role": "assistant", "content": "replacement tail is fully indexed"} + }) + ), + ) + .unwrap(); + + let totals = refresh(&data_root, vec![source], ImportExecutionPolicy::Drain); + assert_eq!(totals.fresh_units_pending, 0, "{totals:?}"); + assert_eq!(totals.recovery_units_pending, 0, "{totals:?}"); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + let archive = serde_json::to_string(&store.export_archive().unwrap()).unwrap(); + assert!(archive.contains("replacement growth oracle"), "{archive}"); + assert!( + archive.contains("replacement tail is fully indexed"), + "{archive}" + ); + assert!(!archive.contains("stale-snapshot 0"), "{archive}"); + } + + #[test] + fn same_size_rewrite_with_preserved_mtime_invalidates_staged_snapshot() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source_root = data_root.join("pi-rewritten-same-size"); + let source = write_pi_source(&source_root, 1, "stale-equal"); + let mut runtime = SearchRefreshRuntime::default(); + + let first = refresh_with_runtime(&data_root, &mut runtime, vec![source.clone()]); + assert!( + first.fresh_units_pending + first.recovery_units_pending > 0, + "{first:?}" + ); + let store = Store::open(database_path(data_root.clone())).unwrap(); + assert!(store + .effective_provider_file_publication_has_staged_completion() + .unwrap()); + drop(store); + + let path = source_root.join("000.jsonl"); + let original_modified = fs::metadata(&path).unwrap().modified().unwrap(); + let original = fs::read_to_string(&path).unwrap(); + let replacement = original.replace("stale-equal 0", "fresh-equal 0"); + assert_ne!(replacement, original); + assert_eq!(replacement.len(), original.len()); + fs::write(&path, replacement).unwrap(); + fs::File::open(&path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(original_modified)) + .unwrap(); + + let totals = refresh(&data_root, vec![source], ImportExecutionPolicy::Drain); + assert_eq!(totals.fresh_units_pending, 0, "{totals:?}"); + assert_eq!(totals.recovery_units_pending, 0, "{totals:?}"); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + let archive = serde_json::to_string(&store.export_archive().unwrap()).unwrap(); + assert!(archive.contains("fresh-equal 0"), "{archive}"); + assert!(!archive.contains("stale-equal 0"), "{archive}"); + } + + #[test] + fn search_refresh_resumes_same_generation_publication_that_wins_the_bulk_lock() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + fs::create_dir_all(&data_root).unwrap(); + let db_path = database_path(data_root.clone()); + let source = write_pi_source(&data_root.join("pi-race-resume"), 1, "bulk-resume"); + let mut lock_store = Store::open(&db_path).unwrap(); + let inventory = inventory_import_sources(&lock_store, vec![source.clone()], false).unwrap(); + let waiting_plan = ImportPlan::build(&lock_store, inventory.sources.clone()).unwrap(); + let winning_plan = ImportPlan::build(&lock_store, inventory.sources).unwrap(); + assert_eq!(waiting_plan.fresh_units, 1); + + let guard = lock_store.begin_event_search_bulk_mode().unwrap(); + let (waiting_tx, waiting_rx) = std::sync::mpsc::channel(); + let waiting_db_path = db_path.clone(); + let waiter = std::thread::spawn(move || { + let mut waiting_store = Store::open(waiting_db_path).unwrap(); + let progress = ProgressReporter::new(ProgressArg::None, false, "search-refresh", 0); + let mut totals = ImportTotals::default(); + let mut first_refresh_failure = None; + let mut imported_sources = BTreeSet::new(); + let mut failed_sources = BTreeSet::new(); + let mut execution_state = + crate::commands::import::ImportExecutionState::for_plan(&waiting_plan); + let result = execute_search_refresh_plan_class_with_pre_lock_hook( + &mut waiting_store, + &waiting_plan, + &mut execution_state, + ImportWorkClass::Fresh, + waiting_plan.fresh_units, + None, + &progress, + false, + false, + &mut totals, + &mut first_refresh_failure, + &mut imported_sources, + &mut failed_sources, + false, + || waiting_tx.send(()).unwrap(), + ); + (result, totals, first_refresh_failure, failed_sources) + }); + waiting_rx.recv().unwrap(); + + let winning_slice = winning_plan + .select_slice( + &lock_store, + ImportWorkClass::Fresh, + winning_plan.fresh_units, + ) + .unwrap(); + let selected = &winning_slice.sources[0]; + let source_plan = &winning_plan.sources[selected.source_index]; + let winner = import_selected_source( + &mut lock_store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + assert_eq!(winner.completed_units, 0); + assert_eq!(winner.deferred_units, 1); + assert!(winner.durable_progress); + lock_store.finish_event_search_bulk_mode(&guard).unwrap(); + drop(guard); + + let (result, totals, first_failure, failed_sources) = waiter.join().unwrap(); + let result = result.unwrap(); + assert_eq!(result.completed_units, 1); + assert_eq!(result.deferred_units, 0); + assert!(result.made_durable_progress()); + assert_eq!(totals.fresh_units_processed, 1); + assert!(first_failure.is_none()); + assert!(failed_sources.is_empty()); + + let completed = refresh(&data_root, vec![source], ImportExecutionPolicy::Drain); + assert_eq!( + completed + .fresh_units_processed + .saturating_add(completed.recovery_units_processed), + 0, + "{completed:?}" + ); + assert_eq!(completed.fresh_units_pending, 0, "{completed:?}"); + assert_eq!(completed.recovery_units_pending, 0, "{completed:?}"); + assert!(!lock_store.has_pending_provider_file_publications().unwrap()); + } + + #[test] + fn search_refresh_drops_superseded_completion_that_wins_the_bulk_lock() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + fs::create_dir_all(&data_root).unwrap(); + let db_path = database_path(data_root.clone()); + let source = write_pi_source(&data_root.join("pi-race"), 1, "bulk-winner"); + let mut lock_store = Store::open(&db_path).unwrap(); + let inventory = inventory_import_sources(&lock_store, vec![source.clone()], false).unwrap(); + let waiting_plan = ImportPlan::build(&lock_store, inventory.sources).unwrap(); + assert_eq!(waiting_plan.fresh_units, 1); + + let guard = lock_store.begin_event_search_bulk_mode().unwrap(); + let (waiting_tx, waiting_rx) = std::sync::mpsc::channel(); + let waiting_db_path = db_path.clone(); + let waiter = std::thread::spawn(move || { + let mut waiting_store = Store::open(waiting_db_path).unwrap(); + let progress = ProgressReporter::new(ProgressArg::None, false, "search-refresh", 0); + let mut totals = ImportTotals::default(); + let mut first_refresh_failure = None; + let mut imported_sources = BTreeSet::new(); + let mut failed_sources = BTreeSet::new(); + let mut execution_state = + crate::commands::import::ImportExecutionState::for_plan(&waiting_plan); + let result = execute_search_refresh_plan_class_with_pre_lock_hook( + &mut waiting_store, + &waiting_plan, + &mut execution_state, + ImportWorkClass::Fresh, + waiting_plan.fresh_units, + None, + &progress, + false, + false, + &mut totals, + &mut first_refresh_failure, + &mut imported_sources, + &mut failed_sources, + false, + || waiting_tx.send(()).unwrap(), + ); + ( + result, + totals, + first_refresh_failure, + imported_sources, + failed_sources, + ) + }); + waiting_rx.recv().unwrap(); + + let completion_inventory = + inventory_import_sources(&lock_store, vec![source], false).unwrap(); + let completion_plan = ImportPlan::build(&lock_store, completion_inventory.sources).unwrap(); + let completion_slice = completion_plan + .select_slice( + &lock_store, + ImportWorkClass::Fresh, + completion_plan.fresh_units, + ) + .unwrap(); + let selected = &completion_slice.sources[0]; + let source_plan = &completion_plan.sources[selected.source_index]; + let mut completion_finished = false; + for _ in 0..64 { + let completion = import_selected_source( + &mut lock_store, + &source_plan.source, + None, + &selected.preinventory, + &selected.work, + ) + .unwrap(); + if completion.outcome.completed_units == 1 { + completion_finished = true; + break; + } + assert_eq!(completion.outcome.deferred_units, 1); + assert!(completion.outcome.made_durable_progress()); + } + assert!(completion_finished, "winning import did not converge"); + lock_store.finish_event_search_bulk_mode(&guard).unwrap(); + drop(guard); + + let (result, totals, first_failure, imported_sources, failed_sources) = + waiter.join().unwrap(); + result.unwrap(); + assert_eq!(totals.fresh_units_processed, 0); + assert_eq!(totals.failed_sources, 0); + assert!(first_failure.is_none()); + assert!(imported_sources.is_empty()); + assert!(failed_sources.is_empty()); + } + + #[test] + fn background_refresh_isolates_source_removed_before_locked_revalidation() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + fs::create_dir_all(&data_root).unwrap(); + let source = write_pi_source(&data_root.join("pi-removed"), 1, "removed"); + let source_root = source.path.clone(); + let mut store = Store::open(database_path(data_root)).unwrap(); + let inventory = inventory_import_sources(&store, vec![source], false).unwrap(); + let plan = ImportPlan::build(&store, inventory.sources).unwrap(); + assert_eq!(plan.fresh_units, 1); + + let progress = ProgressReporter::new(ProgressArg::None, false, "search-refresh", 0); + let mut totals = ImportTotals::default(); + let mut first_refresh_failure = None; + let mut imported_sources = BTreeSet::new(); + let mut failed_sources = BTreeSet::new(); + let mut execution_state = crate::commands::import::ImportExecutionState::for_plan(&plan); + let result = execute_search_refresh_plan_class_with_pre_lock_hook( + &mut store, + &plan, + &mut execution_state, + ImportWorkClass::Fresh, + plan.fresh_units, + None, + &progress, + false, + true, + &mut totals, + &mut first_refresh_failure, + &mut imported_sources, + &mut failed_sources, + false, + || fs::remove_dir_all(&source_root).unwrap(), + ) + .unwrap(); + + assert_eq!(result.selected_units, 1); + assert_eq!(result.completed_units, 0); + assert_eq!(totals.failed_sources, 1); + assert!(first_refresh_failure.is_some()); + assert!(imported_sources.is_empty()); + assert_eq!(failed_sources, BTreeSet::from([0])); + let guard = store.begin_event_search_bulk_mode().unwrap(); + store.finish_event_search_bulk_mode(&guard).unwrap(); + } + + #[test] + fn repeated_foreground_refreshes_prioritize_fresh_work_and_converge() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let backlog = seed_failed_pi_backlog(&data_root, 3); + let fresh = write_pi_source(&data_root.join("pi-fresh"), 1, "fresh-first"); + + let first = refresh( + &data_root, + vec![backlog.clone(), fresh.clone()], + ImportExecutionPolicy::Interactive, + ); + assert_eq!(first.recovery_units_processed, 0); + assert_eq!(first.recovery_units_pending, 3); + + let mut fresh_completed = first.fresh_units_processed; + let mut recovery_completed = first.recovery_units_processed; + let mut pending = first + .fresh_units_pending + .saturating_add(first.recovery_units_pending); + for _ in 0..32 { + if pending == 0 { + break; + } + let outcome = refresh( + &data_root, + vec![backlog.clone(), fresh.clone()], + ImportExecutionPolicy::Interactive, + ); + fresh_completed = fresh_completed.saturating_add(outcome.fresh_units_processed); + recovery_completed = + recovery_completed.saturating_add(outcome.recovery_units_processed); + pending = outcome + .fresh_units_pending + .saturating_add(outcome.recovery_units_pending); + } + assert_eq!(fresh_completed.saturating_add(recovery_completed), 4); + assert_eq!(pending, 0); + } + + #[test] + fn interactive_background_refresh_leaves_a_large_fresh_backlog_bounded() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source = write_pi_source(&data_root.join("pi-fresh"), 130, "bounded-background"); + + let first = refresh(&data_root, vec![source], ImportExecutionPolicy::Interactive); + + assert!(first.fresh_units_pending > 0, "{first:?}"); + } + + #[test] + fn daemon_bounds_each_pass_and_eventually_converges() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let backlog = seed_failed_pi_backlog(&data_root, 3); + + let first = refresh( + &data_root, + vec![backlog.clone()], + ImportExecutionPolicy::Daemon, + ); + assert_eq!(first.recovery_units_processed, 0); + assert_eq!(first.recovery_units_pending, 3); + + let fresh = write_pi_source(&data_root.join("pi-daemon-fresh"), 1, "daemon-fresh"); + let mut fresh_completed = 0usize; + let mut recovery_completed = first.recovery_units_processed; + let mut pending = usize::MAX; + for _ in 0..32 { + let outcome = refresh( + &data_root, + vec![backlog.clone(), fresh.clone()], + ImportExecutionPolicy::Daemon, + ); + assert!( + outcome + .fresh_units_processed + .saturating_add(outcome.recovery_units_processed) + <= 2, + "one daemon pass must stay within one fresh and one recovery slice" + ); + fresh_completed = fresh_completed.saturating_add(outcome.fresh_units_processed); + recovery_completed = + recovery_completed.saturating_add(outcome.recovery_units_processed); + pending = outcome + .fresh_units_pending + .saturating_add(outcome.recovery_units_pending); + if pending == 0 { + break; + } + } + assert_eq!(fresh_completed.saturating_add(recovery_completed), 4); + assert_eq!(pending, 0); + } + + #[test] + fn drain_refresh_serializes_bounded_publications_across_sources() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let first = write_pi_source(&data_root.join("pi-drain-a"), 1, "drain-a"); + let second = write_pi_source(&data_root.join("pi-drain-b"), 1, "drain-b"); + + let totals = refresh( + &data_root, + vec![first, second], + ImportExecutionPolicy::Drain, + ); + + assert_eq!(totals.fresh_units_processed, 2); + assert_eq!(totals.fresh_units_pending, 0); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + } + + #[test] + fn drain_refresh_converges_multiple_files_within_one_source() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source = write_pi_source(&data_root.join("pi-drain-one-root"), 3, "drain-one-root"); + + let totals = refresh(&data_root, vec![source], ImportExecutionPolicy::Drain); + + assert_eq!(totals.fresh_units_processed, 3, "{totals:?}"); + assert_eq!(totals.fresh_units_pending, 0, "{totals:?}"); + assert_eq!(totals.recovery_units_pending, 0, "{totals:?}"); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + } + + #[test] + fn setup_operation_drains_all_recovery_work() { + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let backlog = seed_failed_pi_backlog(&data_root, 3); + let args = ImportArgs { + provider: Some(NativeProviderArg::Pi), + path: Some(backlog.path), + history_source: None, + history_source_manifest: Vec::new(), + reset_cursor: false, + format: None, + all: false, + resume: false, + no_daemon: true, + json: false, + progress: ProgressArg::None, + }; + let report = run_import_internal( + &args, + data_root, + &mut serde_json::Map::new(), + ImportRunOptions { + progress: ProgressArg::None, + json: false, + print_human: false, + allow_empty_sources: false, + include_history_source_plugins: false, + operation: "setup", + }, + ) + .unwrap(); + assert_eq!(report.totals.recovery_units_processed, 3); + assert_eq!(report.totals.recovery_units_pending, 0); + } + + #[test] + fn setup_revisits_fresh_tail_created_by_recovery() { + use std::io::Write as _; + + let temp = tempfile::tempdir().unwrap(); + let data_root = temp.path().join("data"); + let source_root = data_root.join("pi-setup-recovery-tail"); + let source = write_pi_source(&source_root, 1, "setup-recovery-tail"); + stage_pi_recovery_publication(&data_root, &source); + fs::OpenOptions::new() + .append(true) + .open(source_root.join("000.jsonl")) + .unwrap() + .write_all( + format!( + "{}\n", + json!({ + "type": "message", + "id": "setup-recovery-created-fresh-tail", + "timestamp": "2026-07-14T12:00:02Z", + "message": {"role": "assistant", "content": "setup fresh tail after recovery"} + }) + ) + .as_bytes(), + ) + .unwrap(); + + let args = ImportArgs { + provider: Some(NativeProviderArg::Pi), + path: Some(source.path), + history_source: None, + history_source_manifest: Vec::new(), + reset_cursor: false, + format: None, + all: false, + resume: false, + no_daemon: true, + json: false, + progress: ProgressArg::None, + }; + let report = run_import_internal( + &args, + data_root.clone(), + &mut serde_json::Map::new(), + ImportRunOptions { + progress: ProgressArg::None, + json: false, + print_human: false, + allow_empty_sources: false, + include_history_source_plugins: false, + operation: "setup", + }, + ) + .unwrap(); + assert_eq!(report.totals.fresh_units_pending, 0, "{report:?}"); + assert_eq!(report.totals.recovery_units_pending, 0, "{report:?}"); + let store = Store::open(database_path(data_root)).unwrap(); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("setup fresh tail after recovery")); + } +} diff --git a/crates/ctx-cli/src/commands/search/query.rs b/crates/ctx-cli/src/commands/search/query.rs new file mode 100644 index 00000000..a01de2de --- /dev/null +++ b/crates/ctx-cli/src/commands/search/query.rs @@ -0,0 +1,831 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub(crate) enum RefreshArg { + Background, + Off, + Wait, +} + +impl RefreshArg { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Background => "background", + Self::Off => "off", + Self::Wait => "wait", + } + } +} +#[derive(Debug, Clone)] +pub(crate) struct SearchRefreshReport { + mode: RefreshArg, + status: &'static str, + source_count: usize, + totals: ImportTotals, + error: Option, +} + +#[derive(Debug)] +struct SearchRefreshFailure { + error: anyhow::Error, + totals: ImportTotals, +} + +#[derive(Default)] +pub(crate) struct SearchRefreshRuntime { + cached_work: Option, + source_watcher: Option, + source_watcher_paths: Option>, + source_watcher_path_identities: Option>, + last_watcher_fallback_reinventory_at: Option, + disk_io_pacer: Option<(ImportExecutionPolicy, DiskIoPacer)>, +} + +struct SearchRefreshWork { + source_fingerprint: String, + publication_owner: Option, + passes_since_reinventory: usize, + last_reinventory_at: Instant, + plan: ImportPlan, + execution_state: crate::commands::import::ImportExecutionState, + inventory_failures: Vec, + failed_inventory_pending: (usize, usize), + inventoried_source_count: usize, + planned_total_bytes: u64, +} + +struct SearchRefreshSourceWatcher { + changes: Arc>, + healthy: Arc, + _watcher: RecommendedWatcher, +} + +#[derive(Debug, Clone)] +struct SearchRefreshWatch { + source_path: PathBuf, + match_path: PathBuf, + watch_path: PathBuf, + recursive: bool, +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct SearchRefreshSourceChanges { + full_rebuild: bool, + dirty_paths: BTreeSet, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct WatchedSourcePathIdentity { + exists: bool, + is_dir: bool, + stable_id: Option<(u64, u64)>, + fallback_len: Option, + fallback_modified_at: Option, +} + +struct SearchRefreshExecution { + totals: ImportTotals, +} + +const DAEMON_SEARCH_REFRESH_REINVENTORY_INTERVAL: StdDuration = StdDuration::from_secs(5 * 60); +const DAEMON_SEARCH_REFRESH_WATCHER_FALLBACK_INTERVAL: StdDuration = StdDuration::from_secs(30); + +impl SearchRefreshRuntime { + pub(crate) fn install_daemon_disk_io_pacing( + &mut self, + ) -> ctx_history_capture::DiskIoPacingGuard { + ctx_history_capture::install_disk_io_pacer( + self.disk_io_pacer(ImportExecutionPolicy::Daemon), + ) + } + + pub(crate) fn disk_io_pacer(&mut self, policy: ImportExecutionPolicy) -> DiskIoPacer { + if self + .disk_io_pacer + .as_ref() + .is_none_or(|(current, _)| *current != policy) + { + self.disk_io_pacer = Some((policy, policy.disk_io_pacer())); + } + self.disk_io_pacer + .as_ref() + .map(|(_, pacer)| pacer.clone()) + .expect("search refresh pacer must be initialized") + } + + fn invalidate(&mut self) { + self.cached_work = None; + } + + fn watcher_changes(&mut self, sources: &[SourceInfo]) -> Option { + let watched_paths = watched_source_paths(sources); + let watched_path_identities = watched_source_path_identities(&watched_paths); + let paths_changed = self + .source_watcher_paths + .as_ref() + .is_none_or(|current| *current != watched_paths); + let path_identities_changed = self + .source_watcher_path_identities + .as_ref() + .is_none_or(|current| *current != watched_path_identities); + let watcher_unhealthy = self + .source_watcher + .as_ref() + .is_some_and(|watcher| !watcher.is_healthy()); + if paths_changed || path_identities_changed || watcher_unhealthy { + self.cached_work = None; + self.source_watcher = None; + self.source_watcher_paths = Some(watched_paths.clone()); + self.source_watcher_path_identities = Some(watched_path_identities); + } + + let retry_due = daemon_search_refresh_watcher_retry_due(self.source_watcher.is_some()); + if paths_changed || path_identities_changed || watcher_unhealthy || retry_due { + if let Ok(watcher) = SearchRefreshSourceWatcher::new(watched_paths) { + // A recovered watcher cannot account for changes made while it + // was unavailable, so rebuild once before trusting it. + if !paths_changed { + self.cached_work = None; + } + self.source_watcher = Some(watcher); + } + } + if self.source_watcher.is_some() { + self.last_watcher_fallback_reinventory_at = None; + } else if daemon_search_refresh_watcher_fallback_due( + self.last_watcher_fallback_reinventory_at, + ) { + self.cached_work = None; + self.last_watcher_fallback_reinventory_at = Some(Instant::now()); + } + self.source_watcher + .as_ref() + .map(SearchRefreshSourceWatcher::take_changes) + } + + #[cfg(test)] + fn force_source_change_for_test(&self, path: &Path) { + if let Some(watcher) = &self.source_watcher { + watcher.mark_path_dirty(path); + } + } +} + +fn daemon_search_refresh_watcher_retry_due(watcher_available: bool) -> bool { + !watcher_available +} + +fn daemon_search_refresh_watcher_fallback_due(last_reinventory_at: Option) -> bool { + last_reinventory_at + .is_none_or(|last| last.elapsed() >= DAEMON_SEARCH_REFRESH_WATCHER_FALLBACK_INTERVAL) +} + +impl SearchRefreshSourceWatcher { + fn new(watched_paths: Vec) -> Result { + let changes = Arc::new(Mutex::new(SearchRefreshSourceChanges::default())); + let healthy = Arc::new(AtomicBool::new(true)); + let callback_changes = Arc::clone(&changes); + let callback_healthy = Arc::clone(&healthy); + let watches = search_refresh_watch_specs(&watched_paths); + let callback_watches = watches.clone(); + let mut watcher = RecommendedWatcher::new( + move |event: notify::Result| { + note_search_refresh_source_event( + &callback_changes, + &callback_healthy, + &callback_watches, + event, + ); + }, + NotifyConfig::default(), + ) + .context("create daemon search refresh watcher")?; + let mut watch_targets = Vec::<(PathBuf, bool)>::new(); + for watch in &watches { + if let Some((_, recursive)) = watch_targets + .iter_mut() + .find(|(path, _)| *path == watch.watch_path) + { + *recursive |= watch.recursive; + } else { + watch_targets.push((watch.watch_path.clone(), watch.recursive)); + } + } + for (watch_path, recursive) in watch_targets { + let mode = if recursive { + RecursiveMode::Recursive + } else { + RecursiveMode::NonRecursive + }; + watcher + .watch(&watch_path, mode) + .with_context(|| format!("watch search refresh source {}", watch_path.display()))?; + } + Ok(Self { + changes, + healthy, + _watcher: watcher, + }) + } + + fn take_changes(&self) -> SearchRefreshSourceChanges { + let mut changes = self + .changes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + std::mem::take(&mut *changes) + } + + #[cfg(test)] + fn mark_path_dirty(&self, path: &Path) { + self.changes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .dirty_paths + .insert(path.to_path_buf()); + } + + fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::Acquire) + } +} + +fn note_search_refresh_source_event( + changes: &Mutex, + healthy: &AtomicBool, + watches: &[SearchRefreshWatch], + event: notify::Result, +) { + let Ok(event) = event else { + healthy.store(false, Ordering::Release); + changes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .full_rebuild = true; + return; + }; + if event.need_rescan() { + changes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .full_rebuild = true; + return; + } + if search_refresh_event_is_non_mutating_access(event.kind) { + return; + } + + let mut changes = changes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut unclassified = event.paths.is_empty(); + for event_path in &event.paths { + let mut covered = false; + for watch in watches { + covered |= search_refresh_event_path_is_covered_by_watch(event_path, watch); + if search_refresh_event_path_matches_watch(event_path, watch) { + changes.dirty_paths.insert(watch.source_path.clone()); + } + } + unclassified |= !covered; + } + if unclassified { + changes.full_rebuild = true; + } +} + +fn search_refresh_watch_specs(paths: &[PathBuf]) -> Vec { + paths + .iter() + .map(|source_path| { + let is_dir = source_path.is_dir(); + let match_path = fs::canonicalize(source_path).unwrap_or_else(|_| source_path.clone()); + let watch_path = if is_dir { + match_path.clone() + } else { + source_path + .parent() + .map(|parent| fs::canonicalize(parent).unwrap_or_else(|_| parent.to_path_buf())) + .unwrap_or_else(|| match_path.clone()) + }; + SearchRefreshWatch { + source_path: source_path.clone(), + match_path, + watch_path, + recursive: is_dir, + } + }) + .collect() +} + +fn search_refresh_event_path_matches_watch(event_path: &Path, watch: &SearchRefreshWatch) -> bool { + if watch.recursive { + return event_path == watch.match_path || event_path.starts_with(&watch.match_path); + } + if event_path == watch.match_path { + return true; + } + let Some(event_parent) = event_path.parent() else { + return false; + }; + if event_parent != watch.watch_path { + return false; + } + let Some(source_name) = watch.match_path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + let Some(event_name) = event_path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + let companion_prefix = format!("{source_name}-"); + #[cfg(windows)] + { + event_name.eq_ignore_ascii_case(source_name) + || event_name + .to_ascii_lowercase() + .starts_with(&companion_prefix.to_ascii_lowercase()) + } + #[cfg(not(windows))] + { + event_name == source_name || event_name.starts_with(&companion_prefix) + } +} + +fn search_refresh_event_path_is_covered_by_watch( + event_path: &Path, + watch: &SearchRefreshWatch, +) -> bool { + if watch.recursive { + return event_path == watch.watch_path || event_path.starts_with(&watch.watch_path); + } + event_path.parent() == Some(watch.watch_path.as_path()) +} + +fn search_refresh_event_is_non_mutating_access(kind: notify::EventKind) -> bool { + use notify::event::{AccessKind, AccessMode}; + + matches!( + kind, + notify::EventKind::Access( + AccessKind::Read + | AccessKind::Open(_) + | AccessKind::Close(AccessMode::Read | AccessMode::Execute) + ) + ) +} + +fn watched_source_paths(sources: &[SourceInfo]) -> Vec { + let mut unique = BTreeSet::new(); + for source in sources { + unique.insert(source.path.clone()); + } + unique.into_iter().collect() +} + +fn watched_source_path_identities(paths: &[PathBuf]) -> Vec { + paths + .iter() + .map(|path| match fs::metadata(path) { + Ok(metadata) => { + let stable_id = watched_source_path_stable_id(path, &metadata); + WatchedSourcePathIdentity { + exists: true, + is_dir: metadata.is_dir(), + stable_id, + fallback_len: stable_id.is_none().then_some(metadata.len()), + fallback_modified_at: stable_id + .is_none() + .then(|| metadata.modified().ok()) + .flatten(), + } + } + Err(_) => WatchedSourcePathIdentity { + exists: false, + is_dir: false, + stable_id: None, + fallback_len: None, + fallback_modified_at: None, + }, + }) + .collect() +} + +#[cfg(unix)] +fn watched_source_path_stable_id(_path: &Path, metadata: &fs::Metadata) -> Option<(u64, u64)> { + use std::os::unix::fs::MetadataExt; + + Some((metadata.dev(), metadata.ino())) +} + +#[cfg(windows)] +fn watched_source_path_stable_id(path: &Path, _metadata: &fs::Metadata) -> Option<(u64, u64)> { + use std::os::windows::{fs::OpenOptionsExt, io::AsRawHandle}; + use windows_sys::Win32::Storage::FileSystem::{ + GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, FILE_FLAG_BACKUP_SEMANTICS, + FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + }; + + let file = fs::OpenOptions::new() + .access_mode(FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path) + .ok()?; + let mut info = BY_HANDLE_FILE_INFORMATION::default(); + let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle() as _, &mut info) }; + (ok != 0).then(|| { + let file_index = (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow); + (u64::from(info.dwVolumeSerialNumber), file_index) + }) +} + +#[cfg(not(any(unix, windows)))] +fn watched_source_path_stable_id(_path: &Path, _metadata: &fs::Metadata) -> Option<(u64, u64)> { + None +} + +pub(crate) fn search_refresh_source_fingerprint(sources: &[SourceInfo]) -> String { + let mut items = sources + .iter() + .map(|source| { + format!( + "{}|{}|{}", + source.provider.as_str(), + source.source_format, + source.path.display() + ) + }) + .collect::>(); + items.sort(); + let mut hasher = Sha256::new(); + hasher.update(items.join("\n").as_bytes()); + format!("{:x}", hasher.finalize()) +} + +impl std::fmt::Display for SearchRefreshFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(formatter) + } +} + +impl std::error::Error for SearchRefreshFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.error.as_ref()) + } +} + +pub(crate) fn search_refresh_failure_totals(error: &anyhow::Error) -> Option { + error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .map(|failure| failure.totals.clone()) +} + +impl SearchRefreshReport { + pub(crate) fn skipped(mode: RefreshArg, status: &'static str) -> Self { + Self { + mode, + status, + source_count: 0, + totals: ImportTotals::default(), + error: None, + } + } + + fn completed(mode: RefreshArg, source_count: usize, totals: ImportTotals) -> Self { + Self { + mode, + status: "completed", + source_count, + totals, + error: None, + } + } + + fn failed(mode: RefreshArg, source_count: usize, totals: ImportTotals, error: String) -> Self { + Self { + mode, + status: "failed", + source_count, + totals, + error: Some(error), + } + } + + pub(crate) fn to_json(&self) -> Value { + compact_json(json!({ + "mode": self.mode.as_str(), + "status": self.status, + "source_count": self.source_count, + "totals": import_totals_json(&self.totals), + "error": self.error, + })) + } +} + +pub(crate) fn run_search( + args: SearchArgs, + data_root: PathBuf, + analytics_properties: &mut AnalyticsProperties, + config: &config::AppConfig, +) -> Result<()> { + if !search_has_intent(SearchIntentInput { + query: args.query.as_deref(), + terms: &args.term, + file: args.file.as_deref(), + }) { + return Err(missing_search_intent_error()); + } + + let db_path = database_path(data_root.clone()); + let had_existing_store = db_path.exists(); + let indexed_content_before_search = if had_existing_store { + existing_store_indexed_content(&db_path) + } else { + Some(false) + }; + analytics::insert_bool( + analytics_properties, + "had_existing_store_before_search", + had_existing_store, + ); + analytics::insert_bool( + analytics_properties, + "indexed_content_before_search_known", + indexed_content_before_search.is_some(), + ); + analytics::insert_bool( + analytics_properties, + "had_indexed_content_before_search", + indexed_content_before_search.unwrap_or(false), + ); + let refresh_started = Instant::now(); + let refresh = refresh_before_search(&args, &data_root, config.daemon.enabled)?; + analytics::insert_duration( + analytics_properties, + "refresh_duration", + refresh_started.elapsed(), + ); + analytics::insert_str( + analytics_properties, + "search_refresh_mode", + refresh.mode.as_str(), + ); + analytics::insert_str( + analytics_properties, + "search_refresh_status", + refresh.status, + ); + analytics::insert_count_bucket( + analytics_properties, + "search_refresh_source_count_bucket", + refresh.source_count as u64, + ); + let backend_override = args.backend; + let requested_backend = resolve_search_backend(backend_override, config)?; + let semantic_enabled = config.semantic_search_enabled(); + if args.refresh == RefreshArg::Background + && semantic_enabled + && semantic::semantic_query_service_supported() + && matches!( + requested_backend, + SearchBackendArg::Semantic | SearchBackendArg::Hybrid + ) + { + semantic::maybe_autostart_daemon_for_search(&data_root, config); + semantic::wait_for_daemon_query_service(&data_root, StdDuration::from_secs(3)); + } + insert_db_size_bucket(analytics_properties, &db_path); + if args.refresh == RefreshArg::Background + && (refresh.status == "failed" || refresh.source_count > 0) + && !has_usable_search_fallback(indexed_content_before_search) + && !has_usable_search_fallback(existing_store_indexed_content(&db_path)) + { + if refresh.status == "failed" { + return Err(anyhow!( + "search refresh failed and no usable existing ctx index is available; run `ctx import` first or retry with `--refresh wait`: {}", + refresh.error.as_deref().unwrap_or("unknown refresh error") + )); + } + return Err(anyhow!( + "background search refresh has not produced a usable ctx index yet; indexing remains pending, so retry with `--refresh wait`" + )); + } + let store = if args.refresh == RefreshArg::Off + || refresh.status == "failed" + || refresh.status == "completed" + || had_existing_store + { + open_existing_store_read_only(&db_path, "ctx search")? + } else { + Store::open(&db_path)? + }; + analytics::insert_bool( + analytics_properties, + "store_created_by_search", + !had_existing_store && db_path.exists(), + ); + insert_store_analytics_counts(analytics_properties, &store)?; + analytics::insert_bool( + analytics_properties, + "has_indexed_content_after_search", + indexed_history_item_count(&store)? > 0, + ); + let source_identity = SourceIdentityFilterArgs::from(&args); + let query = args.query.unwrap_or_default(); + let query_term_count = query + .split_whitespace() + .filter(|term| !term.trim().is_empty()) + .count() + .saturating_add( + args.term + .iter() + .filter(|term| !term.trim().is_empty()) + .count(), + ); + analytics::insert_text_length_bucket( + analytics_properties, + "query_length_bucket", + query.chars().count(), + ); + analytics::insert_count_bucket( + analytics_properties, + "query_term_count_bucket", + query_term_count as u64, + ); + let event_results = args.events || args.session.is_some(); + let options = ctx_history_search::PacketOptions { + limit: args.limit, + filters: search_filters( + SearchFilterInput { + session: args.session, + provider: args.provider, + source_identity, + workspace: args.workspace.clone(), + since: args.since.clone(), + primary_only: args.primary_only, + include_subagents: args.include_subagents, + event_type: args.event_type.clone(), + file: args.file.clone(), + include_current_session: args.include_current_session, + }, + Some(&store), + )?, + result_mode: if event_results { + ctx_history_search::SearchResultMode::Events + } else { + ctx_history_search::SearchResultMode::Sessions + }, + ..ctx_history_search::PacketOptions::default() + }; + let uses_composed_terms = args.term.iter().any(|term| !term.trim().is_empty()); + let query_started = Instant::now(); + let (packet, retrieval) = search_packet_with_backend( + &store, + &data_root, + &query, + &args.term, + &options, + requested_backend, + semantic_enabled, + args.semantic_weight, + args.refresh, + !args.json, + )?; + analytics::insert_duration( + analytics_properties, + "query_duration", + query_started.elapsed(), + ); + analytics::insert_str( + analytics_properties, + "search_backend_requested", + requested_backend.as_str(), + ); + analytics::insert_str( + analytics_properties, + "search_backend_effective", + retrieval.effective_mode().as_str(), + ); + let result_count = packet.results.len(); + let citation_count = packet + .results + .iter() + .map(|result| result.citations.len()) + .sum::(); + analytics::insert_count_bucket( + analytics_properties, + "result_count_bucket", + result_count as u64, + ); + analytics::insert_count_bucket( + analytics_properties, + "citation_count_bucket", + citation_count as u64, + ); + analytics::insert_bool(analytics_properties, "zero_result", result_count == 0); + let render_started = Instant::now(); + if args.json { + let suggested_next_query = (!uses_composed_terms).then_some(query.as_str()); + print_json(SearchDto::packet( + &store, + &packet, + &refresh, + &retrieval, + suggested_next_query, + ))?; + } else { + if refresh.status == "failed" && args.refresh == RefreshArg::Background { + if let Some(error) = &refresh.error { + eprintln!( + "warning: search refresh failed; serving existing index; use --refresh wait to fail instead: {error}" + ); + } + } + if packet.results.is_empty() { + if let Some(file) = args + .file + .as_deref() + .filter(|_| query.trim().is_empty() && !uses_composed_terms) + { + println!("no indexed events touched {}", file.display()); + let indexed_items = indexed_history_item_count(&store)?; + if indexed_items == 0 { + println!("next: ctx import --all"); + } else { + println!( + "next: ctx search {}", + shell_quote_arg(&file.display().to_string()) + ); + } + } else { + println!( + "no results for {}", + search_no_results_target(&query, &args.term) + ); + let indexed_items = indexed_history_item_count(&store)?; + if indexed_items == 0 { + println!("next: ctx import --all"); + } else { + println!("next: try broader terms with ctx search --term \"\""); + } + } + } + let suggested_next_query = (!uses_composed_terms).then_some(query.as_str()); + for (index, result) in packet.results.iter().enumerate() { + if args.verbose { + print_search_result_verbose(result, suggested_next_query); + } else { + print_search_result_compact(index + 1, result); + } + } + } + analytics::insert_duration( + analytics_properties, + "render_duration", + render_started.elapsed(), + ); + Ok(()) +} + +fn has_usable_search_fallback(indexed_content: Option) -> bool { + indexed_content == Some(true) +} + +pub(crate) fn resolve_search_backend( + backend: Option, + config: &config::AppConfig, +) -> Result { + let semantic_enabled = config.semantic_search_enabled(); + match backend { + Some(SearchBackendArg::Semantic) if !semantic_enabled => Err(anyhow!( + "semantic search is disabled. Set [search] semantic = true in ctx config to enable the local semantic preview" + )), + Some(SearchBackendArg::Semantic) if !semantic::semantic_query_service_supported() => Err( + anyhow!( + "local semantic search is not supported on this platform yet. Set [search] semantic = false or use --backend lexical" + ), + ), + Some(SearchBackendArg::Semantic) if !config.daemon.enabled => Err(anyhow!( + "local semantic search requires the ctx daemon. Set [daemon] enabled = true, set [search] semantic = false, or use --backend lexical" + )), + value + if semantic_enabled + && semantic::semantic_query_service_supported() + && !config.daemon.enabled + && !matches!(value, Some(SearchBackendArg::Lexical)) => + { + Err(anyhow!( + "local semantic search requires the ctx daemon. Set [daemon] enabled = true, set [search] semantic = false, or use --backend lexical" + )) + } + Some(value) => Ok(value), + None if semantic_enabled => Ok(SearchBackendArg::Hybrid), + None => Ok(SearchBackendArg::Lexical), + } +} + +fn existing_store_indexed_content(db_path: &Path) -> Option { + open_existing_store_read_only(db_path, "ctx search analytics preflight") + .and_then(|store| indexed_history_item_count(&store)) + .ok() + .map(|indexed_items| indexed_items > 0) +} diff --git a/crates/ctx-cli/src/commands/search/refresh_execute.rs b/crates/ctx-cli/src/commands/search/refresh_execute.rs new file mode 100644 index 00000000..dce07c6d --- /dev/null +++ b/crates/ctx-cli/src/commands/search/refresh_execute.rs @@ -0,0 +1,733 @@ +#[allow(clippy::too_many_arguments)] +fn execute_search_refresh_work( + data_root: &Path, + store: &mut Store, + refresh_source_count: usize, + had_indexed_content: bool, + search_projection_needs_backfill: bool, + plugin_sources: Vec, + refresh: RefreshArg, + json_output: bool, + execution_policy: ImportExecutionPolicy, + plan: &ImportPlan, + execution_state: &mut crate::commands::import::ImportExecutionState, + inventory_failures: Vec, + failed_inventory_pending: (usize, usize), + planned_total_bytes: u64, +) -> Result { + let mut totals = ImportTotals::default(); + let mut deferred_units = 0usize; + let mut completed_units = 0usize; + + let progress_arg = match refresh { + RefreshArg::Wait if json_output => ProgressArg::Json, + RefreshArg::Wait => ProgressArg::Auto, + RefreshArg::Background | RefreshArg::Off => ProgressArg::None, + }; + let progress = ProgressReporter::new( + progress_arg, + json_output, + "search-refresh", + planned_total_bytes, + ); + totals.fresh_units_pending = failed_inventory_pending.0; + totals.recovery_units_pending = failed_inventory_pending.1; + let mut first_refresh_failure = None::; + for failure in inventory_failures { + first_refresh_failure.get_or_insert_with(|| failure.error.clone()); + totals.add_source_failure(&failure.stats); + progress.warning(format!( + "skipped {} during inventory: {}", + failure.source.provider.as_str(), + one_line_error(&failure.error) + )); + } + let tolerate_source_errors = refresh == RefreshArg::Background; + let mut imported_native_sources = BTreeSet::new(); + let mut failed_native_sources = BTreeSet::new(); + let fresh_slice_limit = execution_policy.fresh_slice_limit(); + let mut fresh_units = plan.fresh_units; + loop { + execution_state.begin_new_pass(); + if fresh_units == 0 { + break; + } + let result = match execute_search_refresh_plan_class( + store, + plan, + execution_state, + ImportWorkClass::Fresh, + fresh_units, + fresh_slice_limit, + &progress, + json_output, + tolerate_source_errors, + &mut totals, + &mut first_refresh_failure, + &mut imported_native_sources, + &mut failed_native_sources, + execution_policy == ImportExecutionPolicy::Drain, + ) { + Ok(result) => result, + Err(error) => { + return Err(refresh_failure_with_totals(error, store, plan, totals)); + } + }; + deferred_units = result.deferred_units; + completed_units = completed_units.saturating_add(result.completed_units); + fresh_units = match plan.pending_count(store, ImportWorkClass::Fresh) { + Ok(pending) => pending, + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + if fresh_units == 0 { + deferred_units = 0; + break; + } + if fresh_slice_limit.is_some() || !result.made_durable_progress() { + break; + } + } + if execution_policy != ImportExecutionPolicy::Drain + && (deferred_units > 0 || (totals.durable_progress && completed_units == 0)) + { + let (fresh_units_pending, recovery_units_pending) = + match reported_pending_counts(store, plan) { + Ok((fresh_units_pending, recovery_units_pending)) => { + totals.fresh_units_pending = + fresh_units_pending.saturating_add(failed_inventory_pending.0); + totals.recovery_units_pending = + recovery_units_pending.saturating_add(failed_inventory_pending.1); + (fresh_units_pending, recovery_units_pending) + } + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + let publication_fenced = match store.has_pending_provider_file_publications() { + Ok(pending) => pending, + Err(error) => { + return Err(refresh_failure_with_totals( + error.into(), + store, + plan, + totals, + )); + } + }; + if publication_fenced && (fresh_units_pending > 0 || recovery_units_pending > 0) { + return Ok(SearchRefreshExecution { totals }); + } + } + + if !plugin_sources.is_empty() { + for plugin_source in plugin_sources { + progress.message( + "refreshing", + format!("running history source plugin {}", plugin_source.label()), + ); + let import_result = + import_history_source_plugin(store, &plugin_source, data_root, false).with_context( + || format!("refresh history source plugin {}", plugin_source.label()), + ); + match import_result { + Ok((summary, stats)) => { + warn_on_rejected_records( + &progress, + json_output, + &plugin_source.label(), + &summary, + ); + totals.add(&summary, &stats); + progress.done( + "refreshing", + format!("refreshed history source plugin {}", plugin_source.label()), + 0, + ); + } + Err(err) + if refresh == RefreshArg::Background + && import_error_scope(&err) == ImportFailureScope::Source => + { + let error = error_summary(&err); + first_refresh_failure.get_or_insert_with(|| error.clone()); + add_refresh_source_failure(&mut totals, &SourceStats::default(), &err); + progress.done( + "refreshing", + format!( + "skipped history source plugin {}: {}", + plugin_source.label(), + one_line_error(&error) + ), + 0, + ); + } + Err(err) => { + return Err(refresh_failure_with_totals(err, store, plan, totals)); + } + } + } + } + + let recovery_slice_limit = execution_policy.recovery_slice_limit(); + let mut maintenance_complete = loop { + execution_state.begin_new_pass(); + let maintenance = match repair_import_maintenance(store, execution_policy) { + Ok(maintenance) => maintenance, + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + totals.durable_progress |= maintenance.processed_rows > 0; + if execution_policy != ImportExecutionPolicy::Drain && !maintenance.complete { + let (fresh_units_pending, recovery_units_pending) = match plan.pending_counts(store) { + Ok(counts) => counts, + Err(error) => { + return Err(refresh_failure_with_totals(error, store, plan, totals)); + } + }; + totals.fresh_units_pending = + fresh_units_pending.saturating_add(failed_inventory_pending.0); + totals.recovery_units_pending = recovery_units_pending + .saturating_add(failed_inventory_pending.1) + .saturating_add(1); + return Ok(SearchRefreshExecution { totals }); + } + let recovery_units = match plan.pending_count(store, ImportWorkClass::Recovery) { + Ok(pending) => pending, + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + let result = match execute_search_refresh_plan_class( + store, + plan, + execution_state, + ImportWorkClass::Recovery, + recovery_units, + recovery_slice_limit, + &progress, + json_output, + tolerate_source_errors, + &mut totals, + &mut first_refresh_failure, + &mut imported_native_sources, + &mut failed_native_sources, + execution_policy == ImportExecutionPolicy::Drain, + ) { + Ok(result) => result, + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + deferred_units = deferred_units.saturating_add(result.deferred_units); + if execution_policy == ImportExecutionPolicy::Drain { + let fresh_units = match plan.pending_count(store, ImportWorkClass::Fresh) { + Ok(pending) => pending, + Err(error) => { + return Err(refresh_failure_with_totals(error, store, plan, totals)); + } + }; + if fresh_units > 0 { + execution_state.begin_new_pass(); + let fresh_result = match execute_search_refresh_plan_class( + store, + plan, + execution_state, + ImportWorkClass::Fresh, + fresh_units, + fresh_slice_limit, + &progress, + json_output, + tolerate_source_errors, + &mut totals, + &mut first_refresh_failure, + &mut imported_native_sources, + &mut failed_native_sources, + true, + ) { + Ok(result) => result, + Err(error) => { + return Err(refresh_failure_with_totals(error, store, plan, totals)); + } + }; + deferred_units = deferred_units.saturating_add(fresh_result.deferred_units); + if fresh_result.made_durable_progress() { + continue; + } + let recovery_units_pending = + match plan.pending_count(store, ImportWorkClass::Recovery) { + Ok(pending) => pending, + Err(error) => { + return Err(refresh_failure_with_totals(error, store, plan, totals)); + } + }; + if recovery_units_pending > 0 + && (maintenance.processed_rows > 0 || result.made_durable_progress()) + { + continue; + } + break maintenance.complete; + } + } + let recovery_units_pending = match plan.pending_count(store, ImportWorkClass::Recovery) { + Ok(pending) => pending, + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + if maintenance.complete && recovery_units_pending == 0 { + break true; + } + if execution_policy != ImportExecutionPolicy::Drain { + break maintenance.complete; + } + if maintenance.processed_rows == 0 && !result.made_durable_progress() { + break maintenance.complete; + } + }; + let trailing_maintenance = match repair_import_maintenance(store, execution_policy) { + Ok(maintenance) => maintenance, + Err(error) => return Err(refresh_failure_with_totals(error, store, plan, totals)), + }; + totals.durable_progress |= trailing_maintenance.processed_rows > 0; + maintenance_complete &= trailing_maintenance.complete; + + if search_projection_needs_backfill { + if let Err(error) = store.refresh_search_index() { + return Err(refresh_failure_with_totals( + error.into(), + store, + plan, + totals, + )); + } + } + + let all_sources_failed = all_refresh_sources_failed(refresh_source_count, &totals); + let all_rejected_without_prior_index = !had_indexed_content + && totals.imported_sessions == 0 + && totals.imported_events == 0 + && totals.failed > 0; + if refresh == RefreshArg::Background && (all_sources_failed || all_rejected_without_prior_index) + { + let detail = first_refresh_failure + .map(|error| format!("; first failure: {error}")) + .or_else(|| { + (totals.failed > 0).then(|| { + format!( + "; background refresh imported no content and reported {} failure(s)", + totals.failed + ) + }) + }) + .unwrap_or_default(); + return Err(refresh_failure_with_totals( + anyhow!("all search refresh sources failed{detail}"), + store, + plan, + totals, + )); + } + + let (fresh_units_pending, recovery_units_pending) = match reported_pending_counts(store, plan) { + Ok(counts) => counts, + Err(error) => { + return Err(refresh_failure_with_totals(error, store, plan, totals)); + } + }; + totals.fresh_units_pending = fresh_units_pending.saturating_add(failed_inventory_pending.0); + totals.recovery_units_pending = recovery_units_pending + .saturating_add(failed_inventory_pending.1) + .saturating_add(usize::from(!maintenance_complete)); + + if let Err(error) = store.checkpoint_wal_truncate_if_larger_than(WAL_TRUNCATE_MIN_BYTES) { + return Err(anyhow::Error::new(SearchRefreshFailure { + error: error.into(), + totals, + })); + } + Ok(SearchRefreshExecution { totals }) +} + +fn all_refresh_sources_failed(source_count: usize, totals: &ImportTotals) -> bool { + source_count > 0 && totals.imported_sources == 0 && totals.failed_sources >= source_count +} + +fn reported_pending_counts(store: &Store, plan: &ImportPlan) -> Result<(usize, usize)> { + let (fresh, mut recovery) = plan.pending_counts(store)?; + if fresh == 0 && recovery == 0 && store.has_pending_provider_file_publications()? { + recovery = 1; + } + Ok((fresh, recovery)) +} + +fn refresh_failure_with_totals( + error: anyhow::Error, + store: &Store, + plan: &ImportPlan, + mut totals: ImportTotals, +) -> anyhow::Error { + if let Ok((fresh, recovery)) = reported_pending_counts(store, plan) { + totals.fresh_units_pending = totals.fresh_units_pending.saturating_add(fresh); + totals.recovery_units_pending = totals.recovery_units_pending.saturating_add(recovery); + } + anyhow::Error::new(SearchRefreshFailure { error, totals }) +} + +#[allow(clippy::too_many_arguments)] +fn execute_search_refresh_plan_class( + store: &mut Store, + plan: &ImportPlan, + execution_state: &mut crate::commands::import::ImportExecutionState, + class: ImportWorkClass, + remaining_units: usize, + max_slices: Option, + progress: &ProgressReporter, + json_output: bool, + tolerate_source_errors: bool, + totals: &mut ImportTotals, + first_refresh_failure: &mut Option, + imported_sources: &mut BTreeSet, + failed_sources: &mut BTreeSet, + drain_retirements: bool, +) -> Result { + execute_search_refresh_plan_class_with_pre_lock_hook( + store, + plan, + execution_state, + class, + remaining_units, + max_slices, + progress, + json_output, + tolerate_source_errors, + totals, + first_refresh_failure, + imported_sources, + failed_sources, + drain_retirements, + || {}, + ) +} + +#[allow(clippy::too_many_arguments)] +fn execute_search_refresh_plan_class_with_pre_lock_hook( + store: &mut Store, + plan: &ImportPlan, + execution_state: &mut crate::commands::import::ImportExecutionState, + class: ImportWorkClass, + mut remaining_units: usize, + max_slices: Option, + progress: &ProgressReporter, + json_output: bool, + tolerate_source_errors: bool, + totals: &mut ImportTotals, + first_refresh_failure: &mut Option, + imported_sources: &mut BTreeSet, + failed_sources: &mut BTreeSet, + drain_retirements: bool, + mut before_bulk_lock: impl FnMut(), +) -> Result { + let mut completed_bytes = 0u64; + let mut completed_slices = 0usize; + let mut execution_result = crate::commands::import::ImportExecutionResult::default(); + while remaining_units > 0 && max_slices.is_none_or(|limit| completed_slices < limit) { + let Some(executable) = plan + .select_slice_for_execution_with_pre_lock_hook( + store, + class, + remaining_units, + execution_state, + &mut before_bulk_lock, + ) + .context("select locked search refresh slice")? + else { + break; + }; + let ExecutableImportSlice { + slice, + bulk_guard, + validation_failures, + } = executable; + if slice.is_empty() && validation_failures.is_empty() { + store.finish_event_search_bulk_mode(&bulk_guard)?; + break; + } + let validation_units = validation_failures + .iter() + .map(|failure| failure.stats.files) + .sum::(); + let selected_units = slice.units.saturating_add(validation_units); + remaining_units = remaining_units.saturating_sub(selected_units); + completed_slices += 1; + let mut system_error = None; + let mut completed_units = 0usize; + let mut deferred_units = 0usize; + let mut maintenance_progress = false; + let mut source_durable_progress = false; + for validation_failure in validation_failures { + if !tolerate_source_errors { + system_error = Some(validation_failure.error); + break; + } + let source_plan = &plan.sources[validation_failure.source_index]; + let error = error_summary(&validation_failure.error); + first_refresh_failure.get_or_insert_with(|| error.clone()); + let first_source_result = failed_sources.insert(validation_failure.source_index); + add_refresh_source_failure( + totals, + &validation_failure.stats, + &validation_failure.error, + ); + if !first_source_result { + totals.failed_sources = totals.failed_sources.saturating_sub(1); + } + progress.done( + "refreshing", + format!( + "skipped {}: {}", + source_plan.source.provider.as_str(), + one_line_error(&error) + ), + completed_bytes, + ); + } + for retirement in &slice.retirements { + if system_error.is_some() { + break; + } + execution_state.record_retirement_attempt(retirement); + progress.message("repairing", "repairing prior hidden provider history"); + match recover_provider_file_publication_retirement(store, retirement, drain_retirements) + { + Ok(outcome) => { + maintenance_progress |= outcome.made_durable_progress; + if outcome.completed { + completed_units = completed_units.saturating_add(1); + } + for warning in outcome.maintenance_warnings { + progress.warning(warning.to_string()); + } + } + Err(error) => { + system_error = Some(error); + break; + } + } + } + for selected in slice.sources { + if system_error.is_some() { + break; + } + let source_plan = &plan.sources[selected.source_index]; + let (phase, message) = import_work_progress_message(class, source_plan.source.provider); + progress.message(phase, message); + let source_progress = + progress.codex_import_callback(&source_plan.source, completed_bytes); + execution_state.record_source_attempt(&selected.work); + if let Err(error) = selected.persist_attempt_started(store) { + system_error = Some(error); + break; + } + let import_result = import_selected_source( + store, + &source_plan.source, + source_progress, + &selected.preinventory, + &selected.work, + ); + let (outcome, import_error) = match import_result { + Ok(result) => (Some(result.outcome), result.remaining_error), + Err(error) => (None, Some(error)), + }; + let mut outcome_completed_units = 0usize; + let mut outcome_completed_bytes = 0u64; + let mut outcome_deferred_units = 0usize; + let mut outcome_rejected_without_content = false; + let had_outcome = outcome.is_some(); + if let Some(outcome) = outcome { + let made_durable_progress = outcome.made_durable_progress(); + execution_state.record_source_outcome( + selected.source_index, + &selected.work, + outcome.post_import_preinventory.clone(), + ); + source_durable_progress |= made_durable_progress; + outcome_completed_units = outcome.completed_units; + outcome_completed_bytes = outcome.completed_bytes; + outcome_deferred_units = outcome.deferred_units; + completed_units = completed_units.saturating_add(outcome.completed_units); + let deferred = outcome.deferred_units; + deferred_units = deferred_units.saturating_add(deferred); + warn_on_rejected_records( + progress, + json_output, + source_plan.source.provider.as_str(), + &outcome.summary, + ); + outcome_rejected_without_content = + outcome.summary.failed > 0 && !outcome.summary.has_accepted_content(); + let reportable_no_op = outcome.completed_units == 0 + && deferred == 0 + && import_error.is_none() + && outcome.summary != ProviderImportSummary::default(); + if outcome.completed_units > 0 || reportable_no_op { + let completed_stats = SourceStats { + files: if reportable_no_op { + selected.stats.files + } else { + outcome.completed_units + }, + bytes: if reportable_no_op { + selected.stats.bytes + } else { + outcome.completed_bytes + }, + change_token: selected.stats.change_token, + }; + completed_bytes = completed_bytes.saturating_add(completed_stats.bytes); + if outcome_rejected_without_content { + let first_source_result = failed_sources.insert(selected.source_index); + totals.add_rejected_source(&outcome.summary, &completed_stats); + if !first_source_result { + totals.failed_sources = totals.failed_sources.saturating_sub(1); + } + } else { + let first_source_result = imported_sources.insert(selected.source_index); + totals.add(&outcome.summary, &completed_stats); + if !first_source_result { + totals.imported_sources = totals.imported_sources.saturating_sub(1); + } + } + let (phase, message) = import_work_progress_done(class, &source_plan.source); + progress.done(phase, message, completed_bytes); + } else if deferred > 0 && !made_durable_progress { + progress.done( + phase, + format!( + "Deferred incomplete {} history.", + source_plan.source.provider.as_str() + ), + completed_bytes, + ); + } + if deferred > 0 && !made_durable_progress { + progress.warning(format!( + "{deferred} {} history unit(s) remain pending until their current write completes.", + source_plan.source.provider.as_str() + )); + } + } + if let Some(err) = import_error { + if !had_outcome { + execution_state.record_source_outcome( + selected.source_index, + &selected.work, + None, + ); + } + if tolerate_source_errors && import_error_scope(&err) == ImportFailureScope::Source + { + if let Some(warning) = publication_recovery_maintenance_warning(&err) { + progress.warning(warning.to_string()); + } + let error = error_summary(&err); + first_refresh_failure.get_or_insert_with(|| error.clone()); + let failure_stats = SourceStats { + files: selected.stats.files.saturating_sub( + outcome_completed_units.saturating_add(outcome_deferred_units), + ), + bytes: selected.stats.bytes.saturating_sub(outcome_completed_bytes), + change_token: selected.stats.change_token, + }; + if !outcome_rejected_without_content { + let first_source_result = failed_sources.insert(selected.source_index); + add_refresh_source_failure(totals, &failure_stats, &err); + if !first_source_result { + totals.failed_sources = totals.failed_sources.saturating_sub(1); + } + } + progress.done( + "refreshing", + format!( + "skipped {}: {}", + source_plan.source.provider.as_str(), + one_line_error(&error) + ), + completed_bytes, + ); + } else { + if let Some(warning) = publication_recovery_maintenance_warning(&err) { + progress.warning(warning.to_string()); + } + system_error = Some(err); + break; + } + } + if outcome_deferred_units > 0 && store.has_pending_provider_file_publications()? { + break; + } + } + store + .finish_event_search_bulk_mode(&bulk_guard) + .context("finish search refresh bulk mode")?; + match class { + ImportWorkClass::Fresh => { + totals.fresh_units_processed = + totals.fresh_units_processed.saturating_add(completed_units); + } + ImportWorkClass::Recovery => { + totals.recovery_units_processed = totals + .recovery_units_processed + .saturating_add(completed_units); + } + } + execution_result.add_slice( + selected_units, + completed_units, + deferred_units, + maintenance_progress || source_durable_progress, + ); + totals.durable_progress |= + completed_units > 0 || maintenance_progress || source_durable_progress; + if let Some(error) = system_error { + return Err(error); + } + if deferred_units > 0 && store.has_pending_provider_file_publications()? { + break; + } + } + Ok(execution_result) +} + +fn add_refresh_source_failure( + totals: &mut ImportTotals, + stats: &SourceStats, + error: &anyhow::Error, +) { + if let Some(summary) = rejected_source_summary(error) { + totals.add_rejected_source(&summary, stats); + } else { + totals.add_source_failure(stats); + } +} + +fn warn_on_rejected_records( + progress: &ProgressReporter, + json_output: bool, + source: &str, + summary: &ProviderImportSummary, +) { + if summary.failed == 0 { + return; + } + let first_failure = summary + .failures + .first() + .map(|failure| { + format!( + "; first failure at line {}: {}", + failure.line, failure.error + ) + }) + .unwrap_or_default(); + let warning = format!( + "refreshed {source} with {} rejected history record(s){first_failure}", + summary.failed + ); + if progress.is_enabled() { + progress.warning(warning); + } else if !json_output { + eprintln!("warning: {warning}"); + } +} diff --git a/crates/ctx-cli/src/commands/search/refresh_plan.rs b/crates/ctx-cli/src/commands/search/refresh_plan.rs new file mode 100644 index 00000000..027a7c86 --- /dev/null +++ b/crates/ctx-cli/src/commands/search/refresh_plan.rs @@ -0,0 +1,361 @@ +pub(crate) fn refresh_before_search( + args: &SearchArgs, + data_root: &Path, + daemon_enabled: bool, +) -> Result { + if args.refresh == RefreshArg::Off { + return Ok(SearchRefreshReport::skipped(RefreshArg::Off, "skipped")); + } + if args.refresh == RefreshArg::Background && daemon_enabled { + return Ok(SearchRefreshReport::skipped( + RefreshArg::Background, + "daemon_background", + )); + } + let execution_policy = match args.refresh { + RefreshArg::Wait => ImportExecutionPolicy::Drain, + RefreshArg::Background | RefreshArg::Off => ImportExecutionPolicy::Interactive, + }; + let _disk_io_pacing = + ctx_history_capture::install_disk_io_pacer(execution_policy.disk_io_pacer()); + let source_identity = normalize_source_identity_filters(SourceIdentityFilterArgs::from(args))?; + if !source_identity.is_empty() + && args + .provider + .is_some_and(|provider| !matches!(provider, ProviderArg::Custom)) + { + return Err(anyhow!( + "custom history source filters can only be combined with --provider custom" + )); + } + let sources = if source_identity.is_empty() { + search_refresh_sources(args.provider) + } else { + Vec::new() + }; + let plugin_sources = + match search_refresh_plugin_sources(data_root, args.provider, &source_identity) { + Ok(sources) => sources, + Err(err) if args.refresh == RefreshArg::Background => { + return Ok(SearchRefreshReport::failed( + RefreshArg::Background, + sources.len(), + ImportTotals::default(), + error_summary(&err), + )); + } + Err(err) => return Err(err.context("search refresh failed")), + }; + if sources.is_empty() + && plugin_sources.is_empty() + && !search_refresh_has_publication_work(data_root)? + { + if args.refresh == RefreshArg::Wait { + return Err(anyhow!( + "wait search refresh found no supported discovered native provider or enabled auto history-source plugin sources; rerun the search with --refresh off to use the existing index" + )); + } + return Ok(SearchRefreshReport::skipped(args.refresh, "no_sources")); + } + let source_count = sources.len().saturating_add(plugin_sources.len()); + match refresh_sources_for_search( + data_root, + sources, + plugin_sources, + args.refresh, + args.json, + execution_policy, + ) { + Ok(totals) => Ok(SearchRefreshReport::completed( + args.refresh, + source_count, + totals, + )), + Err(err) if args.refresh == RefreshArg::Background => Ok(SearchRefreshReport::failed( + RefreshArg::Background, + source_count, + search_refresh_failure_totals(&err).unwrap_or_default(), + error_summary(&err), + )), + Err(err) => Err(err.context("search refresh failed")), + } +} + +pub(crate) fn search_refresh_sources(provider: Option) -> Vec { + let Some(home) = home_dir() else { + return Vec::new(); + }; + let mut sources = if let Some(provider) = provider { + discover_provider_sources_for_provider(&home, provider.capture_provider()) + } else { + discovered_sources() + }; + sources + .drain(..) + .filter(|source| { + source.exists + && source.import_support.is_auto_importable() + && source.status == ProviderSourceStatus::Available + && source.source_format != "codex_history_jsonl" + }) + .collect() +} + +pub(crate) fn search_refresh_plugin_sources( + data_root: &Path, + provider: Option, + source_identity: &SourceIdentityFilters, +) -> Result> { + if !matches!(provider, None | Some(ProviderArg::Custom)) { + return Ok(Vec::new()); + } + Ok(discover_history_source_plugins(data_root, &[])? + .into_iter() + .filter(|source| { + source.enabled + && source.refresh == HistorySourcePluginRefresh::Auto + && source_identity.matches_plugin_source(source) + }) + .collect()) +} + +pub(crate) fn search_refresh_has_publication_work(data_root: &Path) -> Result { + let db_path = database_path(data_root.to_path_buf()); + if !db_path.exists() { + return Ok(false); + } + let store = Store::open(&db_path)?; + Ok(store.has_pending_provider_file_publications()? + || store.provider_file_publication_retirement_work_count()? > 0) +} + +pub(crate) fn refresh_sources_for_search( + data_root: &Path, + sources: Vec, + plugin_sources: Vec, + refresh: RefreshArg, + json_output: bool, + execution_policy: ImportExecutionPolicy, +) -> Result { + refresh_sources_for_search_inner( + data_root, + sources, + plugin_sources, + refresh, + json_output, + execution_policy, + None, + ) +} + +pub(crate) fn refresh_sources_for_search_with_runtime( + data_root: &Path, + sources: Vec, + plugin_sources: Vec, + refresh: RefreshArg, + json_output: bool, + execution_policy: ImportExecutionPolicy, + runtime: &mut SearchRefreshRuntime, +) -> Result { + refresh_sources_for_search_inner( + data_root, + sources, + plugin_sources, + refresh, + json_output, + execution_policy, + Some(runtime), + ) +} + +fn refresh_sources_for_search_inner( + data_root: &Path, + sources: Vec, + plugin_sources: Vec, + refresh: RefreshArg, + json_output: bool, + execution_policy: ImportExecutionPolicy, + mut runtime: Option<&mut SearchRefreshRuntime>, +) -> Result { + let disk_io_pacer = runtime + .as_deref_mut() + .map(|runtime| runtime.disk_io_pacer(execution_policy)) + .unwrap_or_else(|| execution_policy.disk_io_pacer()); + let _disk_io_pacing = ctx_history_capture::install_disk_io_pacer(disk_io_pacer); + fs::create_dir_all(data_root)?; + config::write_default_config(data_root)?; + let db_path = database_path(data_root.to_path_buf()); + let mut store = Store::open(&db_path)?; + let had_indexed_content = store.indexed_history_item_count()? > 0; + let search_projection_needs_backfill = store.event_search_projection_needs_backfill()?; + let source_fingerprint = search_refresh_source_fingerprint(&sources); + let watcher_changes = runtime + .as_deref_mut() + .and_then(|runtime| runtime.watcher_changes(&sources)); + let work = if let Some(runtime) = runtime.as_deref_mut() { + let publication_owner = store.effective_provider_file_publication_inventory_owner()?; + let publication_pending = publication_owner.is_some(); + let periodic_reinventory = runtime.cached_work.as_ref().is_some_and(|cached| { + daemon_search_refresh_reinventory_due( + cached.last_reinventory_at.elapsed(), + publication_pending, + ) + }); + let rebuild = runtime.cached_work.as_ref().is_none_or(|cached| { + cached.source_fingerprint != source_fingerprint + || cached.publication_owner != publication_owner + || watcher_changes + .as_ref() + .is_some_and(|changes| changes.full_rebuild) + || periodic_reinventory + }); + if rebuild { + runtime.cached_work = None; + let inventory = inventory_import_sources(&store, sources, false)?; + let inventoried_source_count = inventory.totals.sources; + let plan = ImportPlan::build(&store, inventory.sources)?; + let inventory_failures = inventory.failures; + let failed_inventory_pending = + failed_inventory_pending_counts(&store, &inventory_failures)?; + runtime.cached_work = Some(SearchRefreshWork { + source_fingerprint, + publication_owner, + passes_since_reinventory: 0, + last_reinventory_at: Instant::now(), + execution_state: crate::commands::import::ImportExecutionState::for_plan(&plan), + plan, + inventory_failures, + failed_inventory_pending, + inventoried_source_count, + planned_total_bytes: inventory.totals.source_bytes, + }); + } else if let Some(changes) = watcher_changes { + if !changes.dirty_paths.is_empty() { + let refresh_result = refresh_dirty_search_sources( + &store, + &sources, + &changes.dirty_paths, + runtime + .cached_work + .as_mut() + .expect("daemon refresh cache must be populated"), + ); + if let Err(error) = refresh_result { + runtime.invalidate(); + return Err(error); + } + } + } + let work = runtime + .cached_work + .as_mut() + .expect("daemon refresh cache must be populated"); + work.execution_state.begin_new_pass(); + work.passes_since_reinventory = work.passes_since_reinventory.saturating_add(1); + work + } else { + let inventory = inventory_import_sources(&store, sources, false)?; + let refresh_source_count = inventory + .totals + .sources + .saturating_add(plugin_sources.len()); + let plan = ImportPlan::build(&store, inventory.sources)?; + let inventory_failures = inventory.failures; + let failed_inventory_pending = + failed_inventory_pending_counts(&store, &inventory_failures)?; + let planned_total_bytes = inventory.totals.source_bytes; + let mut execution_state = crate::commands::import::ImportExecutionState::for_plan(&plan); + return execute_search_refresh_work( + data_root, + &mut store, + refresh_source_count, + had_indexed_content, + search_projection_needs_backfill, + plugin_sources, + refresh, + json_output, + execution_policy, + &plan, + &mut execution_state, + inventory_failures, + failed_inventory_pending, + planned_total_bytes, + ) + .map(|execution| execution.totals); + }; + + let result = execute_search_refresh_work( + data_root, + &mut store, + work.inventoried_source_count + .saturating_add(plugin_sources.len()), + had_indexed_content, + search_projection_needs_backfill, + plugin_sources, + refresh, + json_output, + execution_policy, + &work.plan, + &mut work.execution_state, + work.inventory_failures.clone(), + work.failed_inventory_pending, + work.planned_total_bytes, + ); + match result { + Ok(execution) => Ok(execution.totals), + Err(error) => { + if let Some(runtime) = runtime { + runtime.invalidate(); + } + Err(error) + } + } +} + +fn refresh_dirty_search_sources( + store: &Store, + sources: &[SourceInfo], + dirty_paths: &BTreeSet, + work: &mut SearchRefreshWork, +) -> Result<()> { + let dirty_sources = sources + .iter() + .filter(|source| dirty_paths.contains(&source.path)) + .cloned() + .collect::>(); + if dirty_sources.is_empty() { + return Ok(()); + } + + let inventory = inventory_import_sources(store, dirty_sources, false)?; + let mut planned_sources = work.plan.sources.clone(); + planned_sources.retain(|planned| !dirty_paths.contains(&planned.source.path)); + planned_sources.extend( + inventory + .sources + .into_iter() + .filter(|planned| dirty_paths.contains(&planned.source.path)), + ); + let plan = ImportPlan::build(store, planned_sources)?; + + work.inventory_failures + .retain(|failure| !dirty_paths.contains(&failure.source.path)); + work.inventory_failures.extend(inventory.failures); + work.failed_inventory_pending = + failed_inventory_pending_counts(store, &work.inventory_failures)?; + work.planned_total_bytes = plan.sources.iter().fold(0u64, |total, source| { + total.saturating_add(source.stats.bytes) + }); + work.execution_state = work + .execution_state + .rebase_for_plan(&work.plan, &plan, dirty_paths); + work.plan = plan; + Ok(()) +} + +fn daemon_search_refresh_reinventory_due( + elapsed_since_reinventory: StdDuration, + publication_pending: bool, +) -> bool { + !publication_pending && elapsed_since_reinventory >= DAEMON_SEARCH_REFRESH_REINVENTORY_INTERVAL +} diff --git a/crates/ctx-cli/src/commands/setup.rs b/crates/ctx-cli/src/commands/setup.rs index 0edca15c..d30b2bcf 100644 --- a/crates/ctx-cli/src/commands/setup.rs +++ b/crates/ctx-cli/src/commands/setup.rs @@ -30,6 +30,13 @@ pub(crate) fn run_setup( quiet: bool, config: &config::AppConfig, ) -> Result<()> { + let daemon_backgrounding_enabled = config.daemon.enabled && !args.no_daemon; + let setup_policy = if daemon_backgrounding_enabled && !args.wait && !args.catalog_only { + crate::commands::import::ImportExecutionPolicy::Daemon + } else { + crate::commands::import::ImportExecutionPolicy::Drain + }; + let _disk_io_pacing = ctx_history_capture::install_disk_io_pacer(setup_policy.disk_io_pacer()); fs::create_dir_all(&data_root)?; let db_path = database_path(data_root.clone()); let store = Store::open(&db_path)?; @@ -45,7 +52,6 @@ pub(crate) fn run_setup( let sources = discovered_sources(); let progress_arg = setup_progress_arg(args.progress, quiet); let progress = ProgressReporter::new(progress_arg, args.json, "setup", 0); - let daemon_backgrounding_enabled = config.daemon.enabled && !args.no_daemon; let foreground_import = !args.catalog_only && (args.wait || !daemon_backgrounding_enabled); let mut inventory_only = None; let import_report = if args.catalog_only || !foreground_import { @@ -170,13 +176,7 @@ pub(crate) fn run_setup( "data_root": data_root, "database_path": db_path, "config_path": config_path, - "mode": if args.catalog_only { - "catalog_only" - } else if foreground_import { - "ready" - } else { - "background" - }, + "mode": setup_mode(args.catalog_only, foreground_import, import_report.as_ref()), "indexed_items": indexed_items, "sources": sources_json(&sources), "inventory": inventory_totals_json( @@ -294,6 +294,24 @@ pub(crate) fn run_setup( Ok(()) } +fn setup_mode( + catalog_only: bool, + foreground_import: bool, + report: Option<&ImportReport>, +) -> &'static str { + if catalog_only { + "catalog_only" + } else if !foreground_import { + "background" + } else if report.is_some_and(|report| { + report.totals.fresh_units_pending > 0 || report.totals.recovery_units_pending > 0 + }) { + "partial" + } else { + "ready" + } +} + fn setup_progress_arg(progress: ProgressArg, quiet: bool) -> ProgressArg { if quiet && progress == ProgressArg::Auto { ProgressArg::None @@ -419,7 +437,15 @@ pub(crate) fn print_setup_status_line( println!("ctx is initialized; no local history was indexed"); return; }; - if setup_has_indexed_content(indexed_items) && report.totals.failed_sources > 0 { + let pending_units = report + .totals + .fresh_units_pending + .saturating_add(report.totals.recovery_units_pending); + if setup_has_indexed_content(indexed_items) && pending_units > 0 { + println!( + "ctx indexed available local agent history; {pending_units} history unit(s) are still pending" + ); + } else if setup_has_indexed_content(indexed_items) && report.totals.failed_sources > 0 { println!("ctx indexed available local agent history; some sources were skipped"); } else if setup_has_indexed_content(indexed_items) { println!("ctx local agent history search is ready"); @@ -693,4 +719,18 @@ mod setup_estimate_tests { fn duration_estimate_carries_rounded_minutes_into_hours() { assert_eq!(format_duration_estimate(7_199), "2 hours"); } + + #[test] + fn foreground_setup_json_is_partial_while_import_work_remains() { + let mut report = ImportReport::empty(false); + report.totals.fresh_units_pending = 1; + assert_eq!(setup_mode(false, true, Some(&report)), "partial"); + + report.totals.fresh_units_pending = 0; + report.totals.recovery_units_pending = 1; + assert_eq!(setup_mode(false, true, Some(&report)), "partial"); + + report.totals.recovery_units_pending = 0; + assert_eq!(setup_mode(false, true, Some(&report)), "ready"); + } } diff --git a/crates/ctx-cli/src/progress.rs b/crates/ctx-cli/src/progress.rs index 0ecc767e..7a839efd 100644 --- a/crates/ctx-cli/src/progress.rs +++ b/crates/ctx-cli/src/progress.rs @@ -7,12 +7,9 @@ use std::{ use clap::ValueEnum; use serde_json::json; -use ctx_history_capture::{ - CodexSessionImportProgress, CodexSessionImportProgressCallback, ProviderImportSummary, -}; +use ctx_history_capture::{CodexSessionImportProgress, CodexSessionImportProgressCallback}; use ctx_history_core::CaptureProvider; -use crate::commands::import::{source_error_reason, SourceStats}; use crate::provider_sources::SourceInfo; #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] @@ -22,12 +19,6 @@ pub(crate) enum ProgressArg { Json, None, } -#[derive(Debug, Clone, Copy, Default)] -pub(crate) struct SourceProgressSnapshot { - pub(crate) completed_bytes: u64, - pub(crate) total_bytes: u64, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ProgressRenderMode { None, @@ -203,116 +194,6 @@ impl ProgressReporter { })) } - pub(crate) fn parallel_codex_import_callback( - &self, - source: &SourceInfo, - source_index: usize, - source_states: Arc>>, - ) -> Option { - if !self.is_enabled() || source.provider != CaptureProvider::Codex { - return None; - } - let reporter = self.clone(); - let provider = source.provider.as_str().to_owned(); - Some(Arc::new(move |progress: CodexSessionImportProgress| { - let (completed_bytes, total_bytes) = { - let mut states = source_states - .lock() - .expect("parallel progress state poisoned"); - if let Some(state) = states.get_mut(source_index) { - state.total_bytes = state.total_bytes.max(progress.total_bytes); - state.completed_bytes = progress - .completed_bytes - .min(state.total_bytes.max(progress.completed_bytes)); - } - aggregate_source_progress(&states) - }; - reporter.emit(ProgressLine { - phase: "indexing", - message: provider.clone(), - completed_bytes, - total_bytes: reporter.total_bytes.max(total_bytes).max(completed_bytes), - completed_files: Some(progress.completed_files), - total_files: Some(progress.total_files), - imported_events: Some(progress.imported_events), - done: progress.done, - force: progress.done, - }); - })) - } - - pub(crate) fn parallel_source_done( - &self, - source: &SourceInfo, - source_index: usize, - source_states: &Arc>>, - stats: SourceStats, - summary: &ProviderImportSummary, - ) { - if !self.is_enabled() { - return; - } - let (completed_bytes, total_bytes) = { - let mut states = source_states - .lock() - .expect("parallel progress state poisoned"); - if let Some(state) = states.get_mut(source_index) { - state.total_bytes = state.total_bytes.max(stats.bytes); - state.completed_bytes = state.total_bytes; - } - aggregate_source_progress(&states) - }; - self.emit(ProgressLine { - phase: "indexing", - message: format!("imported {}", source.provider.as_str()), - completed_bytes, - total_bytes: self.total_bytes.max(total_bytes).max(completed_bytes), - completed_files: Some(stats.files), - total_files: Some(stats.files), - imported_events: Some(summary.imported_events), - done: true, - force: true, - }); - } - - pub(crate) fn parallel_source_failed( - &self, - source: &SourceInfo, - source_index: usize, - source_states: &Arc>>, - stats: SourceStats, - error: &str, - ) { - if !self.is_enabled() { - return; - } - let (completed_bytes, total_bytes) = { - let mut states = source_states - .lock() - .expect("parallel progress state poisoned"); - if let Some(state) = states.get_mut(source_index) { - state.total_bytes = state.total_bytes.max(stats.bytes); - state.completed_bytes = state.total_bytes; - } - aggregate_source_progress(&states) - }; - self.emit(ProgressLine { - phase: "indexing", - message: format!( - "skipped {}: {}", - source.provider.as_str(), - source_error_reason(source, error) - ), - completed_bytes, - total_bytes: self.total_bytes.max(total_bytes).max(completed_bytes), - completed_files: Some(stats.files), - total_files: Some(stats.files), - imported_events: Some(0), - done: true, - force: true, - }); - } - fn emit(&self, line: ProgressLine) { let mut state = self.state.lock().expect("progress state poisoned"); let now = Instant::now(); @@ -368,18 +249,6 @@ impl ProgressReporter { } } -fn aggregate_source_progress(states: &[SourceProgressSnapshot]) -> (u64, u64) { - states - .iter() - .fold((0u64, 0u64), |(completed, total), state| { - let source_total = state.total_bytes.max(state.completed_bytes); - ( - completed.saturating_add(state.completed_bytes.min(source_total)), - total.saturating_add(source_total), - ) - }) -} - struct ProgressLine { phase: &'static str, message: String, diff --git a/crates/ctx-cli/src/semantic/daemon.rs b/crates/ctx-cli/src/semantic/daemon.rs index 12402799..33efbcfa 100644 --- a/crates/ctx-cli/src/semantic/daemon.rs +++ b/crates/ctx-cli/src/semantic/daemon.rs @@ -8,6 +8,7 @@ struct DaemonIteration { struct DaemonRuntime { semantic_embedder: Arc>>, semantic_bootstrap_passes_since_refresh: usize, + history_refresh: crate::commands::search::SearchRefreshRuntime, } fn daemon_runtime_embedder_loaded(runtime: &DaemonRuntime) -> bool { @@ -290,7 +291,8 @@ fn bind_daemon_query_listener( break; } } - if let Err(error) = fs::set_permissions(&runtime_dir, fs::Permissions::from_mode(0o700)) { + if let Err(error) = fs::set_permissions(&runtime_dir, fs::Permissions::from_mode(0o700)) + { let _ = fs::remove_dir(&runtime_dir); failures.push(format!("secure {}: {error}", runtime_dir.display())); continue; @@ -298,7 +300,10 @@ fn bind_daemon_query_listener( let path = runtime_dir.join("q.sock"); if path.as_os_str().as_bytes().len() > DAEMON_QUERY_SOCKET_PATH_SAFE_BYTES { let _ = fs::remove_dir(&runtime_dir); - failures.push(format!("fallback socket path is still too long: {}", path.display())); + failures.push(format!( + "fallback socket path is still too long: {}", + path.display() + )); continue; } match UnixListener::bind(&path) { @@ -371,11 +376,8 @@ fn start_daemon_query_service_with_request_timeout( // macOS. A 384-float response exceeds the default // socket buffer, so restore bounded blocking writes // before serving the request. - if configure_daemon_query_stream_unix( - &stream, - request_read_timeout, - ) - .is_err() + if configure_daemon_query_stream_unix(&stream, request_read_timeout) + .is_err() { continue; } @@ -680,8 +682,7 @@ fn create_windows_daemon_query_pipe( FILE_FLAG_FIRST_PIPE_INSTANCE, PIPE_ACCESS_DUPLEX, }; use windows_sys::Win32::System::Pipes::{ - CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, - PIPE_WAIT, + CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_WAIT, }; if !windows_named_pipe_name_is_local(pipe_name) { @@ -760,7 +761,9 @@ fn start_daemon_query_service( _data_root: &Path, _embedder: Arc>>, ) -> Result { - Err(anyhow!("daemon query service is not supported on this platform")) + Err(anyhow!( + "daemon query service is not supported on this platform" + )) } fn handle_daemon_query_stream( @@ -827,7 +830,10 @@ fn handle_daemon_query_stream_inner( if op != "embed_query" { return Err(anyhow!("unknown daemon query operation `{op}`")); } - let model_key = request.get("model_key").and_then(Value::as_str).unwrap_or(""); + let model_key = request + .get("model_key") + .and_then(Value::as_str) + .unwrap_or(""); if model_key != semantic_model_key() { return Err(anyhow!("daemon query model key mismatch")); } @@ -843,10 +849,12 @@ fn handle_daemon_query_stream_inner( { let mut guard = lock_shared_semantic_embedder(embedder)?; if guard.is_none() { - let cache_dir = semantic_worker_cache_dir(data_root); - if !semantic_model_cache_available(&cache_dir) { - return Err(anyhow!("semantic model cache is not available to daemon query service")); - } + let cache_dir = semantic_worker_cache_dir(data_root); + if !semantic_model_cache_available(&cache_dir) { + return Err(anyhow!( + "semantic model cache is not available to daemon query service" + )); + } *guard = Some(new_semantic_embedder(&cache_dir)?); } } @@ -897,7 +905,10 @@ impl Drop for DaemonTestJobHookGuard { #[cfg(all(test, ctx_sqlite_vec))] fn install_daemon_test_job_hooks(hooks: DaemonTestJobHooks) -> DaemonTestJobHookGuard { DAEMON_TEST_JOB_HOOKS.with(|slot| { - assert!(slot.borrow().is_none(), "daemon test job hook already installed"); + assert!( + slot.borrow().is_none(), + "daemon test job hook already installed" + ); *slot.borrow_mut() = Some(hooks); }); DaemonTestJobHookGuard @@ -1059,9 +1070,7 @@ fn run_daemon(args: DaemonRunArgs, data_root: PathBuf, config: &AppConfig) -> Re )); } let semantic_enabled = config.semantic_search_enabled() && semantic_query_service_supported(); - if semantic_enabled { - lower_semantic_worker_priority(); - } + lower_daemon_worker_priority(); let report = match run_daemon_inner( args.clone(), &data_root, @@ -1126,7 +1135,10 @@ fn run_daemon_inner( let mut runtime = DaemonRuntime::default(); let query_service = if semantic_enabled { - Some(start_daemon_query_service(data_root, runtime.semantic_embedder.clone())?) + Some(start_daemon_query_service( + data_root, + runtime.semantic_embedder.clone(), + )?) } else { None }; @@ -1213,12 +1225,19 @@ fn run_daemon_once( if semantic_enabled && semantic_bootstrap_should_run_first(data_root, runtime)? { let history_refresh_job = daemon_history_refresh_skipped_job("semantic_bootstrap_in_progress"); - write_daemon_job_status(&daemon_history_refresh_job_path(data_root), &history_refresh_job)?; - let semantic_job = run_daemon_semantic_job(args, data_root, runtime, deadline, semantic_enabled) - .unwrap_or_else(|error| daemon_semantic_failed_job(data_root, format!("{error:#}"))); + write_daemon_job_status( + &daemon_history_refresh_job_path(data_root), + &history_refresh_job, + )?; + let semantic_job = + run_daemon_semantic_job(args, data_root, runtime, deadline, semantic_enabled) + .unwrap_or_else(|error| { + daemon_semantic_failed_job(data_root, format!("{error:#}")) + }); let semantic_did_work = daemon_semantic_job_did_work(&semantic_job); - runtime.semantic_bootstrap_passes_since_refresh = - runtime.semantic_bootstrap_passes_since_refresh.saturating_add(1); + runtime.semantic_bootstrap_passes_since_refresh = runtime + .semantic_bootstrap_passes_since_refresh + .saturating_add(1); write_daemon_job_status_unless_deadline_skip( &daemon_semantic_job_path(data_root), &semantic_job, @@ -1233,7 +1252,7 @@ fn run_daemon_once( let history_refresh_job = if daemon_deadline_has_min_budget(deadline, DAEMON_MIN_REMAINING_FOR_JOB_SECS) { - run_daemon_history_refresh_job(data_root) + run_daemon_history_refresh_job(data_root, runtime) } else { Ok(daemon_history_refresh_skipped_job("daemon_deadline")) }; @@ -1248,11 +1267,12 @@ fn run_daemon_once( &history_refresh_job, )?; - let semantic_job = if daemon_deadline_has_min_budget(deadline, DAEMON_MIN_REMAINING_FOR_JOB_SECS) { - run_daemon_semantic_job(args, data_root, runtime, deadline, semantic_enabled) - } else { - Ok(daemon_semantic_deadline_skipped_job(data_root)) - }; + let semantic_job = + if daemon_deadline_has_min_budget(deadline, DAEMON_MIN_REMAINING_FOR_JOB_SECS) { + run_daemon_semantic_job(args, data_root, runtime, deadline, semantic_enabled) + } else { + Ok(daemon_semantic_deadline_skipped_job(data_root)) + }; let semantic_job = match semantic_job { Ok(value) => value, Err(error) => daemon_semantic_failed_job(data_root, format!("{error:#}")), @@ -1342,12 +1362,12 @@ fn daemon_deadline_has_min_budget(deadline: Option, min_secs: u64) -> b remaining >= StdDuration::from_secs(min_secs) } -fn run_daemon_history_refresh_job(data_root: &Path) -> Result { +fn run_daemon_history_refresh_job(data_root: &Path, runtime: &mut DaemonRuntime) -> Result { #[cfg(all(test, ctx_sqlite_vec))] if let Some(value) = daemon_test_job("history_refresh") { return Ok(value); } - + let _disk_io_pacing = runtime.history_refresh.install_daemon_disk_io_pacing(); let last_run_at_ms = utc_now().timestamp_millis(); let sources = search_refresh_sources(None); let plugin_sources = search_refresh_plugin_sources( @@ -1356,7 +1376,7 @@ fn run_daemon_history_refresh_job(data_root: &Path) -> Result { &crate::search_filters::SourceIdentityFilters::default(), )?; let source_count = sources.len().saturating_add(plugin_sources.len()); - if source_count == 0 { + if source_count == 0 && !search_refresh_has_publication_work(data_root)? { return Ok(daemon_history_refresh_job_json( "skipped", 0, @@ -1367,12 +1387,14 @@ fn run_daemon_history_refresh_job(data_root: &Path) -> Result { )); } let source_fingerprint = search_refresh_source_fingerprint(&sources); - let mut job = match refresh_sources_for_search( + let mut job = match refresh_sources_for_search_with_runtime( data_root, sources, plugin_sources, RefreshArg::Background, true, + crate::commands::import::ImportExecutionPolicy::Daemon, + &mut runtime.history_refresh, ) { Ok(totals) => daemon_history_refresh_job_json( "completed", @@ -1382,14 +1404,17 @@ fn run_daemon_history_refresh_job(data_root: &Path) -> Result { None, None, ), - Err(error) => daemon_history_refresh_job_json( - "failed", - source_count, - ImportTotals::default(), - last_run_at_ms, - None, - Some(error_summary(&error)), - ), + Err(error) => { + let totals = search_refresh_failure_totals(&error).unwrap_or_default(); + daemon_history_refresh_job_json( + "failed", + source_count, + totals, + last_run_at_ms, + None, + Some(error_summary(&error)), + ) + } }; if let Some(map) = job.as_object_mut() { map.insert("source_fingerprint".to_owned(), json!(source_fingerprint)); @@ -1443,25 +1468,22 @@ fn daemon_history_refresh_job_did_work(value: &Value) -> bool { let Some(totals) = value.get("totals") else { return false; }; - ["imported_sessions", "imported_events", "imported_edges"] - .into_iter() - .any(|key| totals.get(key).and_then(Value::as_u64).unwrap_or(0) > 0) -} - -fn search_refresh_source_fingerprint(sources: &[crate::provider_sources::SourceInfo]) -> String { - let mut items = sources - .iter() - .map(|source| { - format!( - "{}|{}|{}", - source.provider.as_str(), - source.source_format, - source.path.display() - ) - }) - .collect::>(); - items.sort(); - semantic_text_hash(&items.join("\n")) + if totals + .get("durable_progress") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return true; + } + [ + "imported_sessions", + "imported_events", + "imported_edges", + "fresh_units_processed", + "recovery_units_processed", + ] + .into_iter() + .any(|key| totals.get(key).and_then(Value::as_u64).unwrap_or(0) > 0) } fn run_daemon_semantic_job( @@ -1676,10 +1698,7 @@ fn daemon_semantic_requested_seconds(args: &DaemonRunArgs) -> u64 { }) } -fn semantic_daemon_model_load_needed( - report: &SemanticWorkerReport, - runtime_loaded: bool, -) -> bool { +fn semantic_daemon_model_load_needed(report: &SemanticWorkerReport, runtime_loaded: bool) -> bool { report.searchable_items > 0 && !runtime_loaded } @@ -1766,8 +1785,7 @@ fn daemon_semantic_model_load_deferred_job( ); value["retryable"] = Value::Bool(true); value["available_memory_bytes"] = json!(deferred.available_memory_bytes); - value["required_available_memory_bytes"] = - json!(deferred.required_available_memory_bytes); + value["required_available_memory_bytes"] = json!(deferred.required_available_memory_bytes); compact_json(value) } @@ -1814,9 +1832,9 @@ fn semantic_worker_report_for_daemon(data_root: &Path) -> SemanticWorkerReport { if db_path.exists() { match open_existing_store_read_only(&db_path, "ctx daemon status") { Ok(store) => { - return semantic_worker_report_cached(data_root, Some(&store)).unwrap_or_else(|error| { - SemanticWorkerReport::unavailable(data_root, format!("{error:#}")) - }); + return semantic_worker_report_cached(data_root, Some(&store)).unwrap_or_else( + |error| SemanticWorkerReport::unavailable(data_root, format!("{error:#}")), + ); } Err(error) => { return SemanticWorkerReport::unavailable(data_root, format!("{error:#}")); @@ -2313,3 +2331,39 @@ fn daemon_autostart_u64_env(name: &str, default: u64, max: u64) -> u64 { .map(|value| value.min(max)) .unwrap_or(default) } + +#[cfg(test)] +mod freshness_tests { + use super::*; + + #[test] + fn deferred_history_work_does_not_count_as_daemon_productivity() { + let deferred = daemon_history_refresh_job_json( + "completed", + 1, + ImportTotals { + fresh_units_pending: 1, + recovery_units_pending: 2, + ..ImportTotals::default() + }, + 1, + None, + None, + ); + assert!(!daemon_history_refresh_job_did_work(&deferred)); + + let completed = daemon_history_refresh_job_json( + "completed", + 1, + ImportTotals { + recovery_units_processed: 1, + recovery_units_pending: 1, + ..ImportTotals::default() + }, + 2, + None, + None, + ); + assert!(daemon_history_refresh_job_did_work(&completed)); + } +} diff --git a/crates/ctx-cli/src/semantic/paths_status.rs b/crates/ctx-cli/src/semantic/paths_status.rs index 3a5523fa..97f23b5a 100644 --- a/crates/ctx-cli/src/semantic/paths_status.rs +++ b/crates/ctx-cli/src/semantic/paths_status.rs @@ -395,14 +395,25 @@ fn unknown_process_lock_reports_running( } #[cfg(unix)] -fn lower_semantic_worker_priority() { +fn lower_daemon_worker_priority() { unsafe { let _ = libc::setpriority(libc::PRIO_PROCESS, 0, 10); } } -#[cfg(not(unix))] -fn lower_semantic_worker_priority() {} +#[cfg(windows)] +fn lower_daemon_worker_priority() { + use windows_sys::Win32::System::Threading::{ + GetCurrentProcess, SetPriorityClass, BELOW_NORMAL_PRIORITY_CLASS, + }; + + unsafe { + let _ = SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS); + } +} + +#[cfg(not(any(unix, windows)))] +fn lower_daemon_worker_priority() {} fn write_private_json_file(path: &Path, value: &Value) -> Result<()> { if let Some(parent) = path.parent() { diff --git a/crates/ctx-cli/src/semantic/preamble.rs b/crates/ctx-cli/src/semantic/preamble.rs index f646b99d..f2097d09 100644 --- a/crates/ctx-cli/src/semantic/preamble.rs +++ b/crates/ctx-cli/src/semantic/preamble.rs @@ -41,8 +41,9 @@ use ctx_history_store::{EventEmbeddingDocument, Store}; use crate::commands::{ import::{error_summary, import_totals_json, ImportTotals}, search::{ - refresh_sources_for_search, search_refresh_plugin_sources, search_refresh_sources, - RefreshArg, + refresh_sources_for_search_with_runtime, search_refresh_has_publication_work, + search_refresh_failure_totals, search_refresh_plugin_sources, + search_refresh_source_fingerprint, search_refresh_sources, RefreshArg, }, }; use crate::config::{self, AppConfig, CONFIG_FILE}; diff --git a/crates/ctx-cli/tests/native_provider_rejections.rs b/crates/ctx-cli/tests/native_provider_rejections.rs index 1d190ac2..9d403553 100644 --- a/crates/ctx-cli/tests/native_provider_rejections.rs +++ b/crates/ctx-cli/tests/native_provider_rejections.rs @@ -49,7 +49,7 @@ fn antigravity_cli_import_skips_malformed_file_among_valid_files() { } #[test] -fn mixed_source_replay_remains_completed_with_rejections() { +fn unchanged_mixed_source_resume_is_a_noop() { let temp = tempdir(); let brain = write_antigravity_valid_and_malformed_file_tree(&temp); let path = brain.to_str().unwrap(); @@ -77,18 +77,18 @@ fn mixed_source_replay_remains_completed_with_rejections() { "--progress", "none", ])); - assert_eq!(replay["outcome"], "completed_with_rejections", "{replay:#}"); - assert_eq!(replay["failure_scope"], "record", "{replay:#}"); + assert_eq!(replay["outcome"], "success", "{replay:#}"); + assert_eq!(replay["failure_scope"], "none", "{replay:#}"); assert_eq!(replay["totals"]["failed_sources"], 0, "{replay:#}"); - assert_eq!(replay["totals"]["rejected_records"], 1, "{replay:#}"); - assert_eq!( - replay["sources"][0]["status"], "completed_with_rejections", + assert_eq!(replay["totals"]["rejected_records"], 0, "{replay:#}"); + assert!( + replay["sources"].as_array().unwrap().is_empty(), "{replay:#}" ); } #[test] -fn codex_mixed_session_replay_remains_completed_with_rejections() { +fn unchanged_codex_mixed_session_resume_is_a_noop() { let temp = tempdir(); let session = temp.path().join("codex-mixed-replay.jsonl"); fs::write( @@ -105,29 +105,46 @@ fn codex_mixed_session_replay_remains_completed_with_rejections() { .unwrap(); let path = session.to_str().unwrap(); - for resume in [false, true] { - let mut command = ctx(&temp); - command.args([ - "import", - "--provider", - "codex", - "--path", - path, - "--json", - "--progress", - "none", - ]); - if resume { - command.arg("--resume"); - } - let report = json_output(&mut command); - assert_eq!( - report["outcome"], "completed_with_rejections", - "resume={resume}: {report:#}" - ); - assert_eq!(report["totals"]["failed_sources"], 0, "{report:#}"); - assert_eq!(report["totals"]["rejected_records"], 1, "{report:#}"); - } + let first = json_output(ctx(&temp).args([ + "import", + "--provider", + "codex", + "--path", + path, + "--json", + "--progress", + "none", + ])); + assert_eq!(first["outcome"], "completed_with_rejections", "{first:#}"); + assert_eq!(first["totals"]["failed_sources"], 0, "{first:#}"); + assert_eq!(first["totals"]["rejected_records"], 1, "{first:#}"); + + let replay = json_output(ctx(&temp).args([ + "import", + "--provider", + "codex", + "--path", + path, + "--resume", + "--json", + "--progress", + "none", + ])); + assert_eq!(replay["outcome"], "success", "{replay:#}"); + assert_eq!(replay["failure_scope"], "none", "{replay:#}"); + assert_eq!(replay["totals"]["failed_sources"], 0, "{replay:#}"); + assert_eq!(replay["totals"]["rejected_records"], 0, "{replay:#}"); + assert!( + replay["sources"].as_array().unwrap().is_empty(), + "{replay:#}" + ); + + let status = json_output(ctx(&temp).args(["status", "--json"])); + assert_eq!(status["pending_source_import_files"], 0, "{status:#}"); + assert_eq!( + status["completed_with_rejections_source_import_files"], 1, + "{status:#}" + ); let search = json_output(ctx(&temp).args([ "search", @@ -248,15 +265,16 @@ fn codex_append_after_mixed_tail_preserves_the_earlier_safe_checkpoint() { "{first_mixed:#}" ); let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); - let checkpoint_after_rejection: (i64, i64) = conn + let checkpoint_after_rejection: (i64, i64, i64) = conn .query_row( - "SELECT last_imported_file_size_bytes, indexed_file_size_bytes FROM catalog_sessions WHERE source_path = ?1", + "SELECT catalog.last_imported_file_size_bytes, catalog.indexed_file_size_bytes, checkpoint.committed_byte_offset FROM catalog_sessions AS catalog JOIN provider_file_checkpoints AS checkpoint ON checkpoint.provider = catalog.provider AND checkpoint.source_format = catalog.source_format AND checkpoint.source_root = catalog.source_root AND checkpoint.source_path = catalog.source_path WHERE catalog.source_path = ?1", [session.to_str().unwrap()], - |row| Ok((row.get(0)?, row.get(1)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .unwrap(); assert_eq!(checkpoint_after_rejection.0, initial.len() as i64); assert_eq!(checkpoint_after_rejection.1, mixed.len() as i64); + assert_eq!(checkpoint_after_rejection.2, initial.len() as i64); drop(conn); let accepted = r#"{"timestamp":"2026-07-13T12:00:03.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"mixed append accepted after rejection"}]}}"#; @@ -268,15 +286,16 @@ fn codex_append_after_mixed_tail_preserves_the_earlier_safe_checkpoint() { "{appended_result:#}" ); let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); - let checkpoint_after_append: (i64, i64) = conn + let checkpoint_after_append: (Option, i64, i64) = conn .query_row( - "SELECT last_imported_file_size_bytes, indexed_file_size_bytes FROM catalog_sessions WHERE source_path = ?1", + "SELECT catalog.last_imported_file_size_bytes, catalog.indexed_file_size_bytes, checkpoint.committed_byte_offset FROM catalog_sessions AS catalog JOIN provider_file_checkpoints AS checkpoint ON checkpoint.provider = catalog.provider AND checkpoint.source_format = catalog.source_format AND checkpoint.source_root = catalog.source_root AND checkpoint.source_path = catalog.source_path WHERE catalog.source_path = ?1", [session.to_str().unwrap()], - |row| Ok((row.get(0)?, row.get(1)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .unwrap(); - assert_eq!(checkpoint_after_append.0, initial.len() as i64); + assert_eq!(checkpoint_after_append.0, Some(initial.len() as i64)); assert_eq!(checkpoint_after_append.1, appended.len() as i64); + assert_eq!(checkpoint_after_append.2, initial.len() as i64); drop(conn); let unchanged = json_output(ctx(&temp).args(args)); @@ -539,7 +558,7 @@ fn manifested_retry_retaining_existing_content_reports_completed_with_rejections assert_eq!(first["outcome"], "success", "{first:#}"); assert_eq!(first["totals"]["imported_events"], 1, "{first:#}"); - fs::write(&session, "{\"type\":\n").unwrap(); + fs::write(&session, format!(" {valid}\n{{\"type\":\n")).unwrap(); let retained = json_output(ctx(&temp).args(args)); assert_eq!( retained["outcome"], "completed_with_rejections", @@ -556,6 +575,16 @@ fn manifested_retry_retaining_existing_content_reports_completed_with_rejections assert_eq!(retained["totals"]["imported_sessions"], 0); assert_eq!(retained["totals"]["imported_events"], 0); assert_eq!(retained["totals"]["imported_edges"], 0); + let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); + let pending_publications: i64 = conn + .query_row( + "SELECT COUNT(*) FROM provider_file_publications", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(pending_publications, 0); + drop(conn); let search = json_output(ctx(&temp).args([ "search", @@ -728,6 +757,51 @@ fn all_invalid_source_reports_failure_json_and_exits_nonzero() { assert_search_provider_oracle(&search, "antigravity", "write_to_file", 1, "tool_call"); } +#[test] +fn multi_slice_all_invalid_source_reports_one_source_failure() { + const FILE_COUNT: usize = 65; + + let temp = tempdir(); + let brain = temp.path().join("brain"); + for index in 0..FILE_COUNT { + let logs = brain + .join(format!("agy-bad-{index:03}")) + .join(".system_generated") + .join("logs"); + fs::create_dir_all(&logs).unwrap(); + fs::write(logs.join("transcript_full.jsonl"), "{\"step_index\":\n").unwrap(); + } + + let output = ctx(&temp) + .args([ + "import", + "--provider", + "antigravity", + "--path", + brain.to_str().unwrap(), + "--json", + "--progress", + "none", + ]) + .assert() + .failure() + .get_output() + .clone(); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + let sources = report["sources"].as_array().unwrap(); + + assert_eq!(report["outcome"], "failure", "{report:#}"); + assert_eq!(report["totals"]["failed_sources"], 1, "{report:#}"); + assert_eq!( + report["totals"]["rejected_records"], FILE_COUNT, + "{report:#}" + ); + assert_eq!(sources.len(), 1, "{report:#}"); + assert_eq!(sources[0]["source_files"], FILE_COUNT, "{report:#}"); + assert_eq!(sources[0]["rejected_records"], FILE_COUNT, "{report:#}"); + assert_eq!(sources[0]["rejections"].as_array().unwrap().len(), 5); +} + #[test] fn all_rejected_codex_session_leaves_no_import_scaffolding() { let temp = tempdir(); diff --git a/crates/ctx-cli/tests/search_refresh.rs b/crates/ctx-cli/tests/search_refresh.rs index affe05b8..309f2040 100644 --- a/crates/ctx-cli/tests/search_refresh.rs +++ b/crates/ctx-cli/tests/search_refresh.rs @@ -480,7 +480,7 @@ sys.exit(23) ); assert!( - stderr.contains("search refresh failed and no existing ctx index is available"), + stderr.contains("search refresh failed and no usable existing ctx index is available"), "{stderr}" ); assert!( @@ -513,17 +513,10 @@ fn search_refresh_auto_all_malformed_native_history_fails_instead_of_serving_emp ])); assert!( - stderr.contains("search refresh failed and no existing ctx index is available"), - "{stderr}" - ); - assert!( - stderr.contains("all search refresh sources failed; first failure: import codex source"), - "{stderr}" - ); - assert!( - stderr.contains("codex session JSONL contained no real message content"), + stderr.contains("background search refresh has not produced a usable ctx index yet"), "{stderr}" ); + assert!(stderr.contains("retry with `--refresh wait`"), "{stderr}"); } #[test] @@ -857,7 +850,8 @@ fn search_refresh_auto_imports_discovered_top_provider_sources() { search["freshness"]["totals"]["imported_sessions"] .as_u64() .unwrap() - >= 1 + >= 1, + "{cli_provider} first refresh imported no sessions: {search:#}" ); assert_search_provider_oracle(&search, stored_provider, &query, 1, "message"); diff --git a/crates/ctx-cli/tests/search_show_locate_sql.rs b/crates/ctx-cli/tests/search_show_locate_sql.rs index 67f38a22..577655fd 100644 --- a/crates/ctx-cli/tests/search_show_locate_sql.rs +++ b/crates/ctx-cli/tests/search_show_locate_sql.rs @@ -716,7 +716,7 @@ fn doctor_reports_missing_store_without_creating_it() { } #[test] -fn codex_cli_resume_is_idempotent_rescan_and_filters_subagents() { +fn codex_cli_resume_is_noop_for_healthy_sources_and_filters_subagents() { let temp = tempdir(); let fixture = provider_history_fixture("codex-sessions"); @@ -805,9 +805,8 @@ fn codex_cli_resume_is_idempotent_rescan_and_filters_subagents() { assert_eq!(second["totals"]["imported_sessions"], 0); assert_eq!(second["totals"]["imported_events"], 0); assert_eq!(second["totals"]["imported_edges"], 0); - assert!(second["totals"]["skipped"].as_u64().unwrap() > 0); - assert_eq!(second["sources"][0]["imported_sessions"], 0); - assert_eq!(second["sources"][0]["imported_events"], 0); + assert_eq!(second["totals"]["skipped"], 0); + assert!(second["sources"].as_array().unwrap().is_empty()); } #[test] @@ -994,7 +993,8 @@ fn pi_cli_import_search_flow() { assert_eq!(second["resume_mode"], "idempotent_rescan"); assert_eq!(second["totals"]["imported_sessions"], 0); assert_eq!(second["totals"]["imported_events"], 0); - assert_eq!(second["totals"]["skipped"].as_u64().unwrap(), 7); + assert_eq!(second["totals"]["skipped"], 0); + assert!(second["sources"].as_array().unwrap().is_empty()); let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); assert_eq!( diff --git a/crates/ctx-cli/tests/setup_sources_import.rs b/crates/ctx-cli/tests/setup_sources_import.rs index 97f088f6..9a54ff4d 100644 --- a/crates/ctx-cli/tests/setup_sources_import.rs +++ b/crates/ctx-cli/tests/setup_sources_import.rs @@ -480,6 +480,12 @@ fn setup_backgrounds_discovered_codex_sessions_when_daemon_is_enabled_and_wait_i assert_eq!(ready["inventory"]["codex_catalog_sessions"], 1); assert_eq!(ready["catalog"]["cataloged_sessions"], 1); assert_eq!(ready["import"]["ran"], true); + assert_eq!(ready["import"]["totals"]["durable_progress"], true); + assert_eq!(ready["import"]["totals"]["fresh_units_pending_exact"], true); + assert_eq!( + ready["import"]["totals"]["recovery_units_pending_exact"], + true + ); assert_eq!(ready["import"]["totals"]["failed_sources"], 0); assert_eq!(ready["import"]["totals"]["imported_sessions"], 1); assert!( @@ -513,6 +519,60 @@ fn setup_backgrounds_discovered_codex_sessions_when_daemon_is_enabled_and_wait_i assert!(human_setup.contains(" ctx search \"test failure\"")); } +#[test] +fn setup_json_reports_partial_when_foreground_import_leaves_incomplete_history() { + let temp = tempdir(); + install_default_pi_fixture(&temp, "pi baseline setup content"); + + let baseline = + json_output(ctx(&temp).args(["setup", "--wait", "--json", "--progress", "none"])); + assert_eq!(baseline["mode"], "ready", "{baseline:#}"); + + let source_root = temp.path().join(".pi/agent/sessions/--workspace--"); + let incomplete_path = source_root.join("2026-06-24T12-00-00-000Z_pi-default-refresh.jsonl"); + fs::OpenOptions::new() + .append(true) + .open(incomplete_path) + .unwrap() + .write_all(br#"{"type":"message","id":"partial""#) + .unwrap(); + write_pi_session_jsonl( + &source_root.join("2026-06-25T12-00-00-000Z_pi-complete-later.jsonl"), + "pi-complete-later", + "pi complete later setup content", + ); + + let setup = json_output(ctx(&temp).args(["setup", "--wait", "--json", "--progress", "none"])); + assert_eq!(setup["mode"], "partial", "{setup:#}"); + assert_eq!(setup["import"]["ran"], true, "{setup:#}"); + assert_eq!(setup["import"]["totals"]["durable_progress"], true); + assert_eq!(setup["import"]["totals"]["fresh_units_processed"], 1); + assert_eq!(setup["import"]["totals"]["fresh_units_pending"], 1); + assert_eq!(setup["import"]["totals"]["fresh_units_pending_exact"], true); + assert_eq!(setup["import"]["totals"]["recovery_units_pending"], 0); + assert_eq!( + setup["import"]["totals"]["recovery_units_pending_exact"], + true + ); + + let search = json_output(ctx(&temp).args([ + "search", + "pi complete later setup content", + "--provider", + "pi", + "--refresh", + "off", + "--json", + ])); + assert_search_provider_oracle( + &search, + "pi", + "pi complete later setup content", + 1, + "message", + ); +} + #[test] fn setup_import_isolates_empty_codex_session_file() { let temp = tempdir(); diff --git a/crates/ctx-cli/tests/setup_sources_import/source_and_import_tests.rs b/crates/ctx-cli/tests/setup_sources_import/source_and_import_tests.rs index 1785987c..801e9166 100644 --- a/crates/ctx-cli/tests/setup_sources_import/source_and_import_tests.rs +++ b/crates/ctx-cli/tests/setup_sources_import/source_and_import_tests.rs @@ -91,6 +91,13 @@ fn import_progress_json_goes_to_stderr_without_polluting_stdout() { let stdout: Value = serde_json::from_slice(&output.stdout).unwrap(); assert_eq!(stdout["schema_version"], 2); + assert_eq!(stdout["totals"]["durable_progress"], true); + assert!(stdout["totals"]["fresh_units_processed"].as_u64().unwrap() > 0); + assert_eq!(stdout["totals"]["recovery_units_processed"], 0); + assert_eq!(stdout["totals"]["fresh_units_pending"], 0); + assert_eq!(stdout["totals"]["fresh_units_pending_exact"], true); + assert_eq!(stdout["totals"]["recovery_units_pending"], 0); + assert_eq!(stdout["totals"]["recovery_units_pending_exact"], true); assert!(stdout["totals"]["imported_sessions"].as_u64().unwrap() > 0); let stderr = String::from_utf8(output.stderr).unwrap(); diff --git a/crates/ctx-history-capture/src/common/io.rs b/crates/ctx-history-capture/src/common/io.rs index 5386c6cb..0db87d13 100644 --- a/crates/ctx-history-capture/src/common/io.rs +++ b/crates/ctx-history-capture/src/common/io.rs @@ -1,6 +1,6 @@ use std::{ fs::{self, File}, - io::{self, BufRead, Read}, + io::{self, BufRead, Read, Seek, SeekFrom}, path::{Path, PathBuf}, }; @@ -471,7 +471,12 @@ pub(crate) fn ensure_provider_path_parents_are_not_symlinks(path: &Path) -> Resu pub(crate) fn read_text_file_limited(path: &Path, max_bytes: usize, label: &str) -> Result { let file = File::open(path)?; - let mut reader = file.take((max_bytes as u64).saturating_add(1)); + read_text_limited(file, max_bytes, label) +} + +fn read_text_limited(reader: impl Read, max_bytes: usize, label: &str) -> Result { + let reader = crate::disk_io_pacing::PacedReader::new(reader); + let mut reader = reader.take((max_bytes as u64).saturating_add(1)); let mut bytes = Vec::new(); reader.read_to_end(&mut bytes)?; if bytes.len() > max_bytes { @@ -483,6 +488,38 @@ pub(crate) fn read_text_file_limited(path: &Path, max_bytes: usize, label: &str) .map_err(|err| CaptureError::InvalidPayload(format!("{label} is not valid UTF-8: {err}"))) } +pub fn provider_jsonl_range_has_complete_line( + path: &Path, + offset: u64, + observed_size: u64, +) -> Result { + let mut file = open_regular_provider_transcript_file(path)?; + file.seek(SeekFrom::Start(offset))?; + let mut remaining = observed_size.saturating_sub(offset); + let mut scanned = 0usize; + let scan_limit = MAX_PROVIDER_JSONL_LINE_BYTES.saturating_add(1); + let mut buffer = [0u8; 8 * 1024]; + while remaining > 0 && scanned < scan_limit { + let budget = scan_limit.saturating_sub(scanned); + let limit = usize::try_from(remaining.min(buffer.len() as u64)) + .unwrap_or(buffer.len()) + .min(budget); + let read = file.read(&mut buffer[..limit])?; + if read == 0 { + return Ok(false); + } + if buffer[..read].contains(&b'\n') { + return Ok(true); + } + scanned = scanned.saturating_add(read); + remaining = remaining.saturating_sub(read as u64); + } + if remaining > 0 || scanned > MAX_PROVIDER_JSONL_LINE_BYTES { + return Err(provider_jsonl_line_too_large()); + } + Ok(false) +} + pub(crate) fn read_provider_jsonl_line( reader: &mut impl BufRead, buffer: &mut Vec, @@ -570,6 +607,9 @@ pub(crate) fn read_provider_jsonl_line_or_skip_oversized( reader.consume(bytes_to_consume); let (discarded, newline_terminated) = discard_provider_jsonl_line(reader)?; buffer.clear(); + if !newline_terminated { + return Err(provider_jsonl_line_too_large()); + } return Ok(ProviderJsonlLineRead::Oversized { bytes: total .saturating_add(bytes_to_consume) @@ -585,17 +625,19 @@ pub(crate) fn read_provider_jsonl_line_or_skip_oversized( pub(crate) fn discard_provider_jsonl_line(reader: &mut impl BufRead) -> Result<(usize, bool)> { let mut discarded = 0usize; - loop { + while discarded < MAX_PROVIDER_JSONL_LINE_BYTES { let available = reader.fill_buf()?; if available.is_empty() { return Ok((discarded, false)); } - let bytes_to_consume = available + let remaining = MAX_PROVIDER_JSONL_LINE_BYTES.saturating_sub(discarded); + let bounded = &available[..available.len().min(remaining)]; + let bytes_to_consume = bounded .iter() .position(|byte| *byte == b'\n') .map(|index| index + 1) - .unwrap_or(available.len()); - let found_newline = available + .unwrap_or(bounded.len()); + let found_newline = bounded .get(bytes_to_consume.saturating_sub(1)) .is_some_and(|byte| *byte == b'\n'); reader.consume(bytes_to_consume); @@ -604,6 +646,7 @@ pub(crate) fn discard_provider_jsonl_line(reader: &mut impl BufRead) -> Result<( return Ok((discarded, true)); } } + Ok((discarded, false)) } pub(crate) fn provider_jsonl_line_too_large() -> CaptureError { @@ -620,6 +663,93 @@ pub(crate) fn read_json_file_limited(path: &Path, max_bytes: usize, label: &str) #[cfg(test)] mod tests { use super::*; + use std::io::BufReader; + + use crate::{install_disk_io_pacer, DiskIoPacer}; + + struct ReservationObservedReader { + pacer: DiskIoPacer, + expected_reserved_bytes: u64, + } + + impl Read for ReservationObservedReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + assert_eq!(self.pacer.charged_bytes(), self.expected_reserved_bytes); + buffer.fill(b'x'); + Ok(buffer.len()) + } + } + + #[test] + fn limited_text_reads_reserve_shared_budget_before_physical_reads() { + let pacer = DiskIoPacer::new(u64::MAX, u64::MAX); + let _pacing = install_disk_io_pacer(pacer.clone()); + + for expected_reserved_bytes in [4, 8] { + let error = read_text_limited( + ReservationObservedReader { + pacer: pacer.clone(), + expected_reserved_bytes, + }, + 3, + "provider fixture", + ) + .expect_err("the max-plus-one byte must preserve the size error"); + + assert!(matches!(error, CaptureError::InvalidPayload(_))); + } + + assert_eq!(pacer.charged_bytes(), 8); + } + + #[test] + fn oversized_line_discard_is_bounded_without_a_newline() { + let source = std::io::Read::take( + std::io::repeat(b'x'), + (MAX_PROVIDER_JSONL_LINE_BYTES as u64).saturating_mul(4), + ); + let mut reader = BufReader::with_capacity(8 * 1024, source); + + let (discarded, newline_terminated) = discard_provider_jsonl_line(&mut reader).unwrap(); + + assert_eq!(discarded, MAX_PROVIDER_JSONL_LINE_BYTES); + assert!(!newline_terminated); + } + + #[test] + fn oversized_unterminated_line_stops_before_its_tail_can_be_reframed() { + let source = std::io::Read::take( + std::io::repeat(b'x'), + (MAX_PROVIDER_JSONL_LINE_BYTES as u64).saturating_mul(4), + ); + let mut reader = BufReader::with_capacity(8 * 1024, source); + let mut buffer = Vec::new(); + + let error = read_provider_jsonl_line_or_skip_oversized(&mut reader, &mut buffer) + .expect_err("unterminated oversized records must stop parsing"); + + assert!(matches!(error, CaptureError::InvalidPayload(_))); + assert!(buffer.is_empty()); + } + + #[cfg(unix)] + #[test] + fn append_range_probe_rejects_fifo_without_blocking() { + use std::{ffi::CString, os::unix::ffi::OsStrExt}; + + let temp = tempfile::tempdir().expect("tempdir"); + let fifo = temp.path().join("transcript.jsonl"); + let fifo_c = CString::new(fifo.as_os_str().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(fifo_c.as_ptr(), 0o600) }, 0); + + let error = provider_jsonl_range_has_complete_line(&fifo, 0, 1) + .expect_err("provider FIFOs must be rejected"); + + assert!(matches!( + error, + CaptureError::InvalidProviderTranscriptPath { .. } + )); + } #[test] fn explicit_jsonl_file_does_not_require_an_extension() { diff --git a/crates/ctx-history-capture/src/disk_io_pacing.rs b/crates/ctx-history-capture/src/disk_io_pacing.rs new file mode 100644 index 00000000..e8e2b2a9 --- /dev/null +++ b/crates/ctx-history-capture/src/disk_io_pacing.rs @@ -0,0 +1,374 @@ +use std::{ + cell::RefCell, + fmt, + io::{self, Read, Seek, SeekFrom}, + marker::PhantomData, + rc::Rc, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +const NANOS_PER_SECOND: u128 = 1_000_000_000; +const MAX_SLEEP: Duration = Duration::from_millis(25); + +type Clock = Arc Duration + Send + Sync>; +type Sleeper = Arc; + +#[derive(Clone)] +pub struct DiskIoPacer { + bytes_per_second: u64, + burst_bytes: u64, + burst_nanos: u128, + state: Arc>, + clock: Clock, + sleeper: Sleeper, +} + +struct PacerState { + last_nanos: u128, + available_nanos: u128, + charged_bytes: u64, +} + +impl fmt::Debug for DiskIoPacer { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DiskIoPacer") + .field("bytes_per_second", &self.bytes_per_second) + .field("burst_bytes", &self.burst_bytes) + .finish_non_exhaustive() + } +} + +impl DiskIoPacer { + pub fn new(bytes_per_second: u64, burst_bytes: u64) -> Self { + let started = Instant::now(); + Self::with_runtime( + bytes_per_second, + burst_bytes, + Arc::new(move || started.elapsed()), + Arc::new(std::thread::sleep), + ) + } + + fn with_runtime( + bytes_per_second: u64, + burst_bytes: u64, + clock: Clock, + sleeper: Sleeper, + ) -> Self { + let bytes_per_second = bytes_per_second.max(1); + let burst_bytes = burst_bytes.max(1); + let burst_nanos = byte_cost_nanos(burst_bytes, bytes_per_second); + let now = clock().as_nanos(); + Self { + bytes_per_second, + burst_bytes, + burst_nanos, + state: Arc::new(Mutex::new(PacerState { + last_nanos: now, + available_nanos: burst_nanos, + charged_bytes: 0, + })), + clock, + sleeper, + } + } + + pub fn bytes_per_second(&self) -> u64 { + self.bytes_per_second + } + + pub fn burst_bytes(&self) -> u64 { + self.burst_bytes + } + + pub fn pace(&self, bytes: u64) { + if bytes == 0 { + return; + } + let mut remaining_nanos = byte_cost_nanos(bytes, self.bytes_per_second); + { + let mut state = self.lock_state(); + state.charged_bytes = state.charged_bytes.saturating_add(bytes); + } + while remaining_nanos > 0 { + let sleep = { + let mut state = self.lock_state(); + let now = (self.clock)().as_nanos(); + let replenished = state + .available_nanos + .saturating_add(now.saturating_sub(state.last_nanos)); + state.available_nanos = replenished.min(self.burst_nanos); + state.last_nanos = now; + let paid = remaining_nanos.min(state.available_nanos); + state.available_nanos -= paid; + remaining_nanos -= paid; + duration_from_nanos( + remaining_nanos + .min(MAX_SLEEP.as_nanos()) + .min(self.burst_nanos), + ) + }; + if !sleep.is_zero() { + (self.sleeper)(sleep); + } + } + } + + #[cfg(test)] + pub(crate) fn charged_bytes(&self) -> u64 { + self.lock_state().charged_bytes + } + + fn lock_state(&self) -> std::sync::MutexGuard<'_, PacerState> { + self.state.lock().unwrap_or_else(|error| error.into_inner()) + } +} + +fn byte_cost_nanos(bytes: u64, bytes_per_second: u64) -> u128 { + u128::from(bytes) + .saturating_mul(NANOS_PER_SECOND) + .saturating_add(u128::from(bytes_per_second) - 1) + / u128::from(bytes_per_second) +} + +fn duration_from_nanos(nanos: u128) -> Duration { + Duration::from_nanos(nanos.min(u128::from(u64::MAX)) as u64) +} + +thread_local! { + static CURRENT_PACER: RefCell> = const { RefCell::new(None) }; +} + +pub struct DiskIoPacingGuard { + previous: Option, + _store_pacing: ctx_history_store::EventSearchMaintenancePacingGuard, + _not_send: PhantomData>, +} + +impl Drop for DiskIoPacingGuard { + fn drop(&mut self) { + CURRENT_PACER.with(|slot| { + *slot.borrow_mut() = self.previous.take(); + }); + } +} + +pub fn install_disk_io_pacer(pacer: DiskIoPacer) -> DiskIoPacingGuard { + let previous = CURRENT_PACER.with(|slot| slot.borrow_mut().replace(pacer)); + let store_pacing = + ctx_history_store::install_event_search_maintenance_pacer(pace_current_disk_io); + DiskIoPacingGuard { + previous, + _store_pacing: store_pacing, + _not_send: PhantomData, + } +} + +pub fn pace_current_disk_io(bytes: u64) { + let pacer = CURRENT_PACER.with(|slot| slot.borrow().clone()); + if let Some(pacer) = pacer { + pacer.pace(bytes); + } +} + +pub(crate) fn current_disk_io_pacer() -> Option { + CURRENT_PACER.with(|slot| slot.borrow().clone()) +} + +pub(crate) fn current_disk_io_burst_bytes() -> Option { + current_disk_io_pacer().map(|pacer| pacer.burst_bytes()) +} + +#[derive(Debug)] +pub(crate) struct PacedReader { + inner: R, +} + +impl PacedReader { + pub(crate) fn new(inner: R) -> Self { + Self { inner } + } + + pub(crate) fn get_ref(&self) -> &R { + &self.inner + } + + pub(crate) fn get_mut(&mut self) -> &mut R { + &mut self.inner + } +} + +impl Read for PacedReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + pace_current_disk_io(buffer.len() as u64); + self.inner.read(buffer) + } +} + +impl Seek for PacedReader { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.inner.seek(position) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[derive(Default)] + struct FakeTime { + now: Mutex, + sleeps: Mutex>, + } + + fn fake_pacer(rate: u64, burst: u64) -> (DiskIoPacer, Arc) { + let time = Arc::new(FakeTime::default()); + let clock_time = Arc::clone(&time); + let sleep_time = Arc::clone(&time); + let pacer = DiskIoPacer::with_runtime( + rate, + burst, + Arc::new(move || *clock_time.now.lock().unwrap()), + Arc::new(move |duration| { + sleep_time.sleeps.lock().unwrap().push(duration); + *sleep_time.now.lock().unwrap() += duration; + }), + ); + (pacer, time) + } + + #[test] + fn oversized_charge_progresses_with_small_sleeps() { + let (pacer, time) = fake_pacer(1_000, 100); + pacer.pace(1_100); + + let sleeps = time.sleeps.lock().unwrap(); + assert_eq!( + sleeps.iter().copied().sum::(), + Duration::from_secs(1) + ); + assert!(sleeps.iter().all(|sleep| *sleep <= MAX_SLEEP)); + assert_eq!(pacer.charged_bytes(), 1_100); + } + + #[test] + fn idle_credit_is_capped_at_the_burst() { + let (pacer, time) = fake_pacer(1_000, 100); + pacer.pace(100); + *time.now.lock().unwrap() += Duration::from_secs(10); + pacer.pace(150); + + assert_eq!( + time.sleeps + .lock() + .unwrap() + .iter() + .copied() + .sum::(), + Duration::from_millis(50) + ); + } + + #[test] + fn sleep_quantum_does_not_discard_a_small_burst() { + let (pacer, time) = fake_pacer(1_000, 1); + pacer.pace(101); + + let sleeps = time.sleeps.lock().unwrap(); + assert_eq!( + sleeps.iter().copied().sum::(), + Duration::from_millis(100) + ); + assert!(sleeps + .iter() + .all(|sleep| *sleep <= Duration::from_millis(1))); + } + + #[test] + fn scoped_pacer_restores_the_previous_accountant() { + let (outer, _) = fake_pacer(1_000, 1_000); + let (inner, _) = fake_pacer(1_000, 1_000); + let _outer_scope = install_disk_io_pacer(outer.clone()); + pace_current_disk_io(10); + { + let _inner_scope = install_disk_io_pacer(inner.clone()); + pace_current_disk_io(20); + } + pace_current_disk_io(30); + + assert_eq!(outer.charged_bytes(), 40); + assert_eq!(inner.charged_bytes(), 20); + } + + #[test] + fn paced_reader_reserves_budget_before_physical_read() { + struct ObservedRead { + time: Arc, + observed_nanos: Arc, + } + + impl Read for ObservedRead { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let now = self.time.now.lock().unwrap().as_nanos() as u64; + self.observed_nanos.store(now, Ordering::SeqCst); + buffer.fill(b'x'); + Ok(buffer.len()) + } + } + + let (pacer, time) = fake_pacer(1_000, 1); + let _pacing = install_disk_io_pacer(pacer); + let observed_nanos = Arc::new(AtomicU64::new(0)); + let mut reader = PacedReader::new(ObservedRead { + time, + observed_nanos: Arc::clone(&observed_nanos), + }); + reader.read_exact(&mut [0_u8; 101]).unwrap(); + + assert_eq!(observed_nanos.load(Ordering::SeqCst), 100_000_000); + } + + #[test] + fn inherited_worker_pacer_shares_the_aggregate_budget() { + let (pacer, time) = fake_pacer(1_000, 100); + let _pacing = install_disk_io_pacer(pacer.clone()); + pace_current_disk_io(100); + let inherited = current_disk_io_pacer().unwrap(); + std::thread::spawn(move || { + let _pacing = install_disk_io_pacer(inherited); + pace_current_disk_io(100); + }) + .join() + .unwrap(); + + assert_eq!(pacer.charged_bytes(), 200); + assert_eq!(*time.now.lock().unwrap(), Duration::from_millis(100)); + } + + #[test] + fn installed_pacer_accounts_for_event_search_maintenance() { + let temp = tempfile::tempdir().unwrap(); + let (pacer, time) = fake_pacer(1024 * 1024, 1); + let _pacing = install_disk_io_pacer(pacer.clone()); + let store = ctx_history_store::Store::open(temp.path().join("work.sqlite")).unwrap(); + let bulk_guard = store.begin_event_search_bulk_mode().unwrap(); + + assert_eq!(pacer.charged_bytes(), 0); + assert!(store + .finish_event_search_bulk_mode(&bulk_guard) + .unwrap() + .is_complete()); + let charged = pacer.charged_bytes(); + + assert!(charged > 0); + assert!(!time.now.lock().unwrap().is_zero()); + assert!(store + .finish_event_search_bulk_mode(&bulk_guard) + .unwrap() + .is_complete()); + assert_eq!(pacer.charged_bytes(), charged); + } +} diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 6100c7b6..ce12f79e 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -1,12 +1,14 @@ pub mod provider_sources; pub use provider_sources::{ discover_provider_sources, discover_provider_sources_for_provider, - import_append_capable_provider_file, open_provider_jsonl, + import_append_capable_provider_file, observe_ordinary_file, open_provider_jsonl, provider_canonical_material_source_format, provider_file_mutation_contract, - provider_source_for_path, provider_source_spec, provider_source_specs, + provider_jsonl_checkpoint_matches_file, provider_source_for_path, + provider_source_for_persisted_format, provider_source_spec, provider_source_specs, ClaudeProjectsJsonlResumeState, CodexSessionJsonlResumeState, CodexToolCallResumeContext, - ProviderAdmittedJsonlAppendCheckpoint, ProviderAppendFileImportDecision, - ProviderAppendFileImportMode, ProviderAppendFileImportOptions, ProviderAppendFileImportResult, + OrdinaryFileObservation, ProviderAdmittedJsonlAppendCheckpoint, + ProviderAppendFileImportDecision, ProviderAppendFileImportMode, + ProviderAppendFileImportOptions, ProviderAppendFileImportResult, ProviderAppendFileImportWithoutCheckpoint, ProviderCatalogSupport, ProviderDefaultLocation, ProviderFileMutationContract, ProviderFileStableIdentity, ProviderImportDependency, ProviderImportSupport, ProviderImportUnitGrouping, ProviderImportUnitOwner, @@ -16,8 +18,14 @@ pub use provider_sources::{ ProviderSourceSpec, ProviderSourceStatus, TabnineJsonlResumeState, }; +mod disk_io_pacing; +#[doc(hidden)] +pub use disk_io_pacing::{ + install_disk_io_pacer, pace_current_disk_io, DiskIoPacer, DiskIoPacingGuard, +}; + pub const CAPTURE_SCHEMA_VERSION: u32 = 1; -pub(crate) const MAX_PROVIDER_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024; +pub const MAX_PROVIDER_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024; pub(crate) const MAX_PROVIDER_SQLITE_VALUE_BYTES: usize = MAX_PROVIDER_JSONL_LINE_BYTES; pub(crate) const MAX_OPENCLAW_SESSION_INDEX_BYTES: usize = 1024 * 1024; pub(crate) const MAX_OPENCLAW_SESSION_INDEX_PATHS: usize = 256; @@ -104,6 +112,7 @@ pub(crate) mod common { } pub use common::identity::{compute_payload_hash, stable_capture_uuid}; pub(crate) use common::identity::{default_machine_id, fnv1a64, sanitize_filename_component}; +pub use common::io::provider_jsonl_range_has_complete_line; #[doc(hidden)] pub use provider::sqlite_observation::{ observe_sqlite_source_generation, SqliteObservedFile, SqliteSourceGeneration, diff --git a/crates/ctx-history-capture/src/provider/codex/catalog.rs b/crates/ctx-history-capture/src/provider/codex/catalog.rs index f312fd57..67cb6910 100644 --- a/crates/ctx-history-capture/src/provider/codex/catalog.rs +++ b/crates/ctx-history-capture/src/provider/codex/catalog.rs @@ -1,10 +1,9 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fs::{self, File}, + fs::File, io::BufReader, path::{Path, PathBuf}, thread, - time::UNIX_EPOCH, }; use ctx_history_core::{AgentType, CaptureProvider}; @@ -16,12 +15,21 @@ use crate::common::io::{ }; use crate::common::time::{parse_rfc3339_utc, system_time_ms}; use crate::{ - provider_sources::provider_import_revision, CaptureError, CatalogSummary, - CodexSessionCatalogOptions, ProviderImportFailure, Result, CODEX_SESSION_SOURCE_FORMAT, + provider_sources::{ + open_observed_ordinary_file, provider_import_revision, OrdinaryFileObservation, + }, + CaptureError, CatalogSummary, CodexSessionCatalogOptions, ProviderImportFailure, Result, + CODEX_SESSION_SOURCE_FORMAT, }; use crate::provider::codex::session::{apply_codex_session_import_bounds, contains_bytes}; +const CATALOG_PERSIST_BATCH_SESSIONS: usize = 64; +const CATALOG_PERSIST_BATCH_BYTES: usize = 8 * 1024 * 1024; +const CATALOG_PERSIST_ROW_OVERHEAD_BYTES: u64 = 256; +const QUIET_CATALOG_MAX_PARALLELISM: usize = 2; +const INTERACTIVE_CATALOG_MAX_PARALLELISM: usize = 8; + pub fn catalog_codex_session_tree( root: impl AsRef, store: &Store, @@ -39,6 +47,8 @@ pub fn catalog_codex_session_tree( }; let import_revision = provider_import_revision(CaptureProvider::Codex, CODEX_SESSION_SOURCE_FORMAT); + let initial_inventory = + prepare_initial_catalog_inventory(store, &source_root, observation_generation)?; let cataloged_at_ms = options.cataloged_at.timestamp_millis(); let mut paths = Vec::new(); collect_jsonl_paths(root, &mut paths)?; @@ -63,8 +73,8 @@ pub fn catalog_codex_session_tree( let mut paths_to_parse = Vec::new(); let mut has_metadata_failures = false; for path in paths { - let metadata = match fs::metadata(&path) { - Ok(metadata) => metadata, + let observation = match crate::observe_ordinary_file(&path) { + Ok(observation) => observation, Err(err) => { summary.failed_sessions += 1; has_metadata_failures = true; @@ -76,12 +86,12 @@ pub fn catalog_codex_session_tree( } }; summary.source_files += 1; - summary.source_bytes = summary.source_bytes.saturating_add(metadata.len()); + summary.source_bytes = summary.source_bytes.saturating_add(observation.len()); let source_path = codex_catalog_session_identity(&path)?.to_owned(); current_paths.push(source_path.clone()); if let Some(session) = cached_catalog_session_if_unchanged( existing.get(&source_path), - &metadata, + &observation, cataloged_at_ms, import_revision, ) { @@ -143,18 +153,32 @@ pub fn catalog_codex_session_tree( } summary.cataloged_sessions = parsed_session_count.saturating_add(cached_session_count); + let initial_inventory_persisted = if initial_inventory && !sessions_to_persist.is_empty() { + persist_initial_catalog_sessions_bounded( + store, + observation_generation, + &sessions_to_persist, + )? + } else { + initial_inventory + }; + store.begin_immediate_batch()?; let persist = (|| -> Result<()> { - if !sessions_to_persist.is_empty() { - store.upsert_catalog_sessions(observation_generation, &sessions_to_persist)?; - } + persist_catalog_sessions_for_publication( + store, + observation_generation, + &sessions_to_persist, + initial_inventory_persisted, + )?; if stale_session_count > 0 || has_missing_existing_paths { - store.mark_catalog_source_missing_paths_stale( + store.mark_catalog_source_missing_paths_stale_paced( CaptureProvider::Codex, &source_root, ¤t_paths, cataloged_at_ms, observation_generation, + crate::pace_current_disk_io, )?; } if !store.complete_catalog_inventory_generation( @@ -197,16 +221,26 @@ pub fn catalog_codex_session_files( store.allocate_catalog_inventory_generation(CaptureProvider::Codex, &source_root)? } }; + let initial_inventory = + prepare_initial_catalog_inventory(store, &source_root, observation_generation)?; let cataloged_at_ms = options.cataloged_at.timestamp_millis(); let (scan_summary, sessions) = catalog_codex_session_paths(paths, &source_root, cataloged_at_ms, options.parallelism)?; let mut summary = scan_summary; summary.cataloged_sessions = sessions.len(); + let initial_inventory_persisted = if initial_inventory && !sessions.is_empty() { + persist_initial_catalog_sessions_bounded(store, observation_generation, &sessions)? + } else { + initial_inventory + }; store.begin_immediate_batch()?; let persist = (|| -> Result<()> { - if !sessions.is_empty() { - store.upsert_catalog_sessions(observation_generation, &sessions)?; - } + persist_catalog_sessions_for_publication( + store, + observation_generation, + &sessions, + initial_inventory_persisted, + )?; if !store.complete_catalog_inventory_generation( CaptureProvider::Codex, &source_root, @@ -226,19 +260,246 @@ pub fn catalog_codex_session_files( Ok(summary) } +fn prepare_initial_catalog_inventory( + store: &Store, + source_root: &str, + observation_generation: u64, +) -> Result { + if !store.catalog_inventory_generation_is_unpublished( + CaptureProvider::Codex, + source_root, + observation_generation, + )? { + return Ok(false); + } + loop { + let Some((deleted, _bytes)) = store.delete_unpublished_catalog_sessions_batch_paced( + CaptureProvider::Codex, + source_root, + observation_generation, + CATALOG_PERSIST_BATCH_SESSIONS, + crate::pace_current_disk_io, + )? + else { + return Err(CaptureError::InventorySuperseded); + }; + if deleted == 0 { + return Ok(true); + } + } +} + +fn persist_initial_catalog_sessions_bounded( + store: &Store, + observation_generation: u64, + sessions: &[CatalogSession], +) -> Result { + persist_initial_catalog_sessions_bounded_with_observer( + store, + observation_generation, + sessions, + |_| {}, + ) +} + +fn persist_catalog_sessions_paced_in_current_batch( + store: &Store, + observation_generation: u64, + sessions: &[CatalogSession], +) -> Result<()> { + let byte_limit = crate::disk_io_pacing::current_disk_io_burst_bytes() + .and_then(|bytes| usize::try_from(bytes).ok()) + .map_or(CATALOG_PERSIST_BATCH_BYTES, |bytes| { + bytes.clamp(1, CATALOG_PERSIST_BATCH_BYTES) + }); + let mut start = 0; + while start < sessions.len() { + let (count, bytes) = catalog_persist_batch(&sessions[start..], byte_limit)?; + crate::pace_current_disk_io(bytes); + store.upsert_catalog_sessions( + observation_generation, + &sessions[start..start.saturating_add(count)], + )?; + start = start.saturating_add(count); + } + Ok(()) +} + +fn persist_catalog_sessions_for_publication( + store: &Store, + observation_generation: u64, + sessions: &[CatalogSession], + staged: bool, +) -> Result<()> { + let source_root = sessions + .first() + .map(|session| session.source_root.as_str()) + .unwrap_or_default(); + let staged_complete = staged + && store.catalog_sessions_all_owned_by_source_paced( + CaptureProvider::Codex, + source_root, + sessions, + crate::pace_current_disk_io, + )?; + if !staged_complete && !sessions.is_empty() { + persist_catalog_sessions_paced_in_current_batch(store, observation_generation, sessions)?; + } + Ok(()) +} + +fn persist_initial_catalog_sessions_bounded_with_observer( + store: &Store, + observation_generation: u64, + sessions: &[CatalogSession], + mut batch_committed: impl FnMut(usize), +) -> Result { + let source_root = sessions + .first() + .map(|session| session.source_root.as_str()) + .unwrap_or_default(); + if store.catalog_sessions_have_external_path_owners_paced( + CaptureProvider::Codex, + source_root, + sessions, + crate::pace_current_disk_io, + )? { + return Ok(false); + } + let byte_limit = crate::disk_io_pacing::current_disk_io_burst_bytes() + .and_then(|bytes| usize::try_from(bytes).ok()) + .map_or(CATALOG_PERSIST_BATCH_BYTES, |bytes| { + bytes.clamp(1, CATALOG_PERSIST_BATCH_BYTES) + }); + let mut start = 0; + while start < sessions.len() { + let (end, bytes) = catalog_persist_batch(&sessions[start..], byte_limit)?; + crate::pace_current_disk_io(bytes); + store.begin_immediate_batch()?; + let persist = (|| -> Result { + let source_root = sessions[start].source_root.as_str(); + if !store.catalog_inventory_generation_is_unpublished( + CaptureProvider::Codex, + source_root, + observation_generation, + )? { + return Err(CaptureError::InventorySuperseded); + } + if store.catalog_sessions_have_external_path_owners_paced( + CaptureProvider::Codex, + source_root, + &sessions[start..start + end], + crate::pace_current_disk_io, + )? { + return Ok(false); + } + store.upsert_catalog_sessions(observation_generation, &sessions[start..start + end])?; + Ok(true) + })(); + match persist { + Ok(true) => store.commit_batch()?, + Ok(false) => { + store.rollback_batch()?; + return Ok(false); + } + Err(err) => { + let _ = store.rollback_batch(); + return Err(err); + } + } + start += end; + batch_committed(start); + } + Ok(true) +} + +#[cfg(test)] +pub(crate) fn persist_initial_catalog_sessions_bounded_for_test( + store: &Store, + observation_generation: u64, + sessions: &[CatalogSession], + batch_committed: impl FnMut(usize), +) -> Result { + persist_initial_catalog_sessions_bounded_with_observer( + store, + observation_generation, + sessions, + batch_committed, + ) +} + +#[cfg(test)] +pub(crate) fn persist_catalog_sessions_for_publication_for_test( + store: &Store, + observation_generation: u64, + sessions: &[CatalogSession], + staged: bool, +) -> Result<()> { + persist_catalog_sessions_for_publication(store, observation_generation, sessions, staged) +} + +fn catalog_persist_batch(sessions: &[CatalogSession], byte_limit: usize) -> Result<(usize, u64)> { + let mut count = 0usize; + let mut bytes = 0u64; + for session in sessions.iter().take(CATALOG_PERSIST_BATCH_SESSIONS) { + let session_bytes = catalog_session_persist_bytes(session)?; + if count > 0 && bytes.saturating_add(session_bytes) > byte_limit as u64 { + break; + } + count += 1; + bytes = bytes.saturating_add(session_bytes); + if bytes >= byte_limit as u64 { + break; + } + } + Ok((count, bytes)) +} + +fn catalog_session_persist_bytes(session: &CatalogSession) -> Result { + let mut bytes = CATALOG_PERSIST_ROW_OVERHEAD_BYTES; + for value in [ + Some(session.source_format.as_str()), + Some(session.source_root.as_str()), + Some(session.source_path.as_str()), + session.external_session_id.as_deref(), + session.parent_external_session_id.as_deref(), + session.role_hint.as_deref(), + session.external_agent_id.as_deref(), + session.cwd.as_deref(), + ] + .into_iter() + .flatten() + { + bytes = bytes.saturating_add(value.len() as u64); + } + bytes = bytes.saturating_add(serde_json::to_vec(&session.metadata)?.len() as u64); + Ok(bytes) +} + +#[cfg(test)] +pub(crate) fn catalog_session_persist_bytes_for_test(session: &CatalogSession) -> Result { + catalog_session_persist_bytes(session) +} + pub(crate) fn cached_catalog_session_if_unchanged( session: Option<&CatalogSession>, - metadata: &fs::Metadata, + observation: &OrdinaryFileObservation, cataloged_at_ms: i64, import_revision: u32, ) -> Option { let session = session?; - let modified_at_ms = system_time_ms(metadata.modified().unwrap_or(UNIX_EPOCH)); + let modified_at_ms = system_time_ms(observation.modified_at()); + let observation_token = observation.token_hex(); if session.provider == CaptureProvider::Codex && session.source_format == CODEX_SESSION_SOURCE_FORMAT && session.import_revision == import_revision - && session.file_size_bytes == metadata.len() + && session.file_size_bytes == observation.len() && session.file_modified_at_ms == modified_at_ms + && session + .metadata + .get("file_observation_token_v1") + .and_then(Value::as_str) + == Some(observation_token.as_str()) { let mut session = session.clone(); session.cataloged_at_ms = cataloged_at_ms; @@ -268,12 +529,16 @@ pub(crate) fn catalog_codex_session_paths( )] } else { let chunk_size = paths.len().div_ceil(parallelism).max(1); + let disk_io_pacer = crate::disk_io_pacing::current_disk_io_pacer(); thread::scope(|scope| -> Result> { let mut handles = Vec::new(); for chunk in paths.chunks(chunk_size) { let chunk = chunk.to_vec(); let source_root = source_root.to_owned(); + let disk_io_pacer = disk_io_pacer.clone(); handles.push(scope.spawn(move || { + let _disk_io_pacing = + disk_io_pacer.map(crate::disk_io_pacing::install_disk_io_pacer); catalog_codex_session_chunk(chunk, source_root, cataloged_at_ms) })); } @@ -307,8 +572,8 @@ pub(crate) fn catalog_codex_session_chunk( ..CatalogWorkerBatch::default() }; for path in paths { - let metadata = match fs::metadata(&path) { - Ok(metadata) => metadata, + let (file, observation) = match open_observed_ordinary_file(&path) { + Ok(observed) => observed, Err(err) => { batch.summary.failed_sessions += 1; batch.summary.sample_failure(ProviderImportFailure { @@ -319,8 +584,14 @@ pub(crate) fn catalog_codex_session_chunk( } }; batch.summary.source_files += 1; - batch.summary.source_bytes = batch.summary.source_bytes.saturating_add(metadata.len()); - match catalog_codex_session_file(&path, source_root.as_str(), &metadata, cataloged_at_ms) { + batch.summary.source_bytes = batch.summary.source_bytes.saturating_add(observation.len()); + match catalog_codex_session_file( + &path, + file, + source_root.as_str(), + &observation, + cataloged_at_ms, + ) { Ok(session) => { batch.summary.parsed_sessions += 1; batch.sessions.push(session); @@ -343,20 +614,33 @@ pub(crate) fn catalog_parallelism( if path_count <= 1 { return 1; } + let pacing_limit = crate::disk_io_pacing::current_disk_io_pacer() + .map(|pacer| { + if pacer.bytes_per_second() <= 8 * 1024 * 1024 { + QUIET_CATALOG_MAX_PARALLELISM + } else if pacer.bytes_per_second() <= 32 * 1024 * 1024 { + INTERACTIVE_CATALOG_MAX_PARALLELISM + } else { + 32 + } + }) + .unwrap_or(32); requested_parallelism .or_else(|| thread::available_parallelism().ok().map(usize::from)) .unwrap_or(1) .clamp(1, 32) + .min(pacing_limit) .min(path_count) } pub(crate) fn catalog_codex_session_file( path: &Path, + file: File, source_root: &str, - metadata: &fs::Metadata, + observation: &OrdinaryFileObservation, cataloged_at_ms: i64, ) -> Result { let source_path = codex_catalog_session_identity(path)?; - let session_meta = read_codex_session_meta(path)?; + let session_meta = read_codex_session_meta_from_file(file)?; let payload = session_meta.as_ref().and_then(|value| value.get("payload")); let source = payload .and_then(|payload| payload.get("source")) @@ -412,8 +696,8 @@ pub(crate) fn catalog_codex_session_file( .filter(|cwd| !cwd.trim().is_empty()) .map(str::to_owned), session_started_at_ms, - file_size_bytes: metadata.len(), - file_modified_at_ms: system_time_ms(metadata.modified().unwrap_or(UNIX_EPOCH)), + file_size_bytes: observation.len(), + file_modified_at_ms: system_time_ms(observation.modified_at()), import_revision: provider_import_revision( CaptureProvider::Codex, CODEX_SESSION_SOURCE_FORMAT, @@ -426,6 +710,7 @@ pub(crate) fn catalog_codex_session_file( "source_kind": codex_source_kind(&source), "source": source, "catalog_scope": "session_meta", + "file_observation_token_v1": observation.token_hex(), }), }) } @@ -452,9 +737,9 @@ fn validate_codex_catalog_session_paths(paths: &[PathBuf]) -> Result<()> { } Ok(()) } -pub(crate) fn read_codex_session_meta(path: &Path) -> Result> { - let file = File::open(path)?; - let mut reader = BufReader::new(file); + +fn read_codex_session_meta_from_file(file: File) -> Result> { + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut line = Vec::new(); for _ in 0..32 { match read_provider_jsonl_line_or_skip_oversized(&mut reader, &mut line)? { diff --git a/crates/ctx-history-capture/src/provider/codex/events.rs b/crates/ctx-history-capture/src/provider/codex/events.rs index 5435f128..3b97d68e 100644 --- a/crates/ctx-history-capture/src/provider/codex/events.rs +++ b/crates/ctx-history-capture/src/provider/codex/events.rs @@ -676,7 +676,8 @@ pub(crate) fn codex_tool_output_event( }), )) } + include!("events/payload.rs"); #[cfg(test)] -mod resume_tests; +include!("events/tests.rs"); diff --git a/crates/ctx-history-capture/src/provider/codex/events/resume_tests.rs b/crates/ctx-history-capture/src/provider/codex/events/resume_tests.rs deleted file mode 100644 index 4747089c..00000000 --- a/crates/ctx-history-capture/src/provider/codex/events/resume_tests.rs +++ /dev/null @@ -1,48 +0,0 @@ -use super::*; - -#[test] -fn codex_tool_call_resume_frontier_evicts_oldest_with_bounded_encoding() { - let mut contexts = CodexToolCallContexts::default(); - for index in 0..(CODEX_RESUME_MAX_PENDING_TOOL_CALLS + 9) { - contexts.insert( - format!("call-{index:03}"), - CodexToolCallContext { - tool_name: "exec_command".to_owned(), - command_preview: Some(format!("command-{index:03}")), - arguments_preview: None, - }, - ); - } - - let state = contexts.resume_state(); - assert_eq!(contexts.len(), CODEX_RESUME_MAX_PENDING_TOOL_CALLS); - assert_eq!(state.dropped_tool_calls, 9); - assert_eq!(state.pending_tool_calls[0].call_id, "call-009"); - assert_eq!( - state.pending_tool_calls.last().unwrap().call_id, - format!("call-{:03}", CODEX_RESUME_MAX_PENDING_TOOL_CALLS + 8) - ); - assert!(state.encoded_len().unwrap() <= CODEX_RESUME_MAX_ENCODED_BYTES); - - let restarted = CodexToolCallContexts::from_resume_state(state.clone()); - assert_eq!(restarted.resume_state(), state); -} - -#[test] -fn codex_tool_call_resume_frontier_evicts_an_oversized_context() { - let mut contexts = CodexToolCallContexts::default(); - contexts.insert( - "oversized".to_owned(), - CodexToolCallContext { - tool_name: "exec_command".to_owned(), - command_preview: Some("x".repeat(CODEX_RESUME_MAX_ENCODED_BYTES)), - arguments_preview: None, - }, - ); - - let state = contexts.resume_state(); - assert_eq!(contexts.len(), 0); - assert_eq!(state.dropped_tool_calls, 1); - assert!(state.pending_tool_calls.is_empty()); - assert!(state.encoded_len().unwrap() <= CODEX_RESUME_MAX_ENCODED_BYTES); -} diff --git a/crates/ctx-history-capture/src/provider/codex/events/tests.rs b/crates/ctx-history-capture/src/provider/codex/events/tests.rs new file mode 100644 index 00000000..885935eb --- /dev/null +++ b/crates/ctx-history-capture/src/provider/codex/events/tests.rs @@ -0,0 +1,50 @@ +mod resume_tests { + use super::*; + + #[test] + fn codex_tool_call_resume_frontier_evicts_oldest_with_bounded_encoding() { + let mut contexts = CodexToolCallContexts::default(); + for index in 0..(CODEX_RESUME_MAX_PENDING_TOOL_CALLS + 9) { + contexts.insert( + format!("call-{index:03}"), + CodexToolCallContext { + tool_name: "exec_command".to_owned(), + command_preview: Some(format!("command-{index:03}")), + arguments_preview: None, + }, + ); + } + + let state = contexts.resume_state(); + assert_eq!(contexts.len(), CODEX_RESUME_MAX_PENDING_TOOL_CALLS); + assert_eq!(state.dropped_tool_calls, 9); + assert_eq!(state.pending_tool_calls[0].call_id, "call-009"); + assert_eq!( + state.pending_tool_calls.last().unwrap().call_id, + format!("call-{:03}", CODEX_RESUME_MAX_PENDING_TOOL_CALLS + 8) + ); + assert!(state.encoded_len().unwrap() <= CODEX_RESUME_MAX_ENCODED_BYTES); + + let restarted = CodexToolCallContexts::from_resume_state(state.clone()); + assert_eq!(restarted.resume_state(), state); + } + + #[test] + fn codex_tool_call_resume_frontier_evicts_an_oversized_context() { + let mut contexts = CodexToolCallContexts::default(); + contexts.insert( + "oversized".to_owned(), + CodexToolCallContext { + tool_name: "exec_command".to_owned(), + command_preview: Some("x".repeat(CODEX_RESUME_MAX_ENCODED_BYTES)), + arguments_preview: None, + }, + ); + + let state = contexts.resume_state(); + assert_eq!(contexts.len(), 0); + assert_eq!(state.dropped_tool_calls, 1); + assert!(state.pending_tool_calls.is_empty()); + assert!(state.encoded_len().unwrap() <= CODEX_RESUME_MAX_ENCODED_BYTES); + } +} diff --git a/crates/ctx-history-capture/src/provider/codex/fast_import.rs b/crates/ctx-history-capture/src/provider/codex/fast_import.rs index 65a81d36..aef3d26f 100644 --- a/crates/ctx-history-capture/src/provider/codex/fast_import.rs +++ b/crates/ctx-history-capture/src/provider/codex/fast_import.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use ctx_history_core::{ CaptureProvider, Event, EventType, ProviderEventEnvelope, ProviderSourceTrust, }; -use ctx_history_store::Store; +use ctx_history_store::{EventSearchBulkMaintenanceOutcome, Store}; use serde_json::{json, Value}; use uuid::Uuid; @@ -52,7 +52,14 @@ pub(crate) fn import_codex_session_paths_fast( import_codex_session_paths_fast_bounded(paths, store, &options, skipped_by_bounds); let finish_result = store.finish_event_search_bulk_mode(&bulk_guard); match (import_result, finish_result) { - (Ok(summary), Ok(())) => Ok(summary), + (Ok(summary), Ok(EventSearchBulkMaintenanceOutcome::Complete)) => Ok(summary), + (Ok(mut summary), Ok(EventSearchBulkMaintenanceOutcome::Pending)) => { + summary.push_maintenance_warning( + crate::ProviderImportMaintenanceKind::EventSearchFinalizationPending, + "event search maintenance remains queued", + ); + Ok(summary) + } (Ok(mut summary), Err(error)) => { summary.push_maintenance_warning( crate::ProviderImportMaintenanceKind::EventSearchFinalization, @@ -60,7 +67,7 @@ pub(crate) fn import_codex_session_paths_fast( ); Ok(summary) } - (Err(err), Err(_)) | (Err(err), Ok(())) => Err(err), + (Err(err), _) => Err(err), } } @@ -576,7 +583,24 @@ pub(crate) fn import_codex_session_reader_bounded( })(); let finish_result = store.finish_event_search_bulk_mode(&bulk_guard); match (import_result, finish_result) { - (Ok(summary), Ok(())) => Ok(summary), + (Ok(summary), Ok(EventSearchBulkMaintenanceOutcome::Complete)) => Ok(summary), + ( + Ok(CodexSessionBoundedImport::Imported { + mut summary, + boundary, + }), + Ok(EventSearchBulkMaintenanceOutcome::Pending), + ) => { + summary.push_maintenance_warning( + crate::ProviderImportMaintenanceKind::EventSearchFinalizationPending, + "event search maintenance remains queued", + ); + Ok(CodexSessionBoundedImport::Imported { summary, boundary }) + } + ( + Ok(CodexSessionBoundedImport::ReplacementRequired(reason)), + Ok(EventSearchBulkMaintenanceOutcome::Pending), + ) => Ok(CodexSessionBoundedImport::ReplacementRequired(reason)), ( Ok(CodexSessionBoundedImport::Imported { mut summary, diff --git a/crates/ctx-history-capture/src/provider/codex/session.rs b/crates/ctx-history-capture/src/provider/codex/session.rs index 5cd11e48..7093f2ba 100644 --- a/crates/ctx-history-capture/src/provider/codex/session.rs +++ b/crates/ctx-history-capture/src/provider/codex/session.rs @@ -6,7 +6,7 @@ use std::{ }; use ctx_history_core::{CaptureProvider, EventRole, EventType, ProviderEventEnvelope}; -use ctx_history_store::Store; +use ctx_history_store::{EventSearchBulkMaintenanceOutcome, Store}; use serde_json::Value; use crate::CodexSessionJsonlAdapter; @@ -502,7 +502,14 @@ pub fn import_codex_session_jsonl_tail( import_codex_session_jsonl_tail_bounded(path, start_offset, total_bytes, store, &options); let finish_result = store.finish_event_search_bulk_mode(&bulk_guard); match (import_result, finish_result) { - (Ok(summary), Ok(())) => Ok(summary), + (Ok(summary), Ok(EventSearchBulkMaintenanceOutcome::Complete)) => Ok(summary), + (Ok(mut summary), Ok(EventSearchBulkMaintenanceOutcome::Pending)) => { + summary.push_maintenance_warning( + crate::ProviderImportMaintenanceKind::EventSearchFinalizationPending, + "event search maintenance remains queued", + ); + Ok(summary) + } (Ok(mut summary), Err(error)) => { summary.push_maintenance_warning( crate::ProviderImportMaintenanceKind::EventSearchFinalization, @@ -553,7 +560,7 @@ fn import_codex_session_jsonl_tail_bounded( let mut transaction = None; let import = (|| -> Result { let file = File::open(path)?; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut line = Vec::new(); let mut line_number = 0usize; let mut position = 0u64; @@ -860,7 +867,14 @@ pub(crate) fn import_codex_session_paths_parallel_normalized( ); let finish_result = store.finish_event_search_bulk_mode(&bulk_guard); match (import_result, finish_result) { - (Ok(summary), Ok(())) => Ok(summary), + (Ok(summary), Ok(EventSearchBulkMaintenanceOutcome::Complete)) => Ok(summary), + (Ok(mut summary), Ok(EventSearchBulkMaintenanceOutcome::Pending)) => { + summary.push_maintenance_warning( + crate::ProviderImportMaintenanceKind::EventSearchFinalizationPending, + "event search maintenance remains queued", + ); + Ok(summary) + } (Ok(mut summary), Err(error)) => { summary.push_maintenance_warning( crate::ProviderImportMaintenanceKind::EventSearchFinalization, @@ -997,11 +1011,15 @@ pub(crate) fn normalize_codex_session_paths_parallel( } let chunk_size = paths.len().div_ceil(parallelism).max(1); + let disk_io_pacer = crate::disk_io_pacing::current_disk_io_pacer(); let mut batches = thread::scope(|scope| { let mut handles = Vec::new(); for (chunk_index, chunk) in paths.chunks(chunk_size).enumerate() { let chunk = chunk.to_vec(); + let disk_io_pacer = disk_io_pacer.clone(); handles.push(scope.spawn(move || { + let _disk_io_pacing = + disk_io_pacer.map(crate::disk_io_pacing::install_disk_io_pacer); let mut normalized = Vec::with_capacity(chunk.len()); let base_index = chunk_index * chunk_size; for (offset, path) in chunk.iter().enumerate() { diff --git a/crates/ctx-history-capture/src/provider/custom_history_jsonl.rs b/crates/ctx-history-capture/src/provider/custom_history_jsonl.rs index 28e52029..86ad3175 100644 --- a/crates/ctx-history-capture/src/provider/custom_history_jsonl.rs +++ b/crates/ctx-history-capture/src/provider/custom_history_jsonl.rs @@ -64,7 +64,7 @@ pub(crate) fn normalize_custom_history_jsonl_v1( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); normalize_custom_history_jsonl_v1_reader(reader, context) } diff --git a/crates/ctx-history-capture/src/provider/importer/batches.rs b/crates/ctx-history-capture/src/provider/importer/batches.rs index 505c2872..f68e1d13 100644 --- a/crates/ctx-history-capture/src/provider/importer/batches.rs +++ b/crates/ctx-history-capture/src/provider/importer/batches.rs @@ -490,10 +490,19 @@ impl<'a> ProviderNormalizationStreamImporter<'a> { fn finish_bulk_search(&mut self) -> Result<()> { match self.bulk_search_guard.take() { - Some(guard) => self - .store - .finish_event_search_bulk_mode(&guard) - .map_err(CaptureError::from), + Some(guard) => { + let outcome = self + .store + .finish_event_search_bulk_mode(&guard) + .map_err(CaptureError::from)?; + if outcome == ctx_history_store::EventSearchBulkMaintenanceOutcome::Pending { + self.summary.push_maintenance_warning( + crate::ProviderImportMaintenanceKind::EventSearchFinalizationPending, + "event search maintenance remains queued", + ); + } + Ok(()) + } None => Ok(()), } } diff --git a/crates/ctx-history-capture/src/provider/importer/batches/persistence.rs b/crates/ctx-history-capture/src/provider/importer/batches/persistence.rs index 561c6869..5806baef 100644 --- a/crates/ctx-history-capture/src/provider/importer/batches/persistence.rs +++ b/crates/ctx-history-capture/src/provider/importer/batches/persistence.rs @@ -131,7 +131,7 @@ fn import_provider_capture_lines_with_batch_size( Some(guard) => store .finish_event_search_bulk_mode(guard) .map_err(CaptureError::from), - None => Ok(()), + None => Ok(ctx_history_store::EventSearchBulkMaintenanceOutcome::Complete), }; match (import_result, finish_result) { (Ok(mut summary), Err(error)) => { @@ -141,7 +141,16 @@ fn import_provider_capture_lines_with_batch_size( ); Ok(summary) } - (Ok(summary), Ok(())) => Ok(summary), + (Ok(summary), Ok(ctx_history_store::EventSearchBulkMaintenanceOutcome::Complete)) => { + Ok(summary) + } + (Ok(mut summary), Ok(ctx_history_store::EventSearchBulkMaintenanceOutcome::Pending)) => { + summary.push_maintenance_warning( + crate::ProviderImportMaintenanceKind::EventSearchFinalizationPending, + "event search maintenance remains queued", + ); + Ok(summary) + } (Err(err), _) => Err(err), } } @@ -317,6 +326,7 @@ pub(crate) struct ProviderImportTransaction { batch_size: Option, units: usize, bytes: usize, + byte_limit: usize, durable_materialization: bool, maintenance_warning: Option, } @@ -345,6 +355,11 @@ impl ProviderImportTransaction { batch_size, units: 0, bytes: 0, + byte_limit: crate::disk_io_pacing::current_disk_io_burst_bytes() + .and_then(|bytes| usize::try_from(bytes).ok()) + .map_or(IMPORT_TRANSACTION_BATCH_BYTES, |bytes| { + bytes.clamp(1, IMPORT_TRANSACTION_BATCH_BYTES) + }), durable_materialization: false, maintenance_warning: None, }) @@ -376,7 +391,7 @@ impl ProviderImportTransaction { let result = if self.active && self.batch_size.is_some() && self.units > 0 - && self.bytes.saturating_add(unit_bytes) > IMPORT_TRANSACTION_BATCH_BYTES + && self.bytes.saturating_add(unit_bytes) > self.byte_limit { self.rotate(store) } else { @@ -385,6 +400,9 @@ impl ProviderImportTransaction { if result.is_err() { self.rollback(store); } + if matches!(&result, Ok(ProviderImportTransactionStep::Continue)) { + crate::pace_current_disk_io(unit_bytes as u64); + } result } @@ -404,8 +422,7 @@ impl ProviderImportTransaction { let below_unit_limit = self .batch_size .is_none_or(|batch_size| self.units < batch_size.get()); - let below_byte_limit = - self.batch_size.is_none() || self.bytes < IMPORT_TRANSACTION_BATCH_BYTES; + let below_byte_limit = self.batch_size.is_none() || self.bytes < self.byte_limit; if below_unit_limit && below_byte_limit { return Ok(ProviderImportTransactionStep::Continue); } @@ -417,6 +434,7 @@ impl ProviderImportTransaction { } fn rotate(&mut self, store: &Store) -> Result { + crate::pace_current_disk_io(u64::try_from(self.bytes).unwrap_or(u64::MAX)); store.commit_batch()?; self.active = false; self.durable_materialization = true; @@ -442,6 +460,7 @@ impl ProviderImportTransaction { pub(crate) fn commit(&mut self, store: &Store) -> Result<()> { let result = if self.active { + crate::pace_current_disk_io(u64::try_from(self.bytes).unwrap_or(u64::MAX)); store.commit_batch().map_err(CaptureError::from) } else { Ok(()) @@ -488,3 +507,28 @@ impl ProviderImportTransaction { .get_or_insert(crate::ProviderImportMaintenanceWarning { kind, error }); } } + +#[cfg(test)] +mod pacing_tests { + use super::*; + use crate::{install_disk_io_pacer, DiskIoPacer}; + + #[test] + fn durable_transaction_batch_charges_actual_serialized_bytes() { + let temp = tempfile::tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let pacer = DiskIoPacer::new(u64::MAX, u64::MAX); + let _pacing = install_disk_io_pacer(pacer.clone()); + let mut transaction = ProviderImportTransaction::begin_bounded(&store, true).unwrap(); + + assert_eq!( + transaction + .record_unit(&store, IMPORT_TRANSACTION_BATCH_BYTES) + .unwrap(), + ProviderImportTransactionStep::Continue + ); + transaction.commit(&store).unwrap(); + + assert_eq!(pacer.charged_bytes(), IMPORT_TRANSACTION_BATCH_BYTES as u64); + } +} diff --git a/crates/ctx-history-capture/src/provider/providers/codebuddy.rs b/crates/ctx-history-capture/src/provider/providers/codebuddy.rs index dc0ec923..70a5a30f 100644 --- a/crates/ctx-history-capture/src/provider/providers/codebuddy.rs +++ b/crates/ctx-history-capture/src/provider/providers/codebuddy.rs @@ -429,7 +429,7 @@ pub(crate) fn normalize_codebuddy_cli_jsonl_file( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut result = ProviderNormalizationResult::default(); let mut events = Vec::new(); let mut row_count = 0usize; diff --git a/crates/ctx-history-capture/src/provider/providers/junie.rs b/crates/ctx-history-capture/src/provider/providers/junie.rs index c0e40997..d46b5bcf 100644 --- a/crates/ctx-history-capture/src/provider/providers/junie.rs +++ b/crates/ctx-history-capture/src/provider/providers/junie.rs @@ -218,7 +218,7 @@ pub(crate) fn junie_index_meta_for_events_path( pub(crate) fn junie_read_index(path: &Path) -> Result> { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut metas = Vec::new(); let mut line = Vec::new(); loop { @@ -367,7 +367,7 @@ pub(crate) fn normalize_junie_session_events_file( }; let file = File::open(&session_path.events_path)?; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut line = Vec::new(); let mut line_number = 0usize; while read_provider_jsonl_record_or_skip_oversized( diff --git a/crates/ctx-history-capture/src/provider/providers/kimi.rs b/crates/ctx-history-capture/src/provider/providers/kimi.rs index da53e736..2f863176 100644 --- a/crates/ctx-history-capture/src/provider/providers/kimi.rs +++ b/crates/ctx-history-capture/src/provider/providers/kimi.rs @@ -75,7 +75,7 @@ pub(crate) fn normalize_kimi_wire_jsonl_file( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut result = ProviderNormalizationResult::default(); let mut rows = Vec::new(); let mut line = Vec::new(); @@ -539,7 +539,7 @@ pub(crate) fn read_kimi_session_index(path: &Path) -> BTreeMap = "2026-07-14T11:00:00Z".parse().unwrap(); - assert!(normalized - .captures - .iter() - .all(|(_, capture)| capture.session.started_at == expected)); + assert_eq!(normalized.summary.failed, 0); + assert!(!normalized.captures.is_empty()); + let expected: DateTime = "2026-07-14T11:00:00Z".parse().unwrap(); + assert!(normalized + .captures + .iter() + .all(|(_, capture)| capture.session.started_at == expected)); + } } diff --git a/crates/ctx-history-capture/src/provider/providers/openclaw.rs b/crates/ctx-history-capture/src/provider/providers/openclaw.rs index fbd1e8e7..4b2e9ce8 100644 --- a/crates/ctx-history-capture/src/provider/providers/openclaw.rs +++ b/crates/ctx-history-capture/src/provider/providers/openclaw.rs @@ -197,7 +197,7 @@ pub(crate) fn normalize_openclaw_jsonl_file( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut result = ProviderNormalizationResult::default(); let fallback_id = path .file_stem() diff --git a/crates/ctx-history-capture/src/provider/providers/task_json.rs b/crates/ctx-history-capture/src/provider/providers/task_json.rs index b4cae16c..f72875ed 100644 --- a/crates/ctx-history-capture/src/provider/providers/task_json.rs +++ b/crates/ctx-history-capture/src/provider/providers/task_json.rs @@ -1,6 +1,7 @@ use std::{ collections::BTreeMap, - fs::{self}, + fs::{self, File}, + io::Read, path::{Path, PathBuf}, }; @@ -446,7 +447,10 @@ pub(crate) fn read_task_json_value( reason: "provider task JSON file exceeds maximum supported size", }); } - let bytes = fs::read(path)?; + let file = File::open(path)?; + let mut reader = crate::disk_io_pacing::PacedReader::new(file); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes)?; serde_json::from_slice(&bytes).map_err(CaptureError::from) } diff --git a/crates/ctx-history-capture/src/provider/sqlite.rs b/crates/ctx-history-capture/src/provider/sqlite.rs index 76b905cd..3b6d7890 100644 --- a/crates/ctx-history-capture/src/provider/sqlite.rs +++ b/crates/ctx-history-capture/src/provider/sqlite.rs @@ -2,7 +2,7 @@ use std::{ collections::BTreeSet, env, fs, fs::File, - io::{self, Read, Seek, SeekFrom}, + io::{self, Read, Seek, SeekFrom, Write}, ops::Deref, path::{Path, PathBuf}, }; @@ -19,7 +19,7 @@ use crate::provider::sqlite_observation::{ SQLITE_GENERATION_MAX_ATTEMPTS, SQLITE_SNAPSHOT_MAX_BYTES, }; -use crate::{CaptureError, Result, MAX_PROVIDER_SQLITE_VALUE_BYTES}; +use crate::{pace_current_disk_io, CaptureError, Result, MAX_PROVIDER_SQLITE_VALUE_BYTES}; pub(crate) fn sqlite_table_exists(conn: &Connection, table: &str) -> Result { let exists: i64 = conn.query_row( @@ -546,16 +546,37 @@ fn copy_sqlite_snapshot_file( let mut destination_file = create_private_snapshot_file(destination)?; run_snapshot_copy_test_hook(source.path()); source_file.seek(SeekFrom::Start(0))?; - let copied = io::copy( - &mut source_file.by_ref().take(byte_count), - &mut destination_file, - )?; - if copied != byte_count { - return Ok(false); + let mut remaining = byte_count; + let mut buffer = vec![0_u8; 256 * 1024]; + while remaining > 0 { + let limit = usize::try_from(remaining.min(buffer.len() as u64)).unwrap_or(buffer.len()); + pace_current_disk_io(limit as u64); + let read = source_file.read(&mut buffer[..limit])?; + if read == 0 { + return Ok(false); + } + write_all_paced(&mut destination_file, &buffer[..read])?; + remaining = remaining.saturating_sub(read as u64); } Ok(true) } +fn write_all_paced(file: &mut File, bytes: &[u8]) -> io::Result<()> { + let mut offset = 0; + while offset < bytes.len() { + pace_current_disk_io((bytes.len() - offset) as u64); + let written = file.write(&bytes[offset..])?; + if written == 0 { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "SQLite snapshot destination stopped accepting bytes", + )); + } + offset += written; + } + Ok(()) +} + fn create_private_snapshot_dir_in(parent: &Path) -> io::Result { for _ in 0..16 { let path = parent.join(format!("ctx-provider-sqlite-{}", Uuid::new_v4().simple())); diff --git a/crates/ctx-history-capture/src/provider/sqlite/tests.rs b/crates/ctx-history-capture/src/provider/sqlite/tests.rs index 0cb3d8fd..56471b6e 100644 --- a/crates/ctx-history-capture/src/provider/sqlite/tests.rs +++ b/crates/ctx-history-capture/src/provider/sqlite/tests.rs @@ -15,6 +15,7 @@ mod tests { }; #[cfg(unix)] use crate::CaptureError; + use crate::{install_disk_io_pacer, DiskIoPacer}; use super::{ acquire_snapshot_copy_lock, create_private_snapshot_dir_in, create_private_snapshot_file, @@ -152,6 +153,26 @@ mod tests { ); } + #[test] + fn snapshot_copy_accounts_source_and_destination_bytes() { + let temp = tempfile::tempdir().unwrap(); + let db = temp.path().join("paced-snapshot.db"); + let _writer = real_wal_writer(&db); + let generation = observe_sqlite_source_generation(&db).unwrap(); + let snapshot_bytes = generation + .snapshot_files() + .into_iter() + .map(|file| file.snapshot_len()) + .sum::(); + let pacer = DiskIoPacer::new(u64::MAX, u64::MAX); + let _pacing = install_disk_io_pacer(pacer.clone()); + + let connection = open_sqlite_readonly_source(&db).unwrap(); + drop(connection); + + assert!(pacer.charged_bytes() >= snapshot_bytes.saturating_mul(2)); + } + #[test] fn public_open_recovers_the_last_committed_prefix_before_a_bad_wal_frame() { let temp = tempfile::tempdir().unwrap(); @@ -959,6 +980,7 @@ mod tests { writer .execute("UPDATE entries SET value = replace(value, 'a', 'b')", []) .unwrap(); + writer.cache_flush().unwrap(); let journal = sidecar(&source, "-journal"); let journal_bytes = fs::read(&journal).unwrap(); assert!(journal_bytes.starts_with(&super::super::sqlite_observation::JOURNAL_MAGIC)); diff --git a/crates/ctx-history-capture/src/provider/sqlite_observation.rs b/crates/ctx-history-capture/src/provider/sqlite_observation.rs index dcd1a9e0..fab27316 100644 --- a/crates/ctx-history-capture/src/provider/sqlite_observation.rs +++ b/crates/ctx-history-capture/src/provider/sqlite_observation.rs @@ -9,6 +9,8 @@ use std::{ #[cfg(not(any(unix, windows)))] use sha2::{Digest, Sha256}; +use crate::pace_current_disk_io; + pub const SQLITE_GENERATION_MAX_ATTEMPTS: usize = 3; pub const SQLITE_SNAPSHOT_MAX_BYTES: u64 = 2 * 1024 * 1024 * 1024; const SQLITE_HEADER_BYTES: usize = 100; diff --git a/crates/ctx-history-capture/src/provider/sqlite_observation/sentinels.rs b/crates/ctx-history-capture/src/provider/sqlite_observation/sentinels.rs index 0a32b5cd..7e7dbdee 100644 --- a/crates/ctx-history-capture/src/provider/sqlite_observation/sentinels.rs +++ b/crates/ctx-history-capture/src/provider/sqlite_observation/sentinels.rs @@ -91,6 +91,7 @@ fn append_main_file_identity( let mut hasher = Sha256::new(); let mut buffer = [0_u8; 64 * 1024]; loop { + pace_current_disk_io(buffer.len() as u64); let read = file.read(&mut buffer)?; if read == 0 { break; @@ -159,7 +160,7 @@ fn wal_sentinel(path: &Path, file: &mut File, len: u64) -> io::Result io::Re fn read_wal_frame_header(file: &mut File, offset: u64) -> io::Result<[u8; 24]> { let mut header = [0_u8; WAL_FRAME_HEADER_BYTES]; file.seek(SeekFrom::Start(offset))?; - file.read_exact(&mut header)?; + read_exact_paced(file, &mut header)?; Ok(header) } @@ -453,6 +454,7 @@ fn native_super_journal_path(name: Vec) -> io::Result { fn read_prefix(file: &mut File, limit: usize) -> io::Result> { file.seek(SeekFrom::Start(0))?; let mut bytes = vec![0_u8; limit]; + pace_current_disk_io(bytes.len() as u64); let read = file.read(&mut bytes)?; bytes.truncate(read); Ok(bytes) @@ -462,17 +464,33 @@ fn read_tail(file: &mut File, len: u64, limit: usize) -> io::Result> { let count = usize::try_from(len.min(limit as u64)).unwrap_or(limit); file.seek(SeekFrom::Start(len.saturating_sub(count as u64)))?; let mut bytes = vec![0_u8; count]; - file.read_exact(&mut bytes)?; + read_exact_paced(file, &mut bytes)?; Ok(bytes) } fn read_at(file: &mut File, offset: u64, len: usize) -> io::Result> { file.seek(SeekFrom::Start(offset))?; let mut bytes = vec![0_u8; len]; - file.read_exact(&mut bytes)?; + read_exact_paced(file, &mut bytes)?; Ok(bytes) } +fn read_exact_paced(file: &mut File, bytes: &mut [u8]) -> io::Result<()> { + let mut offset = 0; + while offset < bytes.len() { + pace_current_disk_io((bytes.len() - offset) as u64); + let read = file.read(&mut bytes[offset..])?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "SQLite source changed while reading observed bytes", + )); + } + offset += read; + } + Ok(()) +} + fn be_u32(bytes: &[u8]) -> u32 { u32::from_be_bytes(bytes.try_into().unwrap_or_default()) } diff --git a/crates/ctx-history-capture/src/provider/sqlite_observation/tests.rs b/crates/ctx-history-capture/src/provider/sqlite_observation/tests.rs index 77098edc..55f5b58c 100644 --- a/crates/ctx-history-capture/src/provider/sqlite_observation/tests.rs +++ b/crates/ctx-history-capture/src/provider/sqlite_observation/tests.rs @@ -3,6 +3,8 @@ mod tests { use rusqlite::Connection; + use crate::{install_disk_io_pacer, DiskIoPacer}; + use super::*; #[test] @@ -50,6 +52,18 @@ mod tests { } } + #[test] + fn wal_validation_accounts_the_bytes_it_reads() { + let fixture = real_wal_fixture(512); + let pacer = DiskIoPacer::new(u64::MAX, u64::MAX); + let _pacing = install_disk_io_pacer(pacer.clone()); + + let generation = observe_sqlite_source_generation(&fixture.db).unwrap(); + let committed_wal_bytes = generation.wal.as_ref().unwrap().snapshot_len(); + + assert!(pacer.charged_bytes() >= committed_wal_bytes.saturating_mul(2)); + } + #[test] fn real_wal_classifies_stable_corruption_stale_suffix_and_partial_frame() { let fixture = real_wal_fixture(512); diff --git a/crates/ctx-history-capture/src/provider_sources/append_import.rs b/crates/ctx-history-capture/src/provider_sources/append_import.rs index 46259a4e..1b815478 100644 --- a/crates/ctx-history-capture/src/provider_sources/append_import.rs +++ b/crates/ctx-history-capture/src/provider_sources/append_import.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; -use ctx_history_core::CaptureProvider; +use ctx_history_core::{canonical_provider_material_source_format, CaptureProvider}; use ctx_history_store::Store; use rusqlite::{params, Connection}; use serde_json::Value; @@ -32,8 +32,7 @@ use crate::{ ProviderFileMutationContract, ProviderImportFailure, ProviderImportSummary, ProviderJsonlAppendCheckpoint, ProviderJsonlOpenDecision, ProviderJsonlOpenMode, ProviderJsonlReader, ProviderJsonlRecordRead, ProviderJsonlReplacementReason, - ProviderJsonlResumeState, Result, TabnineJsonlResumeState, CLAUDE_PROJECTS_SOURCE_FORMAT, - CODEX_SESSION_SOURCE_FORMAT, TABNINE_CLI_SOURCE_FORMAT, + ProviderJsonlResumeState, Result, TabnineJsonlResumeState, TABNINE_CLI_SOURCE_FORMAT, }; use super::{ClaudeProjectsJsonlResumeState, CodexSessionJsonlResumeState}; @@ -76,6 +75,7 @@ pub struct ProviderAppendFileImportOptions { pub source_root: PathBuf, pub imported_at: DateTime, pub history_record_id: Option, + pub observed_size: u64, pub mode: ProviderAppendFileImportMode, } @@ -89,6 +89,7 @@ pub struct ProviderAppendFileImportResult { pub struct ProviderAppendFileImportWithoutCheckpoint { pub summary: ProviderImportSummary, pub reason: ProviderJsonlReplacementReason, + pub source_prefix_sha256: Option, } #[derive(Debug, Clone)] @@ -178,6 +179,7 @@ fn import_append_capable_provider_file_with_post_materialization( )); } }; + reader.limit_to_observed_size(options.observed_size)?; let context = ProviderAdapterContext { machine_id: options.machine_id, source_path: Some(options.source_path.clone()), @@ -585,13 +587,21 @@ fn import_append_capable_provider_file_with_post_materialization( } }; - if summary.requires_maintenance() { - return Ok(ProviderAppendFileImportDecision::ImportedWithoutCheckpoint( + if summary.has_checkpoint_blocking_maintenance() { + let mut decision = ProviderAppendFileImportDecision::ImportedWithoutCheckpoint( ProviderAppendFileImportWithoutCheckpoint { summary, reason: ProviderJsonlReplacementReason::CommittedMaintenanceIncomplete, + source_prefix_sha256: None, }, - )); + ); + certify_uncheckpointed_replacement( + &mut reader, + options.observed_size, + is_replacement, + &mut decision, + ); + return Ok(decision); } post_materialization(&mut reader); let checkpoint_result = match semantic_boundary { @@ -603,22 +613,38 @@ fn import_append_capable_provider_file_with_post_materialization( let checkpoint_decision = match checkpoint_result { Ok(decision) => decision, Err(error) => { - return Ok(ProviderAppendFileImportDecision::ImportedWithoutCheckpoint( + let mut decision = ProviderAppendFileImportDecision::ImportedWithoutCheckpoint( ProviderAppendFileImportWithoutCheckpoint { summary, reason: unexpected_post_commit_checkpoint_failure(&error), + source_prefix_sha256: None, }, - )); + ); + certify_uncheckpointed_replacement( + &mut reader, + options.observed_size, + is_replacement, + &mut decision, + ); + return Ok(decision); } }; - Ok(finish_import( + let mut decision = finish_import( summary, checkpoint_decision, checkpoint_resume_state, certification_failure, - )) + ); + certify_uncheckpointed_replacement( + &mut reader, + options.observed_size, + is_replacement, + &mut decision, + ); + Ok(decision) } include!("append_import/completion.rs"); + #[cfg(test)] include!("append_import/tests.rs"); diff --git a/crates/ctx-history-capture/src/provider_sources/append_import/completion.rs b/crates/ctx-history-capture/src/provider_sources/append_import/completion.rs index cd77b409..a8700925 100644 --- a/crates/ctx-history-capture/src/provider_sources/append_import/completion.rs +++ b/crates/ctx-history-capture/src/provider_sources/append_import/completion.rs @@ -1,3 +1,19 @@ +fn certify_uncheckpointed_replacement( + reader: &mut ProviderJsonlReader, + observed_size: u64, + is_replacement: bool, + decision: &mut ProviderAppendFileImportDecision, +) { + let ProviderAppendFileImportDecision::ImportedWithoutCheckpoint(result) = decision else { + return; + }; + if is_replacement && result.summary.has_accepted_content() { + // Materialization is already committed. A hash failure stays encoded as + // a missing certificate so the coordinator retires rather than publishes it. + result.source_prefix_sha256 = reader.pinned_prefix_sha256(observed_size).ok(); + } +} + fn unexpected_post_commit_checkpoint_failure( error: &CaptureError, ) -> ProviderJsonlReplacementReason { @@ -310,19 +326,7 @@ pub fn provider_canonical_material_source_format( provider: CaptureProvider, inventory_source_format: &str, ) -> Option<&'static str> { - match (provider, inventory_source_format) { - (CaptureProvider::Codex, "codex_session_jsonl_tree" | "codex_session_jsonl") => { - Some(CODEX_SESSION_SOURCE_FORMAT) - } - (CaptureProvider::Pi, "pi_session_jsonl") => Some("pi_session_jsonl"), - (CaptureProvider::Claude, "claude_projects_jsonl_tree") => { - Some(CLAUDE_PROJECTS_SOURCE_FORMAT) - } - (CaptureProvider::Tabnine, "tabnine_cli_chat_recording_jsonl") => { - Some(TABNINE_CLI_SOURCE_FORMAT) - } - _ => None, - } + canonical_provider_material_source_format(provider, inventory_source_format) } fn finish_import( @@ -338,13 +342,21 @@ fn finish_import( Ok(checkpoint) => checkpoint, Err(reason) => { return ProviderAppendFileImportDecision::ImportedWithoutCheckpoint( - ProviderAppendFileImportWithoutCheckpoint { summary, reason }, + ProviderAppendFileImportWithoutCheckpoint { + summary, + reason, + source_prefix_sha256: None, + }, ); } }; if let Some(reason) = certification_failure { return ProviderAppendFileImportDecision::ImportedWithoutCheckpoint( - ProviderAppendFileImportWithoutCheckpoint { summary, reason }, + ProviderAppendFileImportWithoutCheckpoint { + summary, + reason, + source_prefix_sha256: None, + }, ); } checkpoint.resume_state = resume_state; diff --git a/crates/ctx-history-capture/src/provider_sources/append_import/tests.rs b/crates/ctx-history-capture/src/provider_sources/append_import/tests.rs index b19fe7e3..6d24f548 100644 --- a/crates/ctx-history-capture/src/provider_sources/append_import/tests.rs +++ b/crates/ctx-history-capture/src/provider_sources/append_import/tests.rs @@ -6,6 +6,7 @@ mod tests { }; use serde_json::json; + use sha2::{Digest, Sha256}; use tempfile::tempdir; use super::*; @@ -28,6 +29,7 @@ mod tests { source_root: source_path.parent().unwrap().to_path_buf(), imported_at: "2026-07-14T12:00:00Z".parse().unwrap(), history_record_id: None, + observed_size: fs::metadata(source_path).unwrap().len(), mode, } } diff --git a/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_a.rs b/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_a.rs index 6dd8710f..95e62851 100644 --- a/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_a.rs +++ b/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_a.rs @@ -29,6 +29,13 @@ fn canonical_material_formats_preserve_existing_capture_identities() { ), Some("tabnine_cli_chat_recording_jsonl") ); + assert_eq!( + provider_canonical_material_source_format( + CaptureProvider::MistralVibe, + "mistral_vibe_session_jsonl_tree" + ), + Some("mistral_vibe_session_jsonl") + ); assert_eq!( provider_file_mutation_contract(CaptureProvider::Codex, "codex_history_jsonl"), ProviderFileMutationContract::WholeReplacement @@ -111,6 +118,7 @@ fn truncate_after_tolerant_materialization_dominates_invalid_header_certificatio ProviderAppendFileImportDecision::ImportedWithoutCheckpoint(result) => { assert_eq!(result.reason, ProviderJsonlReplacementReason::FileShrank); assert!(result.summary.imported_events > 0); + assert_eq!(result.source_prefix_sha256, None); } other => panic!("expected mutation to dominate certification, got {other:?}"), } @@ -128,6 +136,10 @@ fn identity_swap_after_tolerant_materialization_dominates_invalid_header_certifi jsonl(codex_header("codex-replaced-after-import")), jsonl(codex_message("user", "materialized before replacement", 1)) ); + let replacement_contents = contents.replacen("materialized", "Materialized", 1); + assert_eq!(replacement_contents.len(), contents.len()); + let source_sha256 = format!("{:x}", Sha256::digest(contents.as_bytes())); + let replacement_sha256 = format!("{:x}", Sha256::digest(replacement_contents.as_bytes())); write_raw(&path, &contents); let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); @@ -142,7 +154,7 @@ fn identity_swap_after_tolerant_materialization_dominates_invalid_header_certifi ), |_| { fs::rename(&path, displaced).unwrap(); - fs::write(&path, contents).unwrap(); + fs::write(&path, replacement_contents).unwrap(); }, ) .unwrap(); @@ -154,6 +166,14 @@ fn identity_swap_after_tolerant_materialization_dominates_invalid_header_certifi ProviderJsonlReplacementReason::StableIdentityChanged ); assert!(result.summary.imported_events > 0); + assert_eq!( + result.source_prefix_sha256.as_deref(), + Some(source_sha256.as_str()) + ); + assert_ne!( + result.source_prefix_sha256.as_deref(), + Some(replacement_sha256.as_str()) + ); } other => panic!("expected mutation to dominate certification, got {other:?}"), } @@ -319,54 +339,54 @@ fn unexpected_checkpoint_permanent_error_after_commit_is_typed_and_retains_summa #[test] fn replacement_header_plus_partial_first_message_is_deferred() { let cases = [ - ( - CaptureProvider::Codex, - "codex_session_jsonl_tree", - "codex_session_jsonl", - format!( - "{}{{\"timestamp\":\"2026-07-14T12:00:01Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"message\",\"role\":\"user\",\"content\":[", - jsonl(codex_header("codex-partial")) + ( + CaptureProvider::Codex, + "codex_session_jsonl_tree", + "codex_session_jsonl", + format!( + "{}{{\"timestamp\":\"2026-07-14T12:00:01Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"message\",\"role\":\"user\",\"content\":[", + jsonl(codex_header("codex-partial")) + ), ), - ), - ( - CaptureProvider::Pi, - "pi_session_jsonl", - "pi_session_jsonl", - format!( - "{}{{\"type\":\"message\",\"id\":\"pi-user", - jsonl(json!({ - "type": "session", - "id": "pi-partial", - "timestamp": "2026-07-14T12:00:00Z" - })) + ( + CaptureProvider::Pi, + "pi_session_jsonl", + "pi_session_jsonl", + format!( + "{}{{\"type\":\"message\",\"id\":\"pi-user", + jsonl(json!({ + "type": "session", + "id": "pi-partial", + "timestamp": "2026-07-14T12:00:00Z" + })) + ), ), - ), - ( - CaptureProvider::Claude, - "claude_projects_jsonl_tree", - "claude_projects_jsonl_tree", - format!( - "{}{{\"sessionId\":\"claude-partial\",\"type\":\"user\",\"message\":", - jsonl(json!({ - "sessionId": "claude-partial", - "timestamp": "2026-07-14T12:00:00Z", - "type": "system" - })) + ( + CaptureProvider::Claude, + "claude_projects_jsonl_tree", + "claude_projects_jsonl_tree", + format!( + "{}{{\"sessionId\":\"claude-partial\",\"type\":\"user\",\"message\":", + jsonl(json!({ + "sessionId": "claude-partial", + "timestamp": "2026-07-14T12:00:00Z", + "type": "system" + })) + ), ), - ), - ( - CaptureProvider::Tabnine, - "tabnine_cli_chat_recording_jsonl", - "tabnine_cli_chat_recording_jsonl", - format!( - "{}{{\"id\":\"tabnine-user\",\"type\":\"user\",\"content\":", - jsonl(json!({ - "sessionId": "tabnine-partial", - "startTime": "2026-07-14T12:00:00Z" - })) + ( + CaptureProvider::Tabnine, + "tabnine_cli_chat_recording_jsonl", + "tabnine_cli_chat_recording_jsonl", + format!( + "{}{{\"id\":\"tabnine-user\",\"type\":\"user\",\"content\":", + jsonl(json!({ + "sessionId": "tabnine-partial", + "startTime": "2026-07-14T12:00:00Z" + })) + ), ), - ), - ]; + ]; for (index, (provider, inventory_format, material_format, contents)) in cases.into_iter().enumerate() @@ -394,6 +414,72 @@ fn replacement_header_plus_partial_first_message_is_deferred() { } } +#[test] +fn append_import_stops_at_the_inventoried_size_when_the_file_grows() { + let temp = tempdir().unwrap(); + let path = temp.path().join("codex/growing.jsonl"); + write_raw( + &path, + &format!( + "{}{}", + jsonl(codex_header("codex-growing")), + jsonl(codex_message("user", "inventoried message", 1)) + ), + ); + let replacement_options = options( + &path, + "codex_session_jsonl_tree", + "codex_session_jsonl", + ProviderAppendFileImportMode::AppendCapableReplacement, + ); + let inventoried_size = replacement_options.observed_size; + append_raw( + &path, + &jsonl(codex_message( + "assistant", + "message appended after inventory", + 2, + )), + ); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let first = imported( + import_append_capable_provider_file( + CaptureProvider::Codex, + &mut store, + replacement_options, + ) + .unwrap(), + ); + assert_eq!(first.summary.imported_events, 1); + assert_eq!(first.checkpoint.committed_offset, inventoried_size); + let first_archive = serde_json::to_string(&store.export_archive().unwrap()).unwrap(); + assert!(first_archive.contains("inventoried message")); + assert!(!first_archive.contains("message appended after inventory")); + + let second = imported( + import_append_capable_provider_file( + CaptureProvider::Codex, + &mut store, + options( + &path, + "codex_session_jsonl_tree", + "codex_session_jsonl", + admitted(first.checkpoint), + ), + ) + .unwrap(), + ); + assert_eq!(second.summary.imported_events, 1); + assert_eq!( + second.checkpoint.committed_offset, + fs::metadata(&path).unwrap().len() + ); + assert!(serde_json::to_string(&store.export_archive().unwrap()) + .unwrap() + .contains("message appended after inventory")); +} + #[test] fn codex_permanent_orphan_keeps_future_large_appends_delta_bounded() { let temp = tempdir().unwrap(); diff --git a/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_b.rs b/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_b.rs index 6a610c3a..67f8de35 100644 --- a/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_b.rs +++ b/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_b.rs @@ -26,13 +26,13 @@ fn codex_append_rejects_a_second_session_header_before_commit() { ); let event_count = store.export_archive().unwrap().events.len(); append_raw( - &path, - &format!( - "{}{}", - jsonl(codex_message("assistant", "must roll back", 2)), - "{\"type\": \"session_meta\", \"payload\": {\"id\": \"codex-second\", \"timestamp\": \"2026-07-14T12:00:03Z\"}}\n" - ), - ); + &path, + &format!( + "{}{}", + jsonl(codex_message("assistant", "must roll back", 2)), + "{\"type\": \"session_meta\", \"payload\": {\"id\": \"codex-second\", \"timestamp\": \"2026-07-14T12:00:03Z\"}}\n" + ), + ); let decision = import_append_capable_provider_file( CaptureProvider::Codex, &mut store, @@ -942,3 +942,230 @@ fn claude_and_tabnine_authoritative_identity_changes_are_not_append_admitted() { )); } } + +#[test] +fn each_operational_format_streams_delta_and_handles_malformed_partial_and_rewrite() { + struct Case { + provider: CaptureProvider, + inventory_format: &'static str, + material_format: &'static str, + initial: String, + tail: String, + malformed: &'static str, + } + + let cases = vec![ + Case { + provider: CaptureProvider::Codex, + inventory_format: "codex_session_jsonl_tree", + material_format: "codex_session_jsonl", + initial: format!( + "{}{}", + jsonl(codex_header("codex-tree-delta")), + jsonl(codex_message("user", "codex tree initial", 1)) + ), + tail: jsonl(codex_message("assistant", "codex tree delta", 2)), + malformed: "{\"type\":\"response_item\",oops}\n", + }, + Case { + provider: CaptureProvider::Codex, + inventory_format: "codex_session_jsonl", + material_format: "codex_session_jsonl", + initial: format!( + "{}{}", + jsonl(codex_header("codex-file-delta")), + jsonl(codex_message("user", "codex file initial", 1)) + ), + tail: jsonl(codex_message("assistant", "codex file delta", 2)), + malformed: "{\"type\":\"response_item\",oops}\n", + }, + Case { + provider: CaptureProvider::Pi, + inventory_format: "pi_session_jsonl", + material_format: "pi_session_jsonl", + initial: format!( + "{}{}", + jsonl(json!({ + "type": "session", + "id": "pi-delta", + "timestamp": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "type": "message", + "id": "pi-initial", + "timestamp": "2026-07-14T12:00:01Z", + "message": {"role": "user", "content": "pi initial"} + })) + ), + tail: jsonl(json!({ + "type": "model_change", + "id": "pi-tail", + "timestamp": "2026-07-14T12:00:02Z", + "provider": "test", + "model": "test-model" + })), + malformed: "{oops}\n", + }, + Case { + provider: CaptureProvider::Claude, + inventory_format: "claude_projects_jsonl_tree", + material_format: "claude_projects_jsonl_tree", + initial: jsonl(json!({ + "sessionId": "claude-delta", + "timestamp": "2026-07-14T12:00:00Z", + "type": "user", + "uuid": "claude-initial", + "message": {"role": "user", "content": "claude initial"} + })), + tail: jsonl(json!({ + "sessionId": "claude-delta", + "timestamp": "2026-07-14T12:00:01Z", + "type": "progress", + "uuid": "claude-tail" + })), + malformed: "{oops}\n", + }, + Case { + provider: CaptureProvider::Tabnine, + inventory_format: "tabnine_cli_chat_recording_jsonl", + material_format: "tabnine_cli_chat_recording_jsonl", + initial: format!( + "{}{}", + jsonl(json!({ + "sessionId": "tabnine-delta", + "startTime": "2026-07-14T12:00:00Z" + })), + jsonl(json!({ + "id": "tabnine-initial", + "timestamp": "2026-07-14T12:00:01Z", + "type": "user", + "content": "tabnine initial" + })) + ), + tail: jsonl(json!({ + "id": "tabnine-tail", + "timestamp": "2026-07-14T12:00:02Z", + "type": "tabnine", + "toolCalls": [{ + "id": "tabnine-call", + "name": "read_file", + "args": {"file_path": "src/lib.rs"} + }] + })), + malformed: "{oops}\n", + }, + ]; + + for (index, case) in cases.into_iter().enumerate() { + let temp = tempdir().unwrap(); + let path = temp.path().join(format!("case-{index}/session.jsonl")); + write_raw(&path, &case.initial); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let initial = imported( + import_append_capable_provider_file( + case.provider, + &mut store, + options( + &path, + case.inventory_format, + case.material_format, + ProviderAppendFileImportMode::AppendCapableReplacement, + ), + ) + .unwrap(), + ); + assert_eq!(initial.summary.failed, 0, "{case:?}", case = case.provider); + let source = store + .list_capture_sources() + .unwrap() + .into_iter() + .find(|source| { + source.descriptor.raw_source_path.as_deref() + == Some(path.to_string_lossy().as_ref()) + }) + .expect("material capture source"); + assert_eq!( + source.descriptor.source_format.as_deref(), + Some(case.material_format) + ); + assert_eq!( + source.descriptor.source_root.as_deref(), + Some(path.parent().unwrap().to_string_lossy().as_ref()) + ); + + append_raw(&path, &case.tail); + let delta = imported( + import_append_capable_provider_file( + case.provider, + &mut store, + options( + &path, + case.inventory_format, + case.material_format, + admitted(initial.checkpoint), + ), + ) + .unwrap(), + ); + assert_eq!(delta.summary.failed, 0, "{case:?}", case = case.provider); + assert_eq!( + delta.summary.imported_events, + 1, + "{case:?}", + case = case.provider + ); + + append_raw(&path, case.malformed); + let malformed = imported( + import_append_capable_provider_file( + case.provider, + &mut store, + options( + &path, + case.inventory_format, + case.material_format, + admitted(delta.checkpoint.clone()), + ), + ) + .unwrap(), + ); + assert_eq!( + malformed.checkpoint.complete_line_count, + delta.checkpoint.complete_line_count + 1, + "{case:?}", + case = case.provider + ); + + append_raw(&path, "{\"partial\""); + let partial = imported( + import_append_capable_provider_file( + case.provider, + &mut store, + options( + &path, + case.inventory_format, + case.material_format, + admitted(malformed.checkpoint.clone()), + ), + ) + .unwrap(), + ); + assert_eq!(partial.checkpoint, malformed.checkpoint); + + fs::write(&path, "rewritten\n").unwrap(); + assert!(matches!( + import_append_capable_provider_file( + case.provider, + &mut store, + options( + &path, + case.inventory_format, + case.material_format, + admitted(partial.checkpoint), + ), + ) + .unwrap(), + ProviderAppendFileImportDecision::ReplacementRequired(_) + )); + } +} diff --git a/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_c.rs b/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_c.rs index 0ac1241c..f9ed5ec9 100644 --- a/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_c.rs +++ b/crates/ctx-history-capture/src/provider_sources/append_import/tests/cases_c.rs @@ -1,230 +1,3 @@ -#[test] -fn each_operational_format_streams_delta_and_handles_malformed_partial_and_rewrite() { - struct Case { - provider: CaptureProvider, - inventory_format: &'static str, - material_format: &'static str, - initial: String, - tail: String, - malformed: &'static str, - } - - let cases = vec![ - Case { - provider: CaptureProvider::Codex, - inventory_format: "codex_session_jsonl_tree", - material_format: "codex_session_jsonl", - initial: format!( - "{}{}", - jsonl(codex_header("codex-tree-delta")), - jsonl(codex_message("user", "codex tree initial", 1)) - ), - tail: jsonl(codex_message("assistant", "codex tree delta", 2)), - malformed: "{\"type\":\"response_item\",oops}\n", - }, - Case { - provider: CaptureProvider::Codex, - inventory_format: "codex_session_jsonl", - material_format: "codex_session_jsonl", - initial: format!( - "{}{}", - jsonl(codex_header("codex-file-delta")), - jsonl(codex_message("user", "codex file initial", 1)) - ), - tail: jsonl(codex_message("assistant", "codex file delta", 2)), - malformed: "{\"type\":\"response_item\",oops}\n", - }, - Case { - provider: CaptureProvider::Pi, - inventory_format: "pi_session_jsonl", - material_format: "pi_session_jsonl", - initial: format!( - "{}{}", - jsonl(json!({ - "type": "session", - "id": "pi-delta", - "timestamp": "2026-07-14T12:00:00Z" - })), - jsonl(json!({ - "type": "message", - "id": "pi-initial", - "timestamp": "2026-07-14T12:00:01Z", - "message": {"role": "user", "content": "pi initial"} - })) - ), - tail: jsonl(json!({ - "type": "model_change", - "id": "pi-tail", - "timestamp": "2026-07-14T12:00:02Z", - "provider": "test", - "model": "test-model" - })), - malformed: "{oops}\n", - }, - Case { - provider: CaptureProvider::Claude, - inventory_format: "claude_projects_jsonl_tree", - material_format: "claude_projects_jsonl_tree", - initial: jsonl(json!({ - "sessionId": "claude-delta", - "timestamp": "2026-07-14T12:00:00Z", - "type": "user", - "uuid": "claude-initial", - "message": {"role": "user", "content": "claude initial"} - })), - tail: jsonl(json!({ - "sessionId": "claude-delta", - "timestamp": "2026-07-14T12:00:01Z", - "type": "progress", - "uuid": "claude-tail" - })), - malformed: "{oops}\n", - }, - Case { - provider: CaptureProvider::Tabnine, - inventory_format: "tabnine_cli_chat_recording_jsonl", - material_format: "tabnine_cli_chat_recording_jsonl", - initial: format!( - "{}{}", - jsonl(json!({ - "sessionId": "tabnine-delta", - "startTime": "2026-07-14T12:00:00Z" - })), - jsonl(json!({ - "id": "tabnine-initial", - "timestamp": "2026-07-14T12:00:01Z", - "type": "user", - "content": "tabnine initial" - })) - ), - tail: jsonl(json!({ - "id": "tabnine-tail", - "timestamp": "2026-07-14T12:00:02Z", - "type": "tabnine", - "toolCalls": [{ - "id": "tabnine-call", - "name": "read_file", - "args": {"file_path": "src/lib.rs"} - }] - })), - malformed: "{oops}\n", - }, - ]; - - for (index, case) in cases.into_iter().enumerate() { - let temp = tempdir().unwrap(); - let path = temp.path().join(format!("case-{index}/session.jsonl")); - write_raw(&path, &case.initial); - let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); - let initial = imported( - import_append_capable_provider_file( - case.provider, - &mut store, - options( - &path, - case.inventory_format, - case.material_format, - ProviderAppendFileImportMode::AppendCapableReplacement, - ), - ) - .unwrap(), - ); - assert_eq!(initial.summary.failed, 0, "{case:?}", case = case.provider); - let source = store - .list_capture_sources() - .unwrap() - .into_iter() - .find(|source| { - source.descriptor.raw_source_path.as_deref() - == Some(path.to_string_lossy().as_ref()) - }) - .expect("material capture source"); - assert_eq!( - source.descriptor.source_format.as_deref(), - Some(case.material_format) - ); - assert_eq!( - source.descriptor.source_root.as_deref(), - Some(path.parent().unwrap().to_string_lossy().as_ref()) - ); - - append_raw(&path, &case.tail); - let delta = imported( - import_append_capable_provider_file( - case.provider, - &mut store, - options( - &path, - case.inventory_format, - case.material_format, - admitted(initial.checkpoint), - ), - ) - .unwrap(), - ); - assert_eq!(delta.summary.failed, 0, "{case:?}", case = case.provider); - assert_eq!( - delta.summary.imported_events, - 1, - "{case:?}", - case = case.provider - ); - - append_raw(&path, case.malformed); - let malformed = imported( - import_append_capable_provider_file( - case.provider, - &mut store, - options( - &path, - case.inventory_format, - case.material_format, - admitted(delta.checkpoint.clone()), - ), - ) - .unwrap(), - ); - assert_eq!( - malformed.checkpoint.complete_line_count, - delta.checkpoint.complete_line_count + 1, - "{case:?}", - case = case.provider - ); - - append_raw(&path, "{\"partial\""); - let partial = imported( - import_append_capable_provider_file( - case.provider, - &mut store, - options( - &path, - case.inventory_format, - case.material_format, - admitted(malformed.checkpoint.clone()), - ), - ) - .unwrap(), - ); - assert_eq!(partial.checkpoint, malformed.checkpoint); - - fs::write(&path, "rewritten\n").unwrap(); - assert!(matches!( - import_append_capable_provider_file( - case.provider, - &mut store, - options( - &path, - case.inventory_format, - case.material_format, - admitted(partial.checkpoint), - ), - ) - .unwrap(), - ProviderAppendFileImportDecision::ReplacementRequired(_) - )); - } -} - #[test] fn large_pi_claude_and_tabnine_units_emit_bounded_normalization_batches() { const ROWS: usize = PROVIDER_NORMALIZATION_STREAM_BATCH_UNITS * 5 + 7; diff --git a/crates/ctx-history-capture/src/provider_sources/discovery.rs b/crates/ctx-history-capture/src/provider_sources/discovery.rs index 26cca2aa..7a239621 100644 --- a/crates/ctx-history-capture/src/provider_sources/discovery.rs +++ b/crates/ctx-history-capture/src/provider_sources/discovery.rs @@ -12,7 +12,7 @@ use super::{ mutation_contracts::provider_file_mutation_contract, probes::{default_location_import_probe, BoundedProbe}, reasons::{empty_source_reason, probe_io_error_reason, unknown_source_reason}, - revisions::provider_import_revision, + revisions::{provider_import_revision, PROVIDER_IMPORT_REVISIONS}, specs::{provider_source_spec, PROVIDER_SPECS, TRAE_STATE_VSCDB_SOURCE_FORMAT}, types::{ ProviderCatalogSupport, ProviderDefaultLocation, ProviderImportSupport, ProviderSource, @@ -933,54 +933,21 @@ pub fn provider_source_for_path(provider: CaptureProvider, path: PathBuf) -> Pro } } -fn provider_source_from_location( - spec: &ProviderSourceSpec, - location: &ProviderDefaultLocation, +pub fn provider_source_for_persisted_format( + provider: CaptureProvider, path: PathBuf, -) -> ProviderSource { - let path_exists = path.try_exists(); - let exists = path_exists.as_ref().copied().unwrap_or(true); - let (status, unsupported_reason) = - if matches!(spec.import_support, ProviderImportSupport::Unsupported) { - (ProviderSourceStatus::Unsupported, spec.unsupported_reason) - } else { - match path_exists { - Ok(false) => (ProviderSourceStatus::Missing, spec.unsupported_reason), - Err(_) => ( - ProviderSourceStatus::Unknown, - probe_io_error_reason(spec.provider), - ), - Ok(true) => match default_location_import_probe(spec.provider, location, &path) { - BoundedProbe::Found => { - (ProviderSourceStatus::Available, spec.unsupported_reason) - } - BoundedProbe::NotFound => ( - ProviderSourceStatus::Empty, - empty_source_reason(spec.provider), - ), - BoundedProbe::BudgetExhausted => ( - ProviderSourceStatus::Unknown, - unknown_source_reason(spec.provider), - ), - BoundedProbe::IoError => ( - ProviderSourceStatus::Unknown, - probe_io_error_reason(spec.provider), - ), - }, - } - }; - ProviderSource { - provider: spec.provider, - path, - exists, - source_format: location.source_format, - import_revision: provider_import_revision(spec.provider, location.source_format), - source_kind: location.source_kind, - import_support: spec.import_support, - catalog_support: spec.catalog_support, - import_unit: provider_import_unit_spec(location.source_format), - mutation_contract: provider_file_mutation_contract(spec.provider, location.source_format), - status, - unsupported_reason, - } + source_format: &str, +) -> Option { + let source_format = PROVIDER_IMPORT_REVISIONS + .iter() + .find(|entry| entry.provider == provider && entry.source_format == source_format)? + .source_format; + let mut source = provider_source_for_path(provider, path); + source.source_format = source_format; + source.import_revision = provider_import_revision(provider, source_format); + source.import_unit = provider_import_unit_spec(source_format); + source.mutation_contract = provider_file_mutation_contract(provider, source_format); + Some(source) } + +include!("discovery/source_from_location.rs"); diff --git a/crates/ctx-history-capture/src/provider_sources/discovery/source_from_location.rs b/crates/ctx-history-capture/src/provider_sources/discovery/source_from_location.rs new file mode 100644 index 00000000..53e3eae4 --- /dev/null +++ b/crates/ctx-history-capture/src/provider_sources/discovery/source_from_location.rs @@ -0,0 +1,51 @@ +fn provider_source_from_location( + spec: &ProviderSourceSpec, + location: &ProviderDefaultLocation, + path: PathBuf, +) -> ProviderSource { + let path_exists = path.try_exists(); + let exists = path_exists.as_ref().copied().unwrap_or(true); + let (status, unsupported_reason) = + if matches!(spec.import_support, ProviderImportSupport::Unsupported) { + (ProviderSourceStatus::Unsupported, spec.unsupported_reason) + } else { + match path_exists { + Ok(false) => (ProviderSourceStatus::Missing, spec.unsupported_reason), + Err(_) => ( + ProviderSourceStatus::Unknown, + probe_io_error_reason(spec.provider), + ), + Ok(true) => match default_location_import_probe(spec.provider, location, &path) { + BoundedProbe::Found => { + (ProviderSourceStatus::Available, spec.unsupported_reason) + } + BoundedProbe::NotFound => ( + ProviderSourceStatus::Empty, + empty_source_reason(spec.provider), + ), + BoundedProbe::BudgetExhausted => ( + ProviderSourceStatus::Unknown, + unknown_source_reason(spec.provider), + ), + BoundedProbe::IoError => ( + ProviderSourceStatus::Unknown, + probe_io_error_reason(spec.provider), + ), + }, + } + }; + ProviderSource { + provider: spec.provider, + path, + exists, + source_format: location.source_format, + import_revision: provider_import_revision(spec.provider, location.source_format), + source_kind: location.source_kind, + import_support: spec.import_support, + catalog_support: spec.catalog_support, + import_unit: provider_import_unit_spec(location.source_format), + mutation_contract: provider_file_mutation_contract(spec.provider, location.source_format), + status, + unsupported_reason, + } +} diff --git a/crates/ctx-history-capture/src/provider_sources/incremental_jsonl.rs b/crates/ctx-history-capture/src/provider_sources/incremental_jsonl.rs index 1e28ccc9..5e813b43 100644 --- a/crates/ctx-history-capture/src/provider_sources/incremental_jsonl.rs +++ b/crates/ctx-history-capture/src/provider_sources/incremental_jsonl.rs @@ -370,7 +370,7 @@ pub enum ProviderJsonlRecordRead { #[derive(Debug)] pub struct ProviderJsonlReader { path: PathBuf, - reader: BufReader, + reader: BufReader>, stable_identity: Option, defer_unterminated_tail: bool, initial_offset: u64, @@ -399,6 +399,75 @@ pub fn open_provider_jsonl( open_provider_jsonl_with_identity(path.as_ref(), mode, stable_file_identity) } +pub fn provider_jsonl_checkpoint_matches_file( + path: impl AsRef, + checkpoint: &ProviderJsonlAppendCheckpoint, +) -> Result { + let mut file = open_regular_provider_transcript_file(path.as_ref())?; + let metadata = file.metadata()?; + let stable_identity = stable_file_identity(&file, &metadata); + Ok(provider_jsonl_checkpoint_replacement_reason( + &mut file, + &metadata, + stable_identity.as_ref(), + checkpoint, + false, + )? + .is_none()) +} + +fn provider_jsonl_checkpoint_replacement_reason( + file: &mut File, + metadata: &std::fs::Metadata, + stable_identity: Option<&ProviderFileStableIdentity>, + checkpoint: &ProviderJsonlAppendCheckpoint, + require_growth: bool, +) -> Result> { + if checkpoint.version != CHECKPOINT_VERSION { + return Ok(Some( + ProviderJsonlReplacementReason::UnsupportedCheckpointVersion, + )); + } + let Some(actual_identity) = stable_identity else { + return Ok(Some( + ProviderJsonlReplacementReason::StableIdentityUnavailable, + )); + }; + if actual_identity != &checkpoint.stable_identity { + return Ok(Some(ProviderJsonlReplacementReason::StableIdentityChanged)); + } + if metadata.len() < checkpoint.committed_offset { + return Ok(Some(ProviderJsonlReplacementReason::FileShrank)); + } + if require_growth && metadata.len() == checkpoint.committed_offset { + return Ok(Some(ProviderJsonlReplacementReason::EqualLengthObservation)); + } + if checkpoint.committed_offset > 0 { + let Some(boundary_byte) = read_range_exact(file, checkpoint.committed_offset - 1, 1)? + else { + return Ok(Some(ProviderJsonlReplacementReason::FileShrank)); + }; + if boundary_byte != b"\n" { + return Ok(Some( + ProviderJsonlReplacementReason::BoundaryNotNewlineAligned, + )); + } + } + let Some(head_sha256) = hash_head_exact(file, checkpoint.committed_offset)? else { + return Ok(Some(ProviderJsonlReplacementReason::FileShrank)); + }; + if head_sha256 != checkpoint.head_sha256 { + return Ok(Some(ProviderJsonlReplacementReason::HeadHashMismatch)); + } + let Some(boundary_sha256) = hash_boundary_exact(file, checkpoint.committed_offset)? else { + return Ok(Some(ProviderJsonlReplacementReason::FileShrank)); + }; + if boundary_sha256 != checkpoint.boundary_sha256 { + return Ok(Some(ProviderJsonlReplacementReason::BoundaryHashMismatch)); + } + Ok(None) +} + fn open_provider_jsonl_with_identity( path: &Path, mode: ProviderJsonlOpenMode, @@ -421,70 +490,18 @@ fn open_provider_jsonl_with_identity( ProviderJsonlOpenMode::WholeReplacement => (false, 0, 0), ProviderJsonlOpenMode::AppendCapableReplacement => (true, 0, 0), ProviderJsonlOpenMode::Append(checkpoint) => { - if checkpoint.version != CHECKPOINT_VERSION { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::UnsupportedCheckpointVersion, - )); - } - let Some(actual_identity) = stable_identity.as_ref() else { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::StableIdentityUnavailable, - )); - }; - if actual_identity != &checkpoint.stable_identity { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::StableIdentityChanged, - )); - } - if metadata.len() < checkpoint.committed_offset { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::FileShrank, - )); - } // The coordinator owns exact unchanged-observation elision. Once // capture is invoked, equality is ambiguous: it can be an // unchanged file or an equal-length rewrite outside the bounded // sentinels, but it is never evidence of an append delta. - if metadata.len() == checkpoint.committed_offset { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::EqualLengthObservation, - )); - } - if checkpoint.committed_offset > 0 { - let Some(boundary_byte) = - read_range_exact(&mut file, checkpoint.committed_offset - 1, 1)? - else { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::FileShrank, - )); - }; - if boundary_byte != b"\n" { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::BoundaryNotNewlineAligned, - )); - } - } - let Some(head_sha256) = hash_head_exact(&mut file, checkpoint.committed_offset)? else { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::FileShrank, - )); - }; - if head_sha256 != checkpoint.head_sha256 { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::HeadHashMismatch, - )); - } - let Some(boundary_sha256) = - hash_boundary_exact(&mut file, checkpoint.committed_offset)? - else { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::FileShrank, - )); - }; - if boundary_sha256 != checkpoint.boundary_sha256 { - return Ok(ProviderJsonlOpenDecision::ReplacementRequired( - ProviderJsonlReplacementReason::BoundaryHashMismatch, - )); + if let Some(reason) = provider_jsonl_checkpoint_replacement_reason( + &mut file, + &metadata, + stable_identity.as_ref(), + &checkpoint, + true, + )? { + return Ok(ProviderJsonlOpenDecision::ReplacementRequired(reason)); } ( true, @@ -498,7 +515,7 @@ fn open_provider_jsonl_with_identity( Ok(ProviderJsonlOpenDecision::Ready(ProviderJsonlReader { path: path.to_path_buf(), - reader: BufReader::new(file), + reader: BufReader::new(crate::disk_io_pacing::PacedReader::new(file)), stable_identity, defer_unterminated_tail, initial_offset: start_offset, @@ -549,6 +566,28 @@ impl ProviderJsonlReader { self.deferred_partial } + pub(crate) fn pinned_prefix_sha256(&mut self, byte_count: u64) -> Result { + let logical_position = self.current_offset; + self.reader.seek(SeekFrom::Start(0))?; + let hash = hash_reader_prefix_exact(&mut self.reader, byte_count); + let restore = self.reader.seek(SeekFrom::Start(logical_position)); + restore?; + hash + } + + pub fn limit_to_observed_size(&mut self, observed_size: u64) -> Result<()> { + if observed_size < self.initial_offset { + return Err(crate::CaptureError::SystemInvariant( + "provider JSONL observation ends before the admitted checkpoint", + )); + } + self.read_limit = Some( + self.read_limit + .map_or(observed_size, |current| current.min(observed_size)), + ); + Ok(()) + } + /// Reads the bounded first complete record from the pinned file handle, then /// restores the reader to its validated import position without advancing /// checkpoint state. @@ -741,15 +780,16 @@ impl ProviderJsonlReader { ProviderJsonlReplacementReason::StableIdentityUnavailable, )); }; - let metadata = match self.reader.get_ref().metadata() { + let metadata = match self.reader.get_ref().get_ref().metadata() { Ok(metadata) => metadata, Err(_) => { return Ok(Err(ProviderJsonlReplacementReason::CheckpointMetadataIo)); } }; - if let Err(reason) = - validate_stable_identity(&stable_identity, identity(self.reader.get_ref(), &metadata)) - { + if let Err(reason) = validate_stable_identity( + &stable_identity, + identity(self.reader.get_ref().get_ref(), &metadata), + ) { return Ok(Err(reason)); } match validate_path_identity(&self.path, &stable_identity, &identity) { @@ -768,7 +808,7 @@ impl ProviderJsonlReader { let logical_position = self.current_offset; let validated_checkpoint = self.validated_checkpoint.clone(); let hashes = { - let file = self.reader.get_mut(); + let file = self.reader.get_mut().get_mut(); checkpoint_hashes(file, committed_offset, validated_checkpoint.as_ref()) }; if self.reader.seek(SeekFrom::Start(logical_position)).is_err() { @@ -780,15 +820,16 @@ impl ProviderJsonlReader { Ok(Err(reason)) => return Ok(Err(reason)), }; - let metadata = match self.reader.get_ref().metadata() { + let metadata = match self.reader.get_ref().get_ref().metadata() { Ok(metadata) => metadata, Err(_) => { return Ok(Err(ProviderJsonlReplacementReason::CheckpointMetadataIo)); } }; - if let Err(reason) = - validate_stable_identity(&stable_identity, identity(self.reader.get_ref(), &metadata)) - { + if let Err(reason) = validate_stable_identity( + &stable_identity, + identity(self.reader.get_ref().get_ref(), &metadata), + ) { return Ok(Err(reason)); } match validate_path_identity(&self.path, &stable_identity, &identity) { @@ -819,6 +860,27 @@ impl ProviderJsonlReader { } } +fn hash_reader_prefix_exact(reader: &mut impl Read, byte_count: u64) -> Result { + let mut hasher = Sha256::new(); + let mut remaining = byte_count; + let mut buffer = [0_u8; 8192]; + while remaining > 0 { + let to_read = buffer.len().min(remaining as usize); + let read = reader.read(&mut buffer[..to_read])?; + if read == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("provider JSONL ended before byte offset {byte_count}"), + ) + .into()); + } + hasher.update(&buffer[..read]); + remaining -= read as u64; + } + Ok(format!("{:x}", hasher.finalize())) +} + include!("incremental_jsonl/identity.rs"); + #[cfg(test)] include!("incremental_jsonl/tests.rs"); diff --git a/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/identity.rs b/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/identity.rs index 81f61687..6b781802 100644 --- a/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/identity.rs +++ b/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/identity.rs @@ -103,6 +103,7 @@ fn hash_range_exact(file: &mut File, start: u64, bytes: u64) -> Result Result>> { file.seek(SeekFrom::Start(start))?; let mut value = vec![0; bytes.min(usize::MAX as u64) as usize]; + crate::pace_current_disk_io(value.len() as u64); match file.read_exact(&mut value) { Ok(()) => Ok(Some(value)), Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => Ok(None), diff --git a/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/tests.rs b/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/tests.rs index b1787398..ec15f7e9 100644 --- a/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/tests.rs +++ b/crates/ctx-history-capture/src/provider_sources/incremental_jsonl/tests.rs @@ -48,12 +48,17 @@ mod tests { fs::write(&path, b"one\ntwo\n").unwrap(); let checkpoint = initial_checkpoint(&path); + assert!(provider_jsonl_checkpoint_matches_file(&path, &checkpoint).unwrap()); + assert!(matches!( - open_provider_jsonl(&path, ProviderJsonlOpenMode::Append(checkpoint)).unwrap(), + open_provider_jsonl(&path, ProviderJsonlOpenMode::Append(checkpoint.clone())).unwrap(), ProviderJsonlOpenDecision::ReplacementRequired( ProviderJsonlReplacementReason::EqualLengthObservation ) )); + + fs::write(&path, b"uno\ndos\n").unwrap(); + assert!(!provider_jsonl_checkpoint_matches_file(&path, &checkpoint).unwrap()); } #[test] diff --git a/crates/ctx-history-capture/src/provider_sources/mod.rs b/crates/ctx-history-capture/src/provider_sources/mod.rs index da3c6b13..6417a3ae 100644 --- a/crates/ctx-history-capture/src/provider_sources/mod.rs +++ b/crates/ctx-history-capture/src/provider_sources/mod.rs @@ -3,6 +3,7 @@ mod discovery; mod import_units; mod incremental_jsonl; mod mutation_contracts; +mod ordinary_file; mod probes; mod reasons; mod revisions; @@ -11,17 +12,21 @@ mod types; pub use discovery::{ discover_provider_sources, discover_provider_sources_for_provider, provider_source_for_path, + provider_source_for_persisted_format, }; pub use incremental_jsonl::{ - open_provider_jsonl, ClaudeProjectsJsonlResumeState, CodexSessionJsonlResumeState, - CodexToolCallResumeContext, ProviderFileStableIdentity, ProviderJsonlAppendCheckpoint, - ProviderJsonlOpenDecision, ProviderJsonlOpenMode, ProviderJsonlReader, ProviderJsonlRecordRead, - ProviderJsonlReplacementReason, ProviderJsonlResumeState, TabnineJsonlResumeState, + open_provider_jsonl, provider_jsonl_checkpoint_matches_file, ClaudeProjectsJsonlResumeState, + CodexSessionJsonlResumeState, CodexToolCallResumeContext, ProviderFileStableIdentity, + ProviderJsonlAppendCheckpoint, ProviderJsonlOpenDecision, ProviderJsonlOpenMode, + ProviderJsonlReader, ProviderJsonlRecordRead, ProviderJsonlReplacementReason, + ProviderJsonlResumeState, TabnineJsonlResumeState, }; pub(crate) use incremental_jsonl::{ CODEX_RESUME_MAX_ENCODED_BYTES, CODEX_RESUME_MAX_PENDING_TOOL_CALLS, }; pub use mutation_contracts::provider_file_mutation_contract; +pub(crate) use ordinary_file::open_observed_ordinary_file; +pub use ordinary_file::{observe_ordinary_file, OrdinaryFileObservation}; pub use revisions::{ provider_import_revision, ProviderImportRevision, DEFAULT_PROVIDER_IMPORT_REVISION, PROVIDER_IMPORT_REVISIONS, diff --git a/crates/ctx-history-capture/src/provider_sources/ordinary_file.rs b/crates/ctx-history-capture/src/provider_sources/ordinary_file.rs new file mode 100644 index 00000000..351670f7 --- /dev/null +++ b/crates/ctx-history-capture/src/provider_sources/ordinary_file.rs @@ -0,0 +1,260 @@ +use std::{ + collections::BTreeSet, + fs::{File, Metadata}, + io::{Read, Seek, SeekFrom}, + path::Path, + time::{SystemTime, UNIX_EPOCH}, +}; + +use sha2::{Digest, Sha256}; + +use crate::{common::io::open_regular_provider_transcript_file, pace_current_disk_io, Result}; + +const TOKEN_DOMAIN: &[u8] = b"ctx-ordinary-file-observation-v1\0"; +const FULL_CONTENT_FINGERPRINT_MAX_BYTES: u64 = 64 * 1024; +const SPARSE_SAMPLE_BYTES: u64 = 8 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OrdinaryFileObservation { + len: u64, + modified_at: SystemTime, + token: [u8; 32], +} + +impl OrdinaryFileObservation { + pub fn len(&self) -> u64 { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn modified_at(&self) -> SystemTime { + self.modified_at + } + + pub fn token(&self) -> &[u8; 32] { + &self.token + } + + pub fn token_hex(&self) -> String { + self.token + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() + } +} + +pub fn observe_ordinary_file(path: impl AsRef) -> Result { + let (_, observation) = open_observed_ordinary_file(path.as_ref())?; + Ok(observation) +} + +pub(crate) fn open_observed_ordinary_file(path: &Path) -> Result<(File, OrdinaryFileObservation)> { + let mut file = open_regular_provider_transcript_file(path)?; + let metadata = file.metadata()?; + let platform_before = platform_token(&file, &metadata); + let content_fingerprint = content_fingerprint(&mut file, &metadata)?; + let current = file.metadata()?; + let platform_after = platform_token(&file, ¤t); + if current.len() != metadata.len() + || current.modified().ok() != metadata.modified().ok() + || platform_after != platform_before + { + return Err(file_changed_during_observation().into()); + } + let token = combined_token(platform_before, content_fingerprint); + file.seek(SeekFrom::Start(0))?; + Ok(( + file, + OrdinaryFileObservation { + len: metadata.len(), + modified_at: metadata.modified().unwrap_or(UNIX_EPOCH), + token, + }, + )) +} + +#[cfg(unix)] +fn platform_token(_file: &File, metadata: &Metadata) -> Option<[u8; 32]> { + use std::os::unix::fs::MetadataExt; + + let mut hasher = Sha256::new(); + hasher.update(TOKEN_DOMAIN); + hasher.update(b"unix\0"); + hasher.update(metadata.dev().to_le_bytes()); + hasher.update(metadata.ino().to_le_bytes()); + hasher.update(metadata.ctime().to_le_bytes()); + hasher.update(metadata.ctime_nsec().to_le_bytes()); + Some(hasher.finalize().into()) +} + +#[cfg(windows)] +fn platform_token(file: &File, _metadata: &Metadata) -> Option<[u8; 32]> { + use std::{mem::MaybeUninit, os::windows::io::AsRawHandle}; + use windows_sys::Win32::Storage::FileSystem::{ + FileBasicInfo, FileIdInfo, GetFileInformationByHandleEx, FILE_BASIC_INFO, FILE_ID_INFO, + }; + + let handle = file.as_raw_handle() as windows_sys::Win32::Foundation::HANDLE; + let mut identity = MaybeUninit::::zeroed(); + let identity_ok = unsafe { + GetFileInformationByHandleEx( + handle, + FileIdInfo, + identity.as_mut_ptr().cast(), + std::mem::size_of::() as u32, + ) + }; + let mut basic = MaybeUninit::::zeroed(); + let basic_ok = unsafe { + GetFileInformationByHandleEx( + handle, + FileBasicInfo, + basic.as_mut_ptr().cast(), + std::mem::size_of::() as u32, + ) + }; + if identity_ok == 0 || basic_ok == 0 { + return None; + } + let identity = unsafe { identity.assume_init() }; + let basic = unsafe { basic.assume_init() }; + let mut hasher = Sha256::new(); + hasher.update(TOKEN_DOMAIN); + hasher.update(b"windows\0"); + hasher.update(identity.VolumeSerialNumber.to_le_bytes()); + hasher.update(identity.FileId.Identifier); + hasher.update(basic.ChangeTime.to_le_bytes()); + Some(hasher.finalize().into()) +} + +#[cfg(not(any(unix, windows)))] +fn platform_token(_file: &File, _metadata: &Metadata) -> Option<[u8; 32]> { + None +} + +fn combined_token(platform_token: Option<[u8; 32]>, content_fingerprint: [u8; 32]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(TOKEN_DOMAIN); + if let Some(platform_token) = platform_token { + hasher.update(b"platform\0"); + hasher.update(platform_token); + } else { + hasher.update(b"portable\0"); + } + hasher.update(content_fingerprint); + hasher.finalize().into() +} + +fn content_fingerprint(file: &mut File, metadata: &Metadata) -> Result<[u8; 32]> { + let len = metadata.len(); + let mut hasher = Sha256::new(); + hasher.update(TOKEN_DOMAIN); + hasher.update(len.to_le_bytes()); + if len <= FULL_CONTENT_FINGERPRINT_MAX_BYTES { + hasher.update(b"full\0"); + hash_range(file, 0, len, &mut hasher)?; + } else { + hasher.update(b"sparse\0"); + for offset in sparse_sample_offsets(len) { + let sample_len = SPARSE_SAMPLE_BYTES.min(len.saturating_sub(offset)); + hasher.update(offset.to_le_bytes()); + hasher.update(sample_len.to_le_bytes()); + hash_range(file, offset, sample_len, &mut hasher)?; + } + } + Ok(hasher.finalize().into()) +} + +fn sparse_sample_offsets(len: u64) -> BTreeSet { + let last = len.saturating_sub(SPARSE_SAMPLE_BYTES); + [0, len / 4, len / 2, len.saturating_mul(3) / 4, last] + .into_iter() + .map(|offset| offset.min(last)) + .collect() +} + +fn hash_range(file: &mut File, offset: u64, len: u64, hasher: &mut Sha256) -> std::io::Result<()> { + file.seek(SeekFrom::Start(offset))?; + let mut buffer = [0_u8; 64 * 1024]; + let mut remaining = len; + while remaining > 0 { + let limit = usize::try_from(remaining.min(buffer.len() as u64)).unwrap_or(buffer.len()); + pace_current_disk_io(limit as u64); + let read = file.read(&mut buffer[..limit])?; + if read == 0 { + return Err(file_changed_during_observation()); + } + hasher.update(&buffer[..read]); + remaining = remaining.saturating_sub(read as u64); + } + Ok(()) +} + +fn file_changed_during_observation() -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "ordinary file changed while it was being observed", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_fallback_hashes_the_complete_file() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("source.jsonl"); + std::fs::write(&path, b"alpha-middle-omega").unwrap(); + let mut file = open_regular_provider_transcript_file(&path).unwrap(); + let metadata = file.metadata().unwrap(); + let first = content_fingerprint(&mut file, &metadata).unwrap(); + + std::fs::write(&path, b"alpha-switch-omega").unwrap(); + let mut file = open_regular_provider_transcript_file(&path).unwrap(); + let metadata = file.metadata().unwrap(); + let second = content_fingerprint(&mut file, &metadata).unwrap(); + + assert_ne!(first, second); + } + + #[cfg(unix)] + #[test] + fn unix_token_detects_same_size_rewrite_with_restored_mtime() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("source.jsonl"); + std::fs::write(&path, b"alpha\n").unwrap(); + let original_modified = std::fs::metadata(&path).unwrap().modified().unwrap(); + let first = observe_ordinary_file(&path).unwrap(); + + std::fs::write(&path, b"omega\n").unwrap(); + File::options() + .write(true) + .open(&path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(original_modified)) + .unwrap(); + let second = observe_ordinary_file(&path).unwrap(); + + assert_eq!(first.len(), second.len()); + assert_eq!(first.modified_at(), second.modified_at()); + assert_ne!(first.token(), second.token()); + } + + #[cfg(unix)] + #[test] + fn observation_rejects_a_symlinked_final_component() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let target = temp.path().join("target.jsonl"); + let link = temp.path().join("link.jsonl"); + std::fs::write(&target, b"content\n").unwrap(); + symlink(&target, &link).unwrap(); + + assert!(observe_ordinary_file(&link).is_err()); + } +} diff --git a/crates/ctx-history-capture/src/provider_sources/probes.rs b/crates/ctx-history-capture/src/provider_sources/probes.rs index e43204fa..76d699c5 100644 --- a/crates/ctx-history-capture/src/provider_sources/probes.rs +++ b/crates/ctx-history-capture/src/provider_sources/probes.rs @@ -181,7 +181,7 @@ fn has_junie_session_events(root: &Path, max_entries: usize) -> BoundedProbe { Ok(file) => file, Err(_) => return BoundedProbe::IoError, }; - let mut reader = BufReader::new(file); + let mut reader = BufReader::new(crate::disk_io_pacing::PacedReader::new(file)); let mut line = Vec::new(); let mut visited = 0usize; loop { diff --git a/crates/ctx-history-capture/src/provider_sources/tests/revisions.rs b/crates/ctx-history-capture/src/provider_sources/tests/revisions.rs index ea2320ed..02e9ba63 100644 --- a/crates/ctx-history-capture/src/provider_sources/tests/revisions.rs +++ b/crates/ctx-history-capture/src/provider_sources/tests/revisions.rs @@ -3,8 +3,8 @@ use std::collections::HashSet; use ctx_history_core::CaptureProvider; use super::super::{ - provider_import_revision, provider_source_specs, DEFAULT_PROVIDER_IMPORT_REVISION, - PROVIDER_IMPORT_REVISIONS, + provider_import_revision, provider_source_for_persisted_format, provider_source_specs, + DEFAULT_PROVIDER_IMPORT_REVISION, PROVIDER_IMPORT_REVISIONS, }; #[test] @@ -62,3 +62,25 @@ fn semantic_output_changes_bump_only_the_affected_material_formats() { DEFAULT_PROVIDER_IMPORT_REVISION ); } + +#[test] +fn persisted_tree_format_does_not_depend_on_a_missing_path_still_being_a_directory() { + let source = provider_source_for_persisted_format( + CaptureProvider::Qoder, + std::path::PathBuf::from("/missing/qoder-root"), + "qoder_transcript_jsonl_tree", + ) + .unwrap(); + + assert_eq!(source.source_format, "qoder_transcript_jsonl_tree"); + assert_eq!( + source.import_revision, + provider_import_revision(CaptureProvider::Qoder, "qoder_transcript_jsonl_tree") + ); + assert!(provider_source_for_persisted_format( + CaptureProvider::Qoder, + std::path::PathBuf::from("/missing/qoder-root"), + "codex_session_jsonl_tree", + ) + .is_none()); +} diff --git a/crates/ctx-history-capture/src/summaries.rs b/crates/ctx-history-capture/src/summaries.rs index 843f0a31..78d5131f 100644 --- a/crates/ctx-history-capture/src/summaries.rs +++ b/crates/ctx-history-capture/src/summaries.rs @@ -61,6 +61,7 @@ pub enum ProviderImportMaintenanceKind { TransactionContinuation, ImportInterruptedAfterCommit, EventSearchFinalization, + EventSearchFinalizationPending, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -97,6 +98,12 @@ impl ProviderImportSummary { !self.maintenance_warnings.is_empty() } + pub(crate) fn has_checkpoint_blocking_maintenance(&self) -> bool { + self.maintenance_warnings.iter().any(|warning| { + warning.kind != ProviderImportMaintenanceKind::EventSearchFinalizationPending + }) + } + pub(crate) fn push_maintenance_warning( &mut self, kind: ProviderImportMaintenanceKind, diff --git a/crates/ctx-history-capture/src/tests.rs b/crates/ctx-history-capture/src/tests.rs index 03e35920..04232c64 100644 --- a/crates/ctx-history-capture/src/tests.rs +++ b/crates/ctx-history-capture/src/tests.rs @@ -1,6 +1,8 @@ mod support; mod codex; +mod codex_catalog_pacing; +mod codex_observation; mod custom_history; mod hermes_batching; mod native_json; diff --git a/crates/ctx-history-capture/src/tests/codex.rs b/crates/ctx-history-capture/src/tests/codex.rs index b3507e2b..b5de2052 100644 --- a/crates/ctx-history-capture/src/tests/codex.rs +++ b/crates/ctx-history-capture/src/tests/codex.rs @@ -316,11 +316,11 @@ fn codex_catalog_cache_rejects_an_older_parser_revision() { .unwrap() .pop() .unwrap(); - let metadata = fs::metadata(&session.source_path).unwrap(); + let observation = crate::observe_ordinary_file(&session.source_path).unwrap(); assert!(cached_catalog_session_if_unchanged( Some(&session), - &metadata, + &observation, session.cataloged_at_ms + 1, session.import_revision + 1, ) @@ -332,7 +332,7 @@ fn codex_session_catalog_skips_oversized_metadata_probe_line() { let temp = tempdir(); let root = temp.path().join("sessions/2026/07/03"); fs::create_dir_all(&root).unwrap(); - write_oversized_jsonl_line(&root.join("oversized.jsonl")); + fs::write(root.join("oversized.jsonl"), oversized_jsonl_line()).unwrap(); let store = Store::open(temp.path().join("work.sqlite")).unwrap(); let summary = catalog_codex_session_tree( diff --git a/crates/ctx-history-capture/src/tests/codex_catalog_pacing.rs b/crates/ctx-history-capture/src/tests/codex_catalog_pacing.rs new file mode 100644 index 00000000..28ff536f --- /dev/null +++ b/crates/ctx-history-capture/src/tests/codex_catalog_pacing.rs @@ -0,0 +1,392 @@ +use ctx_history_core::{AgentType, CaptureProvider}; +use ctx_history_store::{CatalogSession, ImportWorkClass, Store}; +use serde_json::json; + +use crate::provider::codex::catalog::{ + catalog_parallelism, catalog_session_persist_bytes_for_test, + persist_catalog_sessions_for_publication_for_test, + persist_initial_catalog_sessions_bounded_for_test, +}; +use crate::{CaptureError, DiskIoPacer, CODEX_SESSION_SOURCE_FORMAT}; + +#[test] +fn daemon_pacing_caps_catalog_cpu_parallelism() { + let _pacing = crate::install_disk_io_pacer(DiskIoPacer::new(8 * 1024 * 1024, 1024 * 1024)); + + assert_eq!(catalog_parallelism(100, Some(32)), 2); +} + +#[test] +fn initial_codex_catalog_persistence_is_bounded_paced_and_atomically_published() { + let temp = tempfile::tempdir().unwrap(); + let source_root = "/home/user/.codex/sessions"; + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, source_root) + .unwrap(); + let sessions = (0..65) + .map(|index| catalog_persist_test_session(source_root, index)) + .collect::>(); + let expected_bytes = sessions + .iter() + .map(|session| catalog_session_persist_bytes_for_test(session).unwrap()) + .sum::(); + let pacer = DiskIoPacer::new(u64::MAX, u64::MAX); + let _pacing = crate::install_disk_io_pacer(pacer.clone()); + let mut committed = Vec::new(); + + assert!(persist_initial_catalog_sessions_bounded_for_test( + &store, + generation, + &sessions, + |persisted| committed.push(persisted), + ) + .unwrap()); + + assert_eq!(committed, vec![64, 65]); + assert_eq!(pacer.charged_bytes(), expected_bytes); + assert!(store + .list_catalog_sessions_for_source(CaptureProvider::Codex, source_root) + .unwrap() + .is_empty()); + assert!(store + .list_catalog_import_work( + CaptureProvider::Codex, + source_root, + ImportWorkClass::Fresh, + 1, + ) + .unwrap() + .is_empty()); + + store.begin_immediate_batch().unwrap(); + assert!(store + .complete_catalog_inventory_generation(CaptureProvider::Codex, source_root, generation,) + .unwrap()); + store.commit_batch().unwrap(); + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, source_root) + .unwrap() + .len(), + sessions.len() + ); + assert_eq!( + store + .list_catalog_import_work( + CaptureProvider::Codex, + source_root, + ImportWorkClass::Fresh, + sessions.len(), + ) + .unwrap() + .len(), + sessions.len() + ); +} + +#[test] +fn initial_catalog_path_rekey_falls_back_to_one_atomic_publication() { + let temp = tempfile::tempdir().unwrap(); + let first_root = "/history/old-root"; + let second_root = "/history/new-root"; + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let first_generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, first_root) + .unwrap(); + let first = catalog_persist_test_session(first_root, 0); + store.begin_immediate_batch().unwrap(); + store + .upsert_catalog_sessions(first_generation, std::slice::from_ref(&first)) + .unwrap(); + assert!( + store + .complete_catalog_inventory_generation( + CaptureProvider::Codex, + first_root, + first_generation, + ) + .unwrap() + ); + store.commit_batch().unwrap(); + + let second_generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, second_root) + .unwrap(); + let mut second = catalog_persist_test_session(second_root, 0); + second.source_path = first.source_path.clone(); + assert!(!persist_initial_catalog_sessions_bounded_for_test( + &store, + second_generation, + std::slice::from_ref(&second), + |_| {}, + ) + .unwrap()); + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, first_root) + .unwrap() + .len(), + 1 + ); + + store.begin_immediate_batch().unwrap(); + store + .upsert_catalog_sessions(second_generation, std::slice::from_ref(&second)) + .unwrap(); + store.rollback_batch().unwrap(); + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, first_root) + .unwrap() + .len(), + 1 + ); + + store.begin_immediate_batch().unwrap(); + store + .upsert_catalog_sessions(second_generation, std::slice::from_ref(&second)) + .unwrap(); + assert!(store + .complete_catalog_inventory_generation( + CaptureProvider::Codex, + second_root, + second_generation, + ) + .unwrap()); + store.commit_batch().unwrap(); + assert!(store + .list_catalog_sessions_for_source(CaptureProvider::Codex, first_root) + .unwrap() + .is_empty()); + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, second_root) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn initial_catalog_path_rekey_race_falls_back_without_stealing_published_rows() { + let temp = tempfile::tempdir().unwrap(); + let first_root = "/history/new-root"; + let second_root = "/history/racing-root"; + let db_path = temp.path().join("work.sqlite"); + let store = Store::open(&db_path).unwrap(); + let racing_store = Store::open(&db_path).unwrap(); + let first_generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, first_root) + .unwrap(); + let sessions = (0..65) + .map(|index| catalog_persist_test_session(first_root, index)) + .collect::>(); + let mut second_generation = None; + + assert!(!persist_initial_catalog_sessions_bounded_for_test( + &store, + first_generation, + &sessions, + |persisted| { + if persisted != 64 || second_generation.is_some() { + return; + } + let generation = racing_store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, second_root) + .unwrap(); + let mut racing = catalog_persist_test_session(second_root, 0); + racing.source_path = sessions[64].source_path.clone(); + racing_store.begin_immediate_batch().unwrap(); + racing_store + .upsert_catalog_sessions(generation, std::slice::from_ref(&racing)) + .unwrap(); + assert!(racing_store + .complete_catalog_inventory_generation( + CaptureProvider::Codex, + second_root, + generation, + ) + .unwrap()); + racing_store.commit_batch().unwrap(); + second_generation = Some(generation); + }, + ) + .unwrap()); + + assert!(second_generation.is_some()); + assert!(store + .list_catalog_sessions_for_source(CaptureProvider::Codex, first_root) + .unwrap() + .is_empty()); + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, second_root) + .unwrap() + .len(), + 1 + ); + + store.begin_immediate_batch().unwrap(); + store + .upsert_catalog_sessions(first_generation, &sessions) + .unwrap(); + assert!( + store + .complete_catalog_inventory_generation( + CaptureProvider::Codex, + first_root, + first_generation, + ) + .unwrap() + ); + store.commit_batch().unwrap(); + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, first_root) + .unwrap() + .len(), + sessions.len() + ); + assert!(store + .list_catalog_sessions_for_source(CaptureProvider::Codex, second_root) + .unwrap() + .is_empty()); +} + +#[test] +fn final_catalog_publication_reclaims_a_staged_path_stolen_after_the_last_batch() { + let temp = tempfile::tempdir().unwrap(); + let first_root = "/history/new-root"; + let second_root = "/history/racing-root"; + let db_path = temp.path().join("work.sqlite"); + let store = Store::open(&db_path).unwrap(); + let racing_store = Store::open(&db_path).unwrap(); + let first_generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, first_root) + .unwrap(); + let sessions = (0..65) + .map(|index| catalog_persist_test_session(first_root, index)) + .collect::>(); + + assert!(persist_initial_catalog_sessions_bounded_for_test( + &store, + first_generation, + &sessions, + |persisted| { + if persisted != sessions.len() { + return; + } + let generation = racing_store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, second_root) + .unwrap(); + let mut racing = catalog_persist_test_session(second_root, 0); + racing.source_path = sessions[0].source_path.clone(); + racing_store.begin_immediate_batch().unwrap(); + racing_store + .upsert_catalog_sessions(generation, std::slice::from_ref(&racing)) + .unwrap(); + assert!(racing_store + .complete_catalog_inventory_generation( + CaptureProvider::Codex, + second_root, + generation, + ) + .unwrap()); + racing_store.commit_batch().unwrap(); + }, + ) + .unwrap()); + + store.begin_immediate_batch().unwrap(); + persist_catalog_sessions_for_publication_for_test(&store, first_generation, &sessions, true) + .unwrap(); + assert!( + store + .complete_catalog_inventory_generation( + CaptureProvider::Codex, + first_root, + first_generation, + ) + .unwrap() + ); + store.commit_batch().unwrap(); + + assert_eq!( + store + .list_catalog_sessions_for_source(CaptureProvider::Codex, first_root) + .unwrap() + .len(), + sessions.len() + ); + assert!(racing_store + .list_catalog_sessions_for_source(CaptureProvider::Codex, second_root) + .unwrap() + .is_empty()); +} + +#[test] +fn superseded_initial_codex_catalog_batches_remain_hidden_and_recoverable() { + let temp = tempfile::tempdir().unwrap(); + let source_root = "/home/user/.codex/sessions"; + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let first_generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, source_root) + .unwrap(); + let sessions = (0..65) + .map(|index| catalog_persist_test_session(source_root, index)) + .collect::>(); + let mut next_generation = None; + + let error = persist_initial_catalog_sessions_bounded_for_test( + &store, + first_generation, + &sessions, + |_| { + next_generation.get_or_insert_with(|| { + store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, source_root) + .unwrap() + }); + }, + ) + .unwrap_err(); + + assert!(matches!(error, CaptureError::InventorySuperseded)); + assert!(store + .list_catalog_sessions_for_source(CaptureProvider::Codex, source_root) + .unwrap() + .is_empty()); + let (deleted, bytes) = store + .delete_unpublished_catalog_sessions_batch( + CaptureProvider::Codex, + source_root, + next_generation.unwrap(), + 128, + ) + .unwrap() + .unwrap(); + assert_eq!(deleted, 64); + assert!(bytes > 0); +} + +fn catalog_persist_test_session(source_root: &str, index: usize) -> CatalogSession { + CatalogSession { + provider: CaptureProvider::Codex, + source_format: CODEX_SESSION_SOURCE_FORMAT.to_owned(), + source_root: source_root.to_owned(), + source_path: format!("{source_root}/{index:04}.jsonl"), + external_session_id: Some(format!("session-{index:04}")), + parent_external_session_id: None, + agent_type: AgentType::Primary, + role_hint: Some("primary".to_owned()), + external_agent_id: None, + cwd: Some("/workspace".to_owned()), + session_started_at_ms: Some(index as i64), + file_size_bytes: 1_024 + index as u64, + file_modified_at_ms: index as i64, + import_revision: 1, + cataloged_at_ms: 1_000, + metadata: json!({"file_observation_token_v1": format!("token-{index:04}")}), + } +} diff --git a/crates/ctx-history-capture/src/tests/codex_observation.rs b/crates/ctx-history-capture/src/tests/codex_observation.rs new file mode 100644 index 00000000..0b6194a2 --- /dev/null +++ b/crates/ctx-history-capture/src/tests/codex_observation.rs @@ -0,0 +1,55 @@ +use super::support::*; + +#[cfg(unix)] +#[test] +fn codex_catalog_cache_reparses_same_size_rewrite_with_restored_mtime() { + let temp = tempdir(); + let root = temp.path().join("sessions"); + let path = root.join("2026/07/16/rewrite.jsonl"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let first_line = serde_json::json!({ + "timestamp": "2026-07-16T12:00:00Z", + "type": "session_meta", + "payload": {"id": "same-stat-codex", "cwd": "/repo-a"} + }); + let second_line = serde_json::json!({ + "timestamp": "2026-07-16T12:00:00Z", + "type": "session_meta", + "payload": {"id": "same-stat-codex", "cwd": "/repo-b"} + }); + let first_body = format!("{first_line}\n"); + let second_body = format!("{second_line}\n"); + assert_eq!(first_body.len(), second_body.len()); + fs::write(&path, first_body).unwrap(); + let original_modified = fs::metadata(&path).unwrap().modified().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + catalog_codex_session_tree(&root, &store, CodexSessionCatalogOptions::default()).unwrap(); + let first = store + .list_catalog_sessions_for_source(CaptureProvider::Codex, root.to_str().unwrap()) + .unwrap() + .pop() + .unwrap(); + + fs::write(&path, second_body).unwrap(); + fs::File::options() + .write(true) + .open(&path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(original_modified)) + .unwrap(); + let summary = + catalog_codex_session_tree(&root, &store, CodexSessionCatalogOptions::default()).unwrap(); + let second = store + .list_catalog_sessions_for_source(CaptureProvider::Codex, root.to_str().unwrap()) + .unwrap() + .pop() + .unwrap(); + + assert_eq!(summary.cached_sessions, 0); + assert_eq!(summary.parsed_sessions, 1); + assert_eq!(second.cwd.as_deref(), Some("/repo-b")); + assert_ne!( + first.metadata.get("file_observation_token_v1"), + second.metadata.get("file_observation_token_v1") + ); +} diff --git a/crates/ctx-history-capture/src/tests/support.rs b/crates/ctx-history-capture/src/tests/support.rs index d1a2b640..844ae2dc 100644 --- a/crates/ctx-history-capture/src/tests/support.rs +++ b/crates/ctx-history-capture/src/tests/support.rs @@ -253,10 +253,6 @@ pub(super) fn custom_history_fixture(name: &str) -> PathBuf { materialized_fixture("custom-history-jsonl", name) } -pub(super) fn write_oversized_jsonl_line(path: &Path) { - fs::write(path, vec![b'x'; MAX_PROVIDER_JSONL_LINE_BYTES + 1]).unwrap(); -} - pub(super) fn oversized_jsonl_line() -> Vec { let mut line = vec![b'x'; MAX_PROVIDER_JSONL_LINE_BYTES + 1]; line.push(b'\n'); diff --git a/crates/ctx-history-core/src/lib.rs b/crates/ctx-history-core/src/lib.rs index 84140cb1..99b30674 100644 --- a/crates/ctx-history-core/src/lib.rs +++ b/crates/ctx-history-core/src/lib.rs @@ -130,7 +130,11 @@ pub use provider::{ ProviderSupportStatus, PROVIDER_CAPTURE_ENVELOPE_MIN_SUPPORTED_SCHEMA_VERSION, PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, PROVIDER_SUPPORT_MATRIX_SCHEMA_VERSION, }; -pub use source::{CaptureProvider, CaptureSource, CaptureSourceDescriptor, CaptureSourceKind}; +pub use source::{ + canonical_provider_material_source_format, CaptureProvider, CaptureSource, + CaptureSourceDescriptor, CaptureSourceKind, ProviderMaterialSourceFormat, + PROVIDER_MATERIAL_SOURCE_FORMATS, +}; pub use sync::{ AuditActorKind, AuditLogEntry, EntityTimestamps, Fidelity, RedactionState, SyncAlias, SyncBatch, SyncBatchStatus, SyncCursor, SyncDirection, SyncMetadata, SyncOutboxItem, diff --git a/crates/ctx-history-core/src/source.rs b/crates/ctx-history-core/src/source.rs index 9394d09f..34503e81 100644 --- a/crates/ctx-history-core/src/source.rs +++ b/crates/ctx-history-core/src/source.rs @@ -69,6 +69,89 @@ text_enum! { default Unknown } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderMaterialSourceFormat { + pub provider: CaptureProvider, + pub inventory_source_format: &'static str, + pub material_source_format: &'static str, +} + +pub const PROVIDER_MATERIAL_SOURCE_FORMATS: &[ProviderMaterialSourceFormat] = &[ + ProviderMaterialSourceFormat { + provider: CaptureProvider::Codex, + inventory_source_format: "codex_session_jsonl_tree", + material_source_format: "codex_session_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Codex, + inventory_source_format: "codex_session_jsonl", + material_source_format: "codex_session_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Pi, + inventory_source_format: "pi_session_jsonl", + material_source_format: "pi_session_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Claude, + inventory_source_format: "claude_projects_jsonl_tree", + material_source_format: "claude_projects_jsonl_tree", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Tabnine, + inventory_source_format: "tabnine_cli_chat_recording_jsonl", + material_source_format: "tabnine_cli_chat_recording_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Cursor, + inventory_source_format: "cursor_agent_transcript_jsonl_tree", + material_source_format: "cursor_agent_transcript_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Qoder, + inventory_source_format: "qoder_transcript_jsonl_tree", + material_source_format: "qoder_transcript_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::QwenCode, + inventory_source_format: "qwen_code_chat_jsonl_tree", + material_source_format: "qwen_code_chat_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::KimiCodeCli, + inventory_source_format: "kimi_code_cli_wire_jsonl_tree", + material_source_format: "kimi_code_cli_wire_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::MistralVibe, + inventory_source_format: "mistral_vibe_session_jsonl_tree", + material_source_format: "mistral_vibe_session_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Mux, + inventory_source_format: "mux_session_jsonl_tree", + material_source_format: "mux_session_jsonl", + }, + ProviderMaterialSourceFormat { + provider: CaptureProvider::Windsurf, + inventory_source_format: "windsurf_cascade_hook_transcript_jsonl_tree", + material_source_format: "windsurf_cascade_hook_transcript_jsonl", + }, +]; + +pub fn canonical_provider_material_source_format( + provider: CaptureProvider, + inventory_source_format: &str, +) -> Option<&'static str> { + PROVIDER_MATERIAL_SOURCE_FORMATS + .iter() + .find(|mapping| { + mapping.provider == provider + && mapping.inventory_source_format == inventory_source_format + }) + .map(|mapping| mapping.material_source_format) +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct CaptureSourceDescriptor { pub kind: CaptureSourceKind, diff --git a/crates/ctx-history-store/src/bulk_search.rs b/crates/ctx-history-store/src/bulk_search.rs index 26f27e03..393d4d76 100644 --- a/crates/ctx-history-store/src/bulk_search.rs +++ b/crates/ctx-history-store/src/bulk_search.rs @@ -8,8 +8,12 @@ use ctx_history_core::utc_now; use std::{ + cell::Cell, ffi::OsString, + fs, + marker::PhantomData, path::PathBuf, + rc::Rc, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -34,6 +38,9 @@ const ALL_FTS_TABLES: [&str; 5] = [ const BULK_MODE_MARKER_KEY: &str = "event_search_bulk_mode_v1"; const BULK_MODE_AUTOMERGE_KEY_PREFIX: &str = "event_search_bulk_mode_v1:automerge:"; const BULK_MODE_CRISISMERGE_KEY_PREFIX: &str = "event_search_bulk_mode_v1:crisismerge:"; +#[cfg(test)] +const BULK_MODE_TEST_REMAINING_MERGE_PASSES_KEY: &str = + "event_search_bulk_mode_v1:test_remaining_merge_passes"; const FTS_AUTOMERGE_DEFAULT: i64 = 4; const FTS_CRISISMERGE_DEFAULT: i64 = 16; const FTS_BULK_CRISISMERGE: i64 = 1_000_000; @@ -42,8 +49,45 @@ const FTS_BULK_CRISISMERGE: i64 = 1_000_000; // Keep each step deliberately small so checkpoints remain safe on large real // indexes, not only on compact synthetic fixtures. const FTS_MERGE_PAGE_BUDGET: i64 = 16; +// Target one WAL write and one checkpoint copy per requested or observed FTS +// byte. FTS5 may amplify both operations beyond the nominal page estimate. +const FTS_MAINTENANCE_IO_PASSES: u64 = 2; +// Preserve the common changed-pass plus quiescence-pass completion path while +// putting a hard ceiling on large or resumed merge backlogs. +const EVENT_SEARCH_MERGE_PASSES_PER_CALL: usize = 2; const BULK_LOCK_SUFFIX: &str = ".event-search-bulk.lock.sqlite"; +thread_local! { + static EVENT_SEARCH_MAINTENANCE_PACER: Cell> = const { Cell::new(None) }; +} + +/// Restores the previous thread-local event-search maintenance pacer on drop. +#[doc(hidden)] +pub struct EventSearchMaintenancePacingGuard { + previous: Option, + _not_send: PhantomData>, +} + +impl Drop for EventSearchMaintenancePacingGuard { + fn drop(&mut self) { + EVENT_SEARCH_MAINTENANCE_PACER.with(|slot| slot.set(self.previous.take())); + } +} + +/// Installs I/O accounting for bounded FTS merge and checkpoint work. +/// +/// Merge work is precharged from its logical page budget, then supplemented to +/// cover an observed SQLite WAL write and checkpoint copy before each required +/// truncating checkpoint. +#[doc(hidden)] +pub fn install_event_search_maintenance_pacer(pacer: fn(u64)) -> EventSearchMaintenancePacingGuard { + let previous = EVENT_SEARCH_MAINTENANCE_PACER.with(|slot| slot.replace(Some(pacer))); + EventSearchMaintenancePacingGuard { + previous, + _not_send: PhantomData, + } +} + /// Owns the cross-process lock for one event-search bulk operation. /// /// SQLite releases the sidecar database's writer lock if the process exits, @@ -55,6 +99,23 @@ pub struct EventSearchBulkGuard { depth_counted: bool, } +/// Result of one bounded event-search maintenance slice. +/// +/// `Pending` is durable: the bulk-mode marker remains present and a later +/// maintenance call can resume without restarting the merge. It can also mean +/// that an in-progress provider publication currently fences FTS maintenance. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventSearchBulkMaintenanceOutcome { + Complete, + Pending, +} + +impl EventSearchBulkMaintenanceOutcome { + pub fn is_complete(self) -> bool { + self == Self::Complete + } +} + impl Drop for EventSearchBulkGuard { fn drop(&mut self) { if let Some(lock_conn) = &self.lock_conn { @@ -92,10 +153,14 @@ impl Store { } }; guard.depth_counted = true; + if bulk_mode_pending(self)? && self.has_pending_provider_file_publications()? { + return Ok(guard); + } self.begin_immediate_batch()?; let result = (|| { ensure_search_projection_stats_table(self)?; - if !bulk_mode_pending(self)? { + let pending = bulk_mode_pending(self)?; + if !pending { for table in EVENT_SEARCH_FTS_TABLES { if !table_exists(&self.conn, table)? { continue; @@ -113,7 +178,11 @@ impl Store { } save_bulk_mode_config(self, BULK_MODE_MARKER_KEY, 1)?; } - suppress_event_search_merges(self) + if pending && self.has_pending_provider_file_publications()? { + Ok(()) + } else { + suppress_event_search_merges(self) + } })(); if let Err(err) = result { let _ = self.rollback_batch(); @@ -126,31 +195,66 @@ impl Store { Ok(guard) } - /// Compact pending bulk segments in bounded steps, then restore saved settings. + /// Advance pending bulk compaction by one bounded slice. /// /// Bulk finalization deliberately uses positive FTS5 merge commands. Starting /// a full merge with a negative command would assign every pre-existing /// segment to the same level and rewrite the entire shared event index. That /// is appropriate for an explicit optimize, but not for finishing one /// provider import in an already-populated multi-source index. - pub fn finish_event_search_bulk_mode(&self, guard: &EventSearchBulkGuard) -> Result<()> { + pub fn finish_event_search_bulk_mode( + &self, + guard: &EventSearchBulkGuard, + ) -> Result { if guard.store_path != self.path { return Err(StoreError::InvalidBulkSearchGuard); } if guard.lock_conn.is_none() { - return Ok(()); + // The outer guard owns the durable marker and final merge. The + // nested caller has no independent maintenance debt to surface. + return Ok(EventSearchBulkMaintenanceOutcome::Complete); } if guard.depth_counted && guard.depth.load(Ordering::SeqCst) != 1 { return Err(StoreError::InvalidBulkSearchGuard); } if !bulk_mode_pending(self)? { - return Ok(()); + return Ok(EventSearchBulkMaintenanceOutcome::Complete); } - loop { + // A bounded provider publication can intentionally span scheduler + // passes. Keep merge suppression durable until that publication is + // resumed and finalized; FTS maintenance is fenced while its material + // is not yet visible. + if self.has_pending_provider_file_publications()? { + return Ok(EventSearchBulkMaintenanceOutcome::Pending); + } + for _ in 0..EVENT_SEARCH_MERGE_PASSES_PER_CALL { if self.finish_event_search_bulk_mode_step()? { - return Ok(()); + return Ok(EventSearchBulkMaintenanceOutcome::Complete); } } + self.event_search_bulk_maintenance_outcome() + } + + pub fn event_search_bulk_maintenance_outcome( + &self, + ) -> Result { + Ok(if bulk_mode_pending(self)? { + EventSearchBulkMaintenanceOutcome::Pending + } else { + EventSearchBulkMaintenanceOutcome::Complete + }) + } + + pub fn advance_event_search_bulk_maintenance( + &self, + ) -> Result { + let guard = self + .acquire_event_search_bulk_lock(self.busy_timeout)? + .ok_or(StoreError::BulkSearchImportBusy)?; + if !bulk_mode_pending(self)? { + return Ok(EventSearchBulkMaintenanceOutcome::Complete); + } + self.finish_event_search_bulk_mode(&guard) } pub(crate) fn recover_event_search_bulk_mode(&self) -> Result<()> { @@ -160,7 +264,7 @@ impl Store { self.begin_immediate_batch()?; let result = (|| { let pending = bulk_mode_pending(self)?; - if pending { + if pending && !self.has_pending_provider_file_publications()? { suppress_event_search_merges(self)?; } Ok(pending) @@ -179,8 +283,12 @@ impl Store { if !pending { return Ok(()); } - // A live importer owns this lock. A stale marker has no owner, so the - // next writable open adopts and completes its bounded recovery. + // A live importer owns this lock. Leave an unowned stale marker for an + // import or daemon path with an installed pacer; arbitrary writable + // opens must not opportunistically run expensive merge recovery. + if event_search_maintenance_pacer().is_none() { + return Ok(()); + } if let Some(guard) = self.acquire_event_search_bulk_lock(Duration::ZERO)? { self.finish_event_search_bulk_mode(&guard)?; } @@ -193,8 +301,14 @@ impl Store { let guard = self .acquire_event_search_bulk_lock(self.busy_timeout)? .ok_or(StoreError::BulkSearchImportBusy)?; + if self.has_pending_provider_file_publications()? { + return Ok(()); + } if bulk_mode_pending(self)? { self.finish_event_search_bulk_mode(&guard)?; + if bulk_mode_pending(self)? { + return Ok(()); + } } for table in ALL_FTS_TABLES { self.merge_fts_table_bounded(table, true)?; @@ -225,6 +339,7 @@ impl Store { } fn merge_fts_table_step(&self, table: &'static str, page_budget: i64) -> Result { + let nominal_bytes = pace_fts_maintenance(self, 1, page_budget)?; self.begin_immediate_batch()?; let result = merge_fts_table_in_transaction(self, table, page_budget); let changed = match result { @@ -238,6 +353,7 @@ impl Store { let _ = self.rollback_batch(); return Err(err); } + pace_observed_fts_wal(self, nominal_bytes)?; self.checkpoint_wal_truncate_required()?; Ok(changed) } @@ -246,6 +362,8 @@ impl Store { /// A quiescent pass is checkpointed before a second locked pass may restore /// settings, so a failed large-WAL checkpoint always leaves recovery marked. fn finish_event_search_bulk_mode_step(&self) -> Result { + let nominal_bytes = + pace_fts_maintenance(self, EVENT_SEARCH_FTS_TABLES.len(), FTS_MERGE_PAGE_BUDGET)?; self.begin_immediate_batch()?; let result = (|| { if !bulk_mode_pending(self)? { @@ -264,6 +382,7 @@ impl Store { let _ = self.rollback_batch(); return Err(err); } + pace_observed_fts_wal(self, nominal_bytes)?; self.checkpoint_wal_truncate_required()?; if !quiescent { return Ok(false); @@ -275,6 +394,8 @@ impl Store { /// If the final config-only checkpoint is pinned, the preceding potentially /// large merge WAL has already been truncated successfully. fn restore_event_search_bulk_mode_if_quiescent(&self) -> Result { + let nominal_bytes = + pace_fts_maintenance(self, EVENT_SEARCH_FTS_TABLES.len(), FTS_MERGE_PAGE_BUDGET)?; self.begin_immediate_batch()?; let result = (|| { if !bulk_mode_pending(self)? { @@ -298,6 +419,7 @@ impl Store { let _ = self.rollback_batch(); return Err(err); } + pace_observed_fts_wal(self, nominal_bytes)?; self.checkpoint_wal_truncate_required()?; Ok(finished) } @@ -339,6 +461,54 @@ fn merge_fts_table_in_transaction( Ok(store.conn.total_changes().saturating_sub(before) >= 2) } +fn pace_fts_maintenance(store: &Store, table_count: usize, page_budget: i64) -> Result { + let Some(pacer) = event_search_maintenance_pacer() else { + return Ok(0); + }; + let page_size = store + .conn + .query_row("PRAGMA page_size", [], |row| row.get::<_, u64>(0))?; + let logical_bytes = page_size + .saturating_mul(page_budget.unsigned_abs()) + .saturating_mul(table_count as u64) + .saturating_mul(FTS_MAINTENANCE_IO_PASSES); + pacer(logical_bytes); + Ok(logical_bytes) +} + +fn pace_observed_fts_wal(store: &Store, nominal_bytes: u64) -> Result<()> { + let Some(pacer) = event_search_maintenance_pacer() else { + return Ok(()); + }; + if let Some(wal_bytes) = observed_wal_bytes(store)? { + let supplement = observed_wal_supplement_bytes(nominal_bytes, wal_bytes); + if supplement > 0 { + pacer(supplement); + } + } + Ok(()) +} + +fn observed_wal_supplement_bytes(nominal_bytes: u64, wal_bytes: u64) -> u64 { + wal_bytes + .saturating_mul(FTS_MAINTENANCE_IO_PASSES) + .saturating_sub(nominal_bytes) +} + +fn event_search_maintenance_pacer() -> Option { + EVENT_SEARCH_MAINTENANCE_PACER.with(Cell::get) +} + +fn observed_wal_bytes(store: &Store) -> Result> { + let mut path = OsString::from(store.path.as_os_str()); + path.push("-wal"); + match fs::metadata(PathBuf::from(path)) { + Ok(metadata) => Ok(Some(metadata.len())), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(StoreError::Io(err)), + } +} + fn merge_event_search_tables_in_transaction(store: &Store) -> Result { let mut changed = false; for table in EVENT_SEARCH_FTS_TABLES { @@ -346,6 +516,19 @@ fn merge_event_search_tables_in_transaction(store: &Store) -> Result { changed |= merge_fts_table_in_transaction(store, table, FTS_MERGE_PAGE_BUDGET)?; } } + #[cfg(test)] + if let Some(remaining) = bulk_mode_config(store, BULK_MODE_TEST_REMAINING_MERGE_PASSES_KEY)? { + if remaining > 0 { + // Keep fault-injected work durable so reopen tests exercise the + // same marker/config state machine as interrupted FTS work. + save_bulk_mode_config( + store, + BULK_MODE_TEST_REMAINING_MERGE_PASSES_KEY, + remaining - 1, + )?; + changed = true; + } + } Ok(changed) } @@ -461,3 +644,315 @@ fn clear_bulk_mode_state(store: &Store) -> Result<()> { )?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + const LARGE_MERGE_EVENT_COUNT: usize = 512; + const LARGE_MERGE_PAYLOAD_WORDS: usize = 256; + const FORCED_MERGE_PASSES: i64 = 5; + const MAX_CONVERGENCE_CALLS: usize = 256; + + thread_local! { + static MAINTENANCE_CHARGES: RefCell> = const { RefCell::new(Vec::new()) }; + } + + fn record_maintenance_bytes(bytes: u64) { + MAINTENANCE_CHARGES.with(|charges| charges.borrow_mut().push(bytes)); + } + + fn charged_maintenance_bytes() -> u64 { + MAINTENANCE_CHARGES.with(|charges| { + charges + .borrow() + .iter() + .copied() + .fold(0_u64, u64::saturating_add) + }) + } + + fn maintenance_charges() -> Vec { + MAINTENANCE_CHARGES.with(|charges| charges.borrow().clone()) + } + + fn clear_maintenance_charges() { + MAINTENANCE_CHARGES.with(|charges| charges.borrow_mut().clear()); + } + + fn tempdir() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("ctx-history-store-bulk-search-") + .tempdir() + .unwrap() + } + + fn insert_search_events( + store: &Store, + token: &str, + batch: &str, + count: usize, + payload_words: usize, + ) { + let payload = "payload ".repeat(payload_words); + for index in 0..count { + store + .conn + .execute( + r#" + INSERT INTO event_search + (event_id, history_record_id, session_id, role, preview_text, rank_bucket) + VALUES (?1, NULL, NULL, 'user', ?2, 'message') + "#, + params![ + format!("{token}-{batch}-event-{index}"), + format!("{token} {index} {payload}") + ], + ) + .unwrap(); + } + } + + fn seed_large_merge_work(store: &Store, token: &str) { + insert_search_events( + store, + token, + "pending", + LARGE_MERGE_EVENT_COUNT, + LARGE_MERGE_PAYLOAD_WORDS, + ); + save_bulk_mode_config( + store, + BULK_MODE_TEST_REMAINING_MERGE_PASSES_KEY, + FORCED_MERGE_PASSES, + ) + .unwrap(); + } + + fn remaining_forced_merge_passes(store: &Store) -> Option { + bulk_mode_config(store, BULK_MODE_TEST_REMAINING_MERGE_PASSES_KEY).unwrap() + } + + fn marker(store: &Store) -> Option { + bulk_mode_config(store, BULK_MODE_MARKER_KEY).unwrap() + } + + fn search_count(store: &Store, token: &str) -> i64 { + store + .conn + .query_row( + "SELECT COUNT(*) FROM event_search WHERE event_search MATCH ?1", + params![token], + |row| row.get(0), + ) + .unwrap() + } + + fn assert_merge_suppressed(store: &Store) { + for table in EVENT_SEARCH_FTS_TABLES { + assert_eq!(fts_config_value(store, table, "automerge", 4).unwrap(), 0); + assert_eq!( + fts_config_value(store, table, "crisismerge", 16).unwrap(), + FTS_BULK_CRISISMERGE + ); + } + } + + fn assert_merge_config_restored(store: &Store) { + for table in EVENT_SEARCH_FTS_TABLES { + assert_eq!( + fts_config_value(store, table, "automerge", 4).unwrap(), + FTS_AUTOMERGE_DEFAULT + ); + assert_eq!( + fts_config_value(store, table, "crisismerge", 16).unwrap(), + FTS_CRISISMERGE_DEFAULT + ); + } + } + + #[test] + fn observed_wal_supplement_targets_two_passes_minus_nominal_precharge() { + let nominal_bytes = 256; + let wal_bytes = 1_024; + let supplement = observed_wal_supplement_bytes(nominal_bytes, wal_bytes); + + assert_eq!(supplement, 1_792); + assert_eq!(nominal_bytes + supplement, wal_bytes * 2); + assert_eq!(observed_wal_supplement_bytes(2_048, wal_bytes), 0); + assert_eq!(observed_wal_supplement_bytes(4_096, wal_bytes), 0); + } + + #[test] + fn finish_call_is_bounded_and_repeated_calls_converge() { + let temp = tempdir(); + let db_path = temp.path().join("work.sqlite"); + let store = Store::open(&db_path).unwrap(); + let guard = store.begin_event_search_bulk_mode().unwrap(); + seed_large_merge_work(&store, "boundedfinish"); + let page_size = store + .conn + .query_row("PRAGMA page_size", [], |row| row.get::<_, u64>(0)) + .unwrap(); + clear_maintenance_charges(); + let _pacing = install_event_search_maintenance_pacer(record_maintenance_bytes); + + assert_eq!( + store.finish_event_search_bulk_mode(&guard).unwrap(), + EventSearchBulkMaintenanceOutcome::Pending + ); + let nominal_step_bytes = page_size + * FTS_MERGE_PAGE_BUDGET as u64 + * EVENT_SEARCH_FTS_TABLES.len() as u64 + * FTS_MAINTENANCE_IO_PASSES; + let charges = maintenance_charges(); + assert_eq!( + charges.len(), + EVENT_SEARCH_MERGE_PASSES_PER_CALL * 2, + "each merge step must precharge and then supplement for observed WAL" + ); + for step_charges in charges.chunks_exact(2) { + assert_eq!(step_charges[0], nominal_step_bytes); + assert!(step_charges[1] > 0, "checkpoint WAL was not supplemented"); + } + assert!( + charges[1] > nominal_step_bytes, + "fixture did not produce WAL amplification beyond the nominal precharge" + ); + assert_eq!( + observed_wal_bytes(&store).unwrap().unwrap_or_default(), + 0, + "observed WAL must be charged before the truncating checkpoint" + ); + assert_eq!( + store.event_search_bulk_maintenance_outcome().unwrap(), + EventSearchBulkMaintenanceOutcome::Pending + ); + + assert_eq!( + marker(&store), + Some(1), + "one call must leave a large merge restartable" + ); + assert_eq!( + remaining_forced_merge_passes(&store), + Some(FORCED_MERGE_PASSES - EVENT_SEARCH_MERGE_PASSES_PER_CALL as i64), + "one call exceeded its merge-pass bound" + ); + assert_merge_suppressed(&store); + assert_eq!( + search_count(&store, "boundedfinish"), + LARGE_MERGE_EVENT_COUNT as i64 + ); + + let mut calls = 1; + while marker(&store).is_some() && calls < MAX_CONVERGENCE_CALLS { + store.finish_event_search_bulk_mode(&guard).unwrap(); + calls += 1; + } + + assert!( + calls < MAX_CONVERGENCE_CALLS, + "bounded finalization did not converge" + ); + assert!(calls > 1, "fixture did not exceed one bounded call"); + assert_eq!(marker(&store), None); + assert_eq!( + store.event_search_bulk_maintenance_outcome().unwrap(), + EventSearchBulkMaintenanceOutcome::Complete + ); + assert_merge_config_restored(&store); + assert_eq!( + search_count(&store, "boundedfinish"), + LARGE_MERGE_EVENT_COUNT as i64 + ); + } + + #[test] + fn unpaced_reopen_preserves_recovery_for_a_paced_path() { + let temp = tempdir(); + let db_path = temp.path().join("work.sqlite"); + { + let store = Store::open(&db_path).unwrap(); + let _guard = store.begin_event_search_bulk_mode().unwrap(); + seed_large_merge_work(&store, "boundedreopen"); + assert_eq!(marker(&store), Some(1)); + } + + let unpaced_reopen = Store::open(&db_path).unwrap(); + assert_eq!( + marker(&unpaced_reopen), + Some(1), + "an unpaced writable open must preserve the recovery marker" + ); + assert_eq!( + remaining_forced_merge_passes(&unpaced_reopen), + Some(FORCED_MERGE_PASSES), + "an unpaced writable open advanced expensive merge recovery" + ); + assert_merge_suppressed(&unpaced_reopen); + assert_eq!( + search_count(&unpaced_reopen, "boundedreopen"), + LARGE_MERGE_EVENT_COUNT as i64 + ); + drop(unpaced_reopen); + + clear_maintenance_charges(); + let _pacing = install_event_search_maintenance_pacer(record_maintenance_bytes); + let first_paced_reopen = Store::open(&db_path).unwrap(); + assert_eq!(marker(&first_paced_reopen), Some(1)); + assert_eq!( + remaining_forced_merge_passes(&first_paced_reopen), + Some(FORCED_MERGE_PASSES - EVENT_SEARCH_MERGE_PASSES_PER_CALL as i64), + "a paced recovery open must advance one bounded slice" + ); + assert!(charged_maintenance_bytes() > 0); + drop(first_paced_reopen); + + let mut reopens = 1; + loop { + let reopened = Store::open(&db_path).unwrap(); + reopens += 1; + assert_eq!( + search_count(&reopened, "boundedreopen"), + LARGE_MERGE_EVENT_COUNT as i64 + ); + if marker(&reopened).is_none() { + assert_merge_config_restored(&reopened); + break; + } + assert_merge_suppressed(&reopened); + assert!( + reopens < MAX_CONVERGENCE_CALLS, + "bounded reopen recovery did not converge" + ); + } + + assert!( + reopens > 1, + "fixture did not exceed one paced recovery open" + ); + } + + #[test] + fn maintenance_api_reports_and_advances_durable_debt() { + let temp = tempdir(); + let db_path = temp.path().join("work.sqlite"); + let store = Store::open(&db_path).unwrap(); + let guard = store.begin_event_search_bulk_mode().unwrap(); + seed_large_merge_work(&store, "maintenanceapi"); + assert_eq!( + store.finish_event_search_bulk_mode(&guard).unwrap(), + EventSearchBulkMaintenanceOutcome::Pending + ); + drop(guard); + + let before = remaining_forced_merge_passes(&store).unwrap(); + let outcome = store.advance_event_search_bulk_maintenance().unwrap(); + let after = remaining_forced_merge_passes(&store).unwrap_or_default(); + + assert!(after < before); + assert_eq!(outcome, EventSearchBulkMaintenanceOutcome::Pending); + } +} diff --git a/crates/ctx-history-store/src/catalog.rs b/crates/ctx-history-store/src/catalog.rs index 6be15494..e413a7f5 100644 --- a/crates/ctx-history-store/src/catalog.rs +++ b/crates/ctx-history-store/src/catalog.rs @@ -1,12 +1,16 @@ -use ctx_history_core::{AgentType, CaptureProvider}; +use ctx_history_core::{ + canonical_provider_material_source_format, AgentType, CaptureProvider, + PROVIDER_MATERIAL_SOURCE_FORMATS, +}; use rusqlite::{params, OptionalExtension}; use serde_json::Value; +use std::{fmt::Write as _, str::FromStr}; use crate::connection::{ capped_i64, collect_rows, nonnegative_i64_to_u32, nonnegative_i64_to_u64, parse_json, - parse_text_enum, + parse_text_enum, with_immediate_transaction, }; -use crate::{Result, Store, StoreError}; +use crate::{ProviderFileCheckpointKey, ProviderFileInventoryFamily, Result, Store, StoreError}; #[derive(Debug, Clone, PartialEq)] pub struct CatalogSession { @@ -75,6 +79,138 @@ pub struct SourceImportFileIndexUpdate<'a> { pub indexed_at_ms: i64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImportPendingReason { + FreshNew, + FreshChanged, + FreshAppend, + RecoveryRetry, + RecoveryReplacement, + ParserRevision, + MissingMaterial, + AbandonedPublication, + Legacy, + ExplicitRescan, +} + +impl ImportPendingReason { + pub fn as_str(self) -> &'static str { + match self { + Self::FreshNew => "fresh_new", + Self::FreshChanged => "fresh_changed", + Self::FreshAppend => "fresh_append", + Self::RecoveryRetry => "recovery_retry", + Self::RecoveryReplacement => "recovery_replacement", + Self::ParserRevision => "parser_revision", + Self::MissingMaterial => "missing_material", + Self::AbandonedPublication => "abandoned_publication", + Self::Legacy => "legacy", + Self::ExplicitRescan => "explicit_rescan", + } + } + + pub fn class(self) -> ImportWorkClass { + match self { + Self::FreshNew | Self::FreshChanged | Self::FreshAppend => ImportWorkClass::Fresh, + Self::RecoveryRetry + | Self::RecoveryReplacement + | Self::ParserRevision + | Self::MissingMaterial + | Self::AbandonedPublication + | Self::Legacy + | Self::ExplicitRescan => ImportWorkClass::Recovery, + } + } + + pub fn requires_replacement(self) -> bool { + !matches!(self, Self::FreshAppend | Self::RecoveryRetry) + } + + fn retry_after_failure(prior: Option) -> Self { + match prior { + Some(Self::FreshAppend | Self::RecoveryRetry) => Self::RecoveryRetry, + _ => Self::RecoveryReplacement, + } + } +} + +impl FromStr for ImportPendingReason { + type Err = StoreError; + + fn from_str(value: &str) -> Result { + match value { + "fresh_new" => Ok(Self::FreshNew), + "fresh_changed" => Ok(Self::FreshChanged), + "fresh_append" => Ok(Self::FreshAppend), + "recovery_retry" => Ok(Self::RecoveryRetry), + "recovery_replacement" => Ok(Self::RecoveryReplacement), + "parser_revision" => Ok(Self::ParserRevision), + "missing_material" => Ok(Self::MissingMaterial), + "abandoned_publication" => Ok(Self::AbandonedPublication), + "legacy" => Ok(Self::Legacy), + "explicit_rescan" => Ok(Self::ExplicitRescan), + other => Err(StoreError::InvalidImportPendingReason(other.to_owned())), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImportWorkClass { + Fresh, + Recovery, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CatalogImportWork { + pub session: CatalogSession, + pub reason: ImportPendingReason, + pub estimated_bytes: u64, + pub last_attempt_at_ms: Option, + pub has_active_publication: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SourceImportFileWork { + pub file: SourceImportFile, + pub reason: ImportPendingReason, + pub estimated_bytes: u64, + pub last_attempt_at_ms: Option, + pub has_active_publication: bool, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ImportPendingReasonRepairProgress { + pub processed_rows: usize, + pub classified_rows: usize, + pub completed_families: usize, + pub complete: bool, +} + +#[derive(Debug, Clone, Copy)] +enum ImportPendingReasonRepairFamily { + CatalogSessions, + SourceImportFiles, +} + +impl ImportPendingReasonRepairFamily { + const ALL: [Self; 2] = [Self::CatalogSessions, Self::SourceImportFiles]; + + fn as_str(self) -> &'static str { + match self { + Self::CatalogSessions => "catalog_sessions", + Self::SourceImportFiles => "source_import_files", + } + } +} + +#[derive(Debug)] +struct ImportPendingReasonRepairRow { + provider: String, + source_root: String, + source_path: String, + requires_work: bool, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct CatalogCounts { pub total: usize, @@ -134,17 +270,108 @@ impl CatalogIndexedStatus { } } +#[derive(Debug)] +struct CatalogPendingState { + provider: CaptureProvider, + source_format: String, + source_root: String, + file_size_bytes: u64, + file_modified_at_ms: i64, + import_revision: u32, + is_stale: bool, + indexed_file_size_bytes: Option, + indexed_file_modified_at_ms: Option, + indexed_status: CatalogIndexedStatus, + indexed_import_revision: Option, + pending_reason: Option, + metadata_json: String, +} + +#[derive(Debug)] +struct SourceImportPendingState { + source_format: String, + file_size_bytes: u64, + file_modified_at_ms: i64, + import_revision: u32, + is_stale: bool, + indexed_file_size_bytes: Option, + indexed_file_modified_at_ms: Option, + indexed_status: CatalogIndexedStatus, + indexed_import_revision: Option, + pending_reason: Option, + metadata_json: String, +} + +impl FromStr for CatalogIndexedStatus { + type Err = StoreError; + + fn from_str(value: &str) -> Result { + match value { + "pending" => Ok(Self::Pending), + "indexed" => Ok(Self::Indexed), + "completed_with_rejections" => Ok(Self::CompletedWithRejections), + "rejected" => Ok(Self::Rejected), + "failed" => Ok(Self::Failed), + other => Err(StoreError::InvalidImportPendingReason(format!( + "invalid catalog indexed status {other}" + ))), + } + } +} + include!("catalog/inventory.rs"); +include!("catalog/inventory_pacing.rs"); include!("catalog/pending_work.rs"); include!("catalog/source_imports.rs"); include!("catalog/counts.rs"); +fn source_import_metadata_matches_owner_growth(prior_json: &str, current: &Value) -> bool { + let Ok(mut prior) = serde_json::from_str::(prior_json) else { + return false; + }; + let mut current = current.clone(); + for metadata in [&mut prior, &mut current] { + if let Some(object) = metadata.as_object_mut() { + object.remove("change_token_v1"); + } + } + prior == current +} + +fn catalog_observation_metadata_matches(prior_json: &str, current: &Value) -> bool { + let Ok(prior) = serde_json::from_str::(prior_json) else { + return false; + }; + catalog_observation_token(&prior) == catalog_observation_token(current) +} + +fn catalog_observation_token(metadata: &Value) -> Option<&str> { + metadata + .get("file_observation_token_v1") + .and_then(Value::as_str) +} + fn catalog_session_select_sql(tail: &str) -> String { format!( "SELECT source_path, provider, source_format, source_root, external_session_id, parent_external_session_id, agent_type, role_hint, external_agent_id, cwd, session_started_at_ms, file_size_bytes, file_modified_at_ms, import_revision, cataloged_at_ms, metadata_json FROM catalog_sessions {tail}" ) } +pub(crate) fn catalog_inventory_material_published_predicate(alias: &str) -> String { + format!( + r#" + NOT EXISTS ( + SELECT 1 + FROM import_inventory_generations AS catalog_inventory + WHERE catalog_inventory.provider = {alias}.provider + AND catalog_inventory.source_root = {alias}.source_root + AND catalog_inventory.inventory_family = 'catalog_sessions' + AND catalog_inventory.completed_generation = 0 + ) + "# + ) +} + fn source_import_file_select_sql(tail: &str) -> String { format!( "SELECT provider, source_format, source_root, source_path, file_size_bytes, file_modified_at_ms, import_revision, observed_at_ms, metadata_json FROM source_import_files {tail}" @@ -165,6 +392,18 @@ fn source_import_file_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result, +) -> rusqlite::Result { + Ok(SourceImportFileWork { + file: source_import_file_from_row(row)?, + reason: parse_text_enum(row.get(9)?)?, + estimated_bytes: nonnegative_i64_to_u64(row.get(10)?)?, + last_attempt_at_ms: row.get(11)?, + has_active_publication: row.get(12)?, + }) +} + fn catalog_session_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(CatalogSession { source_path: row.get(0)?, @@ -186,11 +425,23 @@ fn catalog_session_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result) -> rusqlite::Result { + Ok(CatalogImportWork { + session: catalog_session_from_row(row)?, + reason: parse_text_enum(row.get(16)?)?, + estimated_bytes: nonnegative_i64_to_u64(row.get(17)?)?, + last_attempt_at_ms: row.get(18)?, + has_active_publication: row.get(19)?, + }) +} + fn catalog_pending_import_condition_sql(alias: &str) -> String { + let material_exists = catalog_material_exists_sql(alias); format!( r#" ( - {alias}.indexed_status IN ('pending', 'failed') + {alias}.pending_reason IS NOT NULL + OR {alias}.indexed_status IN ('pending', 'failed') OR {alias}.indexed_file_size_bytes IS NULL OR {alias}.indexed_file_modified_at_ms IS NULL OR {alias}.indexed_file_size_bytes != {alias}.file_size_bytes @@ -199,21 +450,7 @@ fn catalog_pending_import_condition_sql(alias: &str) -> String { OR {alias}.indexed_import_revision != {alias}.import_revision OR ( {alias}.indexed_status IN ('indexed', 'completed_with_rejections') - AND NOT EXISTS ( - SELECT 1 - FROM sessions AS session - LEFT JOIN capture_sources AS source - ON source.id = session.capture_source_id - WHERE session.provider = {alias}.provider - AND {alias}.external_session_id IS NOT NULL - AND session.external_session_id = {alias}.external_session_id - AND ( - session.capture_source_id IS NULL - OR source.source_root = {alias}.source_root - OR source.raw_source_path = {alias}.source_path - ) - LIMIT 1 - ) + AND NOT ({material_exists}) ) ) "# @@ -221,16 +458,121 @@ fn catalog_pending_import_condition_sql(alias: &str) -> String { } fn source_import_file_pending_condition_sql(alias: &str) -> String { + let material_exists = source_import_material_exists_sql(alias); format!( r#" ( - {alias}.indexed_status IN ('pending', 'failed') + {alias}.pending_reason IS NOT NULL + OR {alias}.indexed_status IN ('pending', 'failed') OR {alias}.indexed_file_size_bytes IS NULL OR {alias}.indexed_file_modified_at_ms IS NULL OR {alias}.indexed_file_size_bytes != {alias}.file_size_bytes OR {alias}.indexed_file_modified_at_ms != {alias}.file_modified_at_ms OR {alias}.indexed_import_revision IS NULL OR {alias}.indexed_import_revision != {alias}.import_revision + OR ( + {alias}.indexed_status IN ('indexed', 'completed_with_rejections') + AND NOT ({material_exists}) + ) + ) + "# + ) +} + +fn import_work_class_predicate(alias: &str, class: ImportWorkClass) -> String { + let reasons = match class { + ImportWorkClass::Fresh => "'fresh_new', 'fresh_changed', 'fresh_append'", + ImportWorkClass::Recovery => { + "'recovery_retry', 'recovery_replacement', 'parser_revision', 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan'" + } + }; + format!("{alias}.pending_reason IN ({reasons})") +} + +fn import_work_order(alias: &str, _class: ImportWorkClass) -> String { + format!("{alias}.indexed_at_ms, {alias}.source_path") +} + +fn expected_material_source_format( + provider: CaptureProvider, + inventory_source_format: &str, +) -> &str { + canonical_provider_material_source_format(provider, inventory_source_format) + .unwrap_or(inventory_source_format) +} + +fn sql_string_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +fn material_source_format_sql_case(alias: &str) -> String { + let mut expression = String::from("CASE"); + for mapping in PROVIDER_MATERIAL_SOURCE_FORMATS { + write!( + expression, + " WHEN {alias}.provider = {} AND {alias}.source_format = {} THEN {}", + sql_string_literal(mapping.provider.as_str()), + sql_string_literal(mapping.inventory_source_format), + sql_string_literal(mapping.material_source_format), + ) + .expect("writing to a String cannot fail"); + } + write!(expression, " ELSE {alias}.source_format END").expect("writing to a String cannot fail"); + expression +} + +fn catalog_material_exists_sql(alias: &str) -> String { + let material_source_format = material_source_format_sql_case(alias); + let owner = crate::provider_files::material_owner_predicate( + "source", + &format!("{alias}.provider"), + &material_source_format, + &format!("{alias}.source_root"), + &format!("{alias}.source_path"), + ); + format!( + r#" + EXISTS ( + SELECT 1 + FROM sessions AS material_session + JOIN capture_sources AS source + ON source.id = material_session.capture_source_id + WHERE material_session.provider = {alias}.provider + AND {alias}.external_session_id IS NOT NULL + AND material_session.external_session_id = {alias}.external_session_id + AND ({owner}) + AND source.external_session_id = {alias}.external_session_id + LIMIT 1 + ) + "# + ) +} + +fn source_import_material_exists_sql(alias: &str) -> String { + let material_source_format = material_source_format_sql_case(alias); + format!( + r#" + EXISTS ( + SELECT 1 + FROM capture_sources AS source + WHERE source.provider = {alias}.provider + AND source.source_format = {material_source_format} + AND ( + ( + json_extract({alias}.metadata_json, '$.inventory_unit') = 'source_root' + AND source.source_root = {alias}.source_root + ) + OR ( + json_extract({alias}.metadata_json, '$.inventory_unit') IS NOT 'source_root' + AND source.raw_source_path = {alias}.source_path + AND ( + source.source_root = {alias}.source_root + OR source.source_root = source.raw_source_path + OR source.source_root IS NULL + ) + ) + ) + LIMIT 1 ) "# ) @@ -238,6 +580,7 @@ fn source_import_file_pending_condition_sql(alias: &str) -> String { fn catalog_indexed_count_sql() -> String { let visible = crate::provider_files::catalog_material_visible_predicate("catalog"); + let material_exists = catalog_material_exists_sql("catalog"); format!( r#" SELECT COUNT(*) @@ -248,21 +591,7 @@ fn catalog_indexed_count_sql() -> String { AND catalog.indexed_file_size_bytes = catalog.file_size_bytes AND catalog.indexed_file_modified_at_ms = catalog.file_modified_at_ms AND catalog.indexed_import_revision = catalog.import_revision - AND EXISTS ( - SELECT 1 - FROM sessions AS session - LEFT JOIN capture_sources AS source - ON source.id = session.capture_source_id - WHERE session.provider = catalog.provider - AND catalog.external_session_id IS NOT NULL - AND session.external_session_id = catalog.external_session_id - AND ( - session.capture_source_id IS NULL - OR source.source_root = catalog.source_root - OR source.raw_source_path = catalog.source_path - ) - LIMIT 1 - ) + AND {material_exists} "# ) } diff --git a/crates/ctx-history-store/src/catalog/counts.rs b/crates/ctx-history-store/src/catalog/counts.rs index c121c88e..a957bf39 100644 --- a/crates/ctx-history-store/src/catalog/counts.rs +++ b/crates/ctx-history-store/src/catalog/counts.rs @@ -83,7 +83,9 @@ impl Store { AND indexed_file_modified_at_ms = file_modified_at_ms AND indexed_import_revision = import_revision AND {visible} - "# + AND {} + "#, + source_import_material_exists_sql("source_import_files") ), [], |row| row.get::<_, i64>(0), diff --git a/crates/ctx-history-store/src/catalog/inventory.rs b/crates/ctx-history-store/src/catalog/inventory.rs index 29bdad8b..4c935cd2 100644 --- a/crates/ctx-history-store/src/catalog/inventory.rs +++ b/crates/ctx-history-store/src/catalog/inventory.rs @@ -15,6 +15,44 @@ impl Store { self.allocate_import_inventory_generation(provider, source_root, "source_import_files") } + pub fn current_source_import_inventory_generation( + &self, + provider: CaptureProvider, + source_root: &str, + ) -> Result> { + self.current_import_inventory_generation(provider, source_root, "source_import_files") + } + + pub fn current_catalog_inventory_generation( + &self, + provider: CaptureProvider, + source_root: &str, + ) -> Result> { + self.current_import_inventory_generation(provider, source_root, "catalog_sessions") + } + + fn current_import_inventory_generation( + &self, + provider: CaptureProvider, + source_root: &str, + inventory_family: &str, + ) -> Result> { + self.conn + .query_row( + r#" + SELECT current_generation + FROM import_inventory_generations + WHERE provider = ?1 + AND source_root = ?2 + AND inventory_family = ?3 + "#, + params![provider.as_str(), source_root, inventory_family], + |row| nonnegative_i64_to_u64(row.get(0)?), + ) + .optional() + .map_err(StoreError::from) + } + pub fn catalog_inventory_generation_is_current( &self, provider: CaptureProvider, @@ -53,6 +91,117 @@ impl Store { .map_err(StoreError::from) } + #[doc(hidden)] + pub fn catalog_inventory_generation_is_unpublished( + &self, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, + ) -> Result { + self.conn + .query_row( + r#" + SELECT current_generation = ?4 AND completed_generation = 0 + FROM import_inventory_generations + WHERE provider = ?1 + AND source_root = ?2 + AND inventory_family = ?3 + "#, + params![ + provider.as_str(), + source_root, + "catalog_sessions", + capped_i64(inventory_generation), + ], + |row| row.get(0), + ) + .optional() + .map(|unpublished| unpublished.unwrap_or(false)) + .map_err(StoreError::from) + } + + #[doc(hidden)] + pub fn delete_unpublished_catalog_sessions_batch( + &self, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, + limit: usize, + ) -> Result> { + self.delete_unpublished_catalog_sessions_batch_paced( + provider, + source_root, + inventory_generation, + limit, + |_| {}, + ) + } + + #[doc(hidden)] + pub fn delete_unpublished_catalog_sessions_batch_paced( + &self, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, + limit: usize, + pace: impl Fn(u64), + ) -> Result> { + with_immediate_transaction(&self.conn, || { + if !self.catalog_inventory_generation_is_unpublished( + provider, + source_root, + inventory_generation, + )? { + return Ok(None); + } + let (rows, bytes) = self.conn.query_row( + r#" + SELECT COUNT(*), COALESCE(SUM( + length(source_path) + length(provider) + length(source_format) + + length(source_root) + COALESCE(length(external_session_id), 0) + + COALESCE(length(parent_external_session_id), 0) + + length(agent_type) + COALESCE(length(role_hint), 0) + + COALESCE(length(external_agent_id), 0) + COALESCE(length(cwd), 0) + + length(metadata_json) + 256 + ), 0) + FROM ( + SELECT * + FROM catalog_sessions + WHERE provider = ?1 + AND source_root = ?2 + ORDER BY source_path + LIMIT ?3 + ) + "#, + params![provider.as_str(), source_root, capped_i64(limit as u64),], + |row| { + Ok(( + row.get::<_, usize>(0)?, + nonnegative_i64_to_u64(row.get(1)?)?, + )) + }, + )?; + pace(bytes); + let deleted = self.conn.execute( + r#" + DELETE FROM catalog_sessions + WHERE rowid IN ( + SELECT rowid + FROM catalog_sessions + WHERE provider = ?1 + AND source_root = ?2 + ORDER BY source_path + LIMIT ?3 + ) + "#, + params![provider.as_str(), source_root, capped_i64(limit as u64),], + )?; + debug_assert_eq!(deleted, rows); + Ok(Some((deleted, bytes))) + }) + } + + #[doc(hidden)] pub fn catalog_inventory_generation_is_complete( &self, provider: CaptureProvider, @@ -203,6 +352,218 @@ impl Store { Ok(changed) } + fn classify_catalog_pending_reason( + &self, + session: &CatalogSession, + ) -> Result> { + let prior = self + .conn + .query_row( + r#" + SELECT provider, source_format, source_root, file_size_bytes, + file_modified_at_ms, import_revision, is_stale, + indexed_file_size_bytes, indexed_file_modified_at_ms, + indexed_status, indexed_import_revision, pending_reason, metadata_json + FROM catalog_sessions + WHERE source_path = ?1 + "#, + params![&session.source_path], + |row| { + Ok(CatalogPendingState { + provider: parse_text_enum(row.get(0)?)?, + source_format: row.get(1)?, + source_root: row.get(2)?, + file_size_bytes: nonnegative_i64_to_u64(row.get(3)?)?, + file_modified_at_ms: row.get(4)?, + import_revision: nonnegative_i64_to_u32(row.get(5)?)?, + is_stale: row.get(6)?, + indexed_file_size_bytes: row + .get::<_, Option>(7)? + .map(nonnegative_i64_to_u64) + .transpose()?, + indexed_file_modified_at_ms: row.get(8)?, + indexed_status: parse_text_enum(row.get(9)?)?, + indexed_import_revision: row + .get::<_, Option>(10)? + .map(nonnegative_i64_to_u32) + .transpose()?, + pending_reason: row + .get::<_, Option>(11)? + .map(parse_text_enum) + .transpose()?, + metadata_json: row.get(12)?, + }) + }, + ) + .optional()?; + let Some(prior) = prior else { + return Ok(Some(ImportPendingReason::FreshNew)); + }; + if self.provider_file_publication_was_abandoned( + session.provider, + "catalog_sessions", + &prior.source_format, + &prior.source_root, + &session.source_path, + )? { + return Ok(Some(ImportPendingReason::AbandonedPublication)); + } + let same_identity = prior.provider == session.provider + && prior.source_format == session.source_format + && prior.source_root == session.source_root; + let same_fingerprint = same_identity + && prior.file_size_bytes == session.file_size_bytes + && prior.file_modified_at_ms == session.file_modified_at_ms + && prior.import_revision == session.import_revision + && catalog_observation_metadata_matches(&prior.metadata_json, &session.metadata) + && !prior.is_stale; + if same_fingerprint && prior.pending_reason == Some(ImportPendingReason::ExplicitRescan) { + return Ok(prior.pending_reason); + } + if !same_fingerprint { + if let Some(reason) = prior + .pending_reason + .filter(|reason| reason.requires_replacement()) + { + return Ok(Some(reason)); + } + let parser_revision_only = same_identity + && prior.file_size_bytes == session.file_size_bytes + && prior.file_modified_at_ms == session.file_modified_at_ms + && catalog_observation_metadata_matches(&prior.metadata_json, &session.metadata) + && prior.import_revision != session.import_revision + && !prior.is_stale; + if parser_revision_only { + return Ok(Some(ImportPendingReason::ParserRevision)); + } + let grew_in_place = same_identity + && prior.import_revision == session.import_revision + && !prior.is_stale + && session.file_size_bytes > prior.file_size_bytes; + if grew_in_place + && matches!( + prior.pending_reason, + Some(ImportPendingReason::FreshAppend | ImportPendingReason::RecoveryRetry) + ) + && self.catalog_incremental_material_is_supported(&prior, session)? + { + return Ok(prior.pending_reason); + } + if self.catalog_observation_is_append(&prior, session)? { + return Ok(Some(ImportPendingReason::FreshAppend)); + } + return Ok(Some(ImportPendingReason::FreshChanged)); + } + match prior.indexed_status { + CatalogIndexedStatus::Failed => Ok(Some(ImportPendingReason::retry_after_failure( + prior.pending_reason, + ))), + CatalogIndexedStatus::Pending => Ok(Some( + prior.pending_reason.unwrap_or(ImportPendingReason::Legacy), + )), + CatalogIndexedStatus::Indexed | CatalogIndexedStatus::CompletedWithRejections => { + let indexed_matches = prior.indexed_file_size_bytes + == Some(session.file_size_bytes) + && prior.indexed_file_modified_at_ms == Some(session.file_modified_at_ms); + if prior.indexed_import_revision != Some(session.import_revision) { + Ok(Some(ImportPendingReason::ParserRevision)) + } else if !indexed_matches { + Ok(Some( + prior.pending_reason.unwrap_or(ImportPendingReason::Legacy), + )) + } else if !self.catalog_session_material_exists(session)? { + Ok(Some(ImportPendingReason::MissingMaterial)) + } else { + Ok(None) + } + } + CatalogIndexedStatus::Rejected => Ok(None), + } + } + + fn catalog_observation_is_append( + &self, + prior: &CatalogPendingState, + session: &CatalogSession, + ) -> Result { + if prior.provider != session.provider + || prior.source_format != session.source_format + || prior.source_root != session.source_root + || prior.import_revision != session.import_revision + || prior.is_stale + || session.file_size_bytes <= prior.file_size_bytes + || !matches!( + prior.indexed_status, + CatalogIndexedStatus::Indexed | CatalogIndexedStatus::CompletedWithRejections + ) + || prior.indexed_file_size_bytes != Some(prior.file_size_bytes) + || prior.indexed_file_modified_at_ms != Some(prior.file_modified_at_ms) + || prior.indexed_import_revision != Some(prior.import_revision) + { + return Ok(false); + } + Ok(self.provider_file_checkpoint_matches_prior_observation( + session.provider, + &session.source_format, + &session.source_root, + &session.source_path, + session.import_revision, + prior.file_size_bytes, + )? && self.catalog_session_material_exists(session)?) + } + + fn catalog_incremental_material_is_supported( + &self, + prior: &CatalogPendingState, + session: &CatalogSession, + ) -> Result { + Ok(self.provider_file_checkpoint_matches_prior_observation( + session.provider, + &session.source_format, + &session.source_root, + &session.source_path, + session.import_revision, + prior.file_size_bytes, + )? && self.catalog_session_material_exists(session)?) + } + + fn catalog_session_material_exists(&self, session: &CatalogSession) -> Result { + let Some(external_session_id) = session.external_session_id.as_deref() else { + return Ok(false); + }; + let material_source_format = + expected_material_source_format(session.provider, &session.source_format); + let owner = + crate::provider_files::material_owner_predicate("source", "?1", "?3", "?5", "?4"); + self.conn + .query_row( + &format!( + r#" + SELECT EXISTS ( + SELECT 1 + FROM sessions AS material_session + JOIN capture_sources AS source + ON source.id = material_session.capture_source_id + WHERE material_session.provider = ?1 + AND material_session.external_session_id = ?2 + AND ({owner}) + AND source.external_session_id = ?2 + LIMIT 1 + ) + "# + ), + params![ + session.provider.as_str(), + external_session_id, + material_source_format, + &session.source_path, + &session.source_root, + ], + |row| row.get(0), + ) + .map_err(Into::into) + } + pub fn upsert_catalog_sessions( &self, inventory_generation: u64, @@ -215,16 +576,17 @@ impl Store { source_path, provider, source_format, source_root, external_session_id, parent_external_session_id, agent_type, role_hint, external_agent_id, cwd, session_started_at_ms, file_size_bytes, - file_modified_at_ms, import_revision, cataloged_at_ms, is_stale, metadata_json + file_modified_at_ms, import_revision, cataloged_at_ms, is_stale, + pending_reason, metadata_json ) - SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, 0, ?16 + SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, 0, ?16, ?17 WHERE EXISTS ( SELECT 1 FROM import_inventory_generations AS inventory WHERE inventory.provider = ?2 AND inventory.source_root = ?4 AND inventory.inventory_family = 'catalog_sessions' - AND inventory.current_generation = ?17 + AND inventory.current_generation = ?18 ) ON CONFLICT(source_path) DO UPDATE SET provider = excluded.provider, @@ -249,6 +611,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_at_ms ELSE NULL END, @@ -259,6 +623,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_file_size_bytes ELSE NULL END, @@ -269,6 +635,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_file_modified_at_ms ELSE NULL END, @@ -279,6 +647,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_status WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -299,6 +669,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_error WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -319,6 +691,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_event_count ELSE NULL END, @@ -329,6 +703,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.indexed_import_revision ELSE NULL END, @@ -339,6 +715,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.last_imported_at_ms WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -361,6 +739,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.last_imported_file_size_bytes WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -383,6 +763,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.last_imported_file_modified_at_ms WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -405,6 +787,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.last_imported_file_sha256 WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -427,6 +811,8 @@ impl Store { AND catalog_sessions.import_revision = excluded.import_revision AND catalog_sessions.file_size_bytes = excluded.file_size_bytes AND catalog_sessions.file_modified_at_ms = excluded.file_modified_at_ms + AND json_extract(catalog_sessions.metadata_json, '$.file_observation_token_v1') + IS json_extract(excluded.metadata_json, '$.file_observation_token_v1') THEN catalog_sessions.last_imported_event_count WHEN excluded.file_size_bytes > catalog_sessions.file_size_bytes AND catalog_sessions.provider IS excluded.provider @@ -442,6 +828,7 @@ impl Store { THEN catalog_sessions.last_imported_event_count ELSE NULL END, + pending_reason = excluded.pending_reason, metadata_json = excluded.metadata_json WHERE EXISTS ( SELECT 1 @@ -449,7 +836,7 @@ impl Store { WHERE inventory.provider = excluded.provider AND inventory.source_root = excluded.source_root AND inventory.inventory_family = 'catalog_sessions' - AND inventory.current_generation = ?17 + AND inventory.current_generation = ?18 ) AND ( catalog_sessions.provider IS NOT excluded.provider @@ -466,12 +853,14 @@ impl Store { OR catalog_sessions.file_modified_at_ms != excluded.file_modified_at_ms OR catalog_sessions.import_revision != excluded.import_revision OR catalog_sessions.is_stale != 0 + OR catalog_sessions.pending_reason IS NOT excluded.pending_reason OR catalog_sessions.metadata_json IS NOT excluded.metadata_json ) "#, )?; let mut changed = 0; for session in sessions { + let pending_reason = self.classify_catalog_pending_reason(session)?; changed += stmt.execute(params![ session.source_path.as_str(), session.provider.as_str(), @@ -488,6 +877,7 @@ impl Store { session.file_modified_at_ms, i64::from(session.import_revision), session.cataloged_at_ms, + pending_reason.map(ImportPendingReason::as_str), serde_json::to_string(&session.metadata)?, capped_i64(inventory_generation), ])?; @@ -538,59 +928,4 @@ impl Store { ) .map_err(Into::into) } - - pub fn mark_catalog_source_missing_paths_stale( - &self, - provider: CaptureProvider, - source_root: &str, - current_paths: &[String], - cataloged_at_ms: i64, - inventory_generation: u64, - ) -> Result { - self.conn.execute( - "CREATE TEMP TABLE IF NOT EXISTS temp_catalog_current_paths(source_path TEXT PRIMARY KEY)", - [], - )?; - self.conn - .execute("DELETE FROM temp_catalog_current_paths", [])?; - { - let mut stmt = self.conn.prepare( - "INSERT OR IGNORE INTO temp_catalog_current_paths(source_path) VALUES (?1)", - )?; - for path in current_paths { - stmt.execute(params![path.as_str()])?; - } - } - let changed = self.conn.execute( - r#" - UPDATE catalog_sessions - SET is_stale = 1, cataloged_at_ms = ?3 - WHERE provider = ?1 - AND source_root = ?2 - AND is_stale = 0 - AND EXISTS ( - SELECT 1 - FROM import_inventory_generations AS inventory - WHERE inventory.provider = ?1 - AND inventory.source_root = ?2 - AND inventory.inventory_family = 'catalog_sessions' - AND inventory.current_generation = ?4 - ) - AND NOT EXISTS ( - SELECT 1 - FROM temp_catalog_current_paths current - WHERE current.source_path = catalog_sessions.source_path - ) - "#, - params![ - provider.as_str(), - source_root, - cataloged_at_ms, - capped_i64(inventory_generation) - ], - )?; - self.conn - .execute("DELETE FROM temp_catalog_current_paths", [])?; - Ok(changed) - } } diff --git a/crates/ctx-history-store/src/catalog/inventory_pacing.rs b/crates/ctx-history-store/src/catalog/inventory_pacing.rs new file mode 100644 index 00000000..5ed44b7a --- /dev/null +++ b/crates/ctx-history-store/src/catalog/inventory_pacing.rs @@ -0,0 +1,227 @@ +impl Store { + pub fn catalog_sessions_have_external_path_owners( + &self, + provider: CaptureProvider, + source_root: &str, + sessions: &[CatalogSession], + ) -> Result { + self.catalog_sessions_have_external_path_owners_paced( + provider, + source_root, + sessions, + |_| {}, + ) + } + + #[doc(hidden)] + pub fn catalog_sessions_have_external_path_owners_paced( + &self, + provider: CaptureProvider, + source_root: &str, + sessions: &[CatalogSession], + pace: impl Fn(u64), + ) -> Result { + let (_, external_owner) = + self.catalog_session_path_ownership(provider, source_root, sessions, pace)?; + Ok(external_owner) + } + + #[doc(hidden)] + pub fn catalog_sessions_all_owned_by_source_paced( + &self, + provider: CaptureProvider, + source_root: &str, + sessions: &[CatalogSession], + pace: impl Fn(u64), + ) -> Result { + let (all_owned, _) = + self.catalog_session_path_ownership(provider, source_root, sessions, pace)?; + Ok(all_owned) + } + + fn catalog_session_path_ownership( + &self, + provider: CaptureProvider, + source_root: &str, + sessions: &[CatalogSession], + pace: impl Fn(u64), + ) -> Result<(bool, bool)> { + const QUERY_PATHS: usize = 256; + + let mut all_owned = true; + let mut external_owner = false; + for sessions in sessions.chunks(QUERY_PATHS) { + pace(sessions.iter().fold(0u64, |bytes, session| { + bytes + .saturating_add(session.source_path.len() as u64) + .saturating_add(64) + })); + let placeholders = vec!["?"; sessions.len()].join(", "); + let sql = format!( + "SELECT\n\ + COALESCE(SUM(CASE WHEN provider = ?1 AND source_root = ?2 THEN 1 ELSE 0 END), 0),\n\ + COALESCE(SUM(CASE WHEN provider != ?1 OR source_root != ?2 THEN 1 ELSE 0 END), 0)\n\ + FROM catalog_sessions\n\ + WHERE source_path IN ({placeholders})" + ); + let parameters = std::iter::once(provider.as_str()) + .chain(std::iter::once(source_root)) + .chain(sessions.iter().map(|session| session.source_path.as_str())); + let (owned, external): (usize, usize) = + self.conn + .query_row(&sql, rusqlite::params_from_iter(parameters), |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + all_owned &= owned == sessions.len(); + external_owner |= external > 0; + } + Ok((all_owned, external_owner)) + } + + pub fn mark_catalog_source_missing_paths_stale( + &self, + provider: CaptureProvider, + source_root: &str, + current_paths: &[String], + cataloged_at_ms: i64, + inventory_generation: u64, + ) -> Result { + self.mark_catalog_source_missing_paths_stale_paced( + provider, + source_root, + current_paths, + cataloged_at_ms, + inventory_generation, + |_| {}, + ) + } + + #[doc(hidden)] + pub fn mark_catalog_source_missing_paths_stale_paced( + &self, + provider: CaptureProvider, + source_root: &str, + current_paths: &[String], + cataloged_at_ms: i64, + inventory_generation: u64, + pace: impl Fn(u64), + ) -> Result { + if self.conn.is_autocommit() { + return with_immediate_transaction(&self.conn, || { + self.mark_catalog_source_missing_paths_stale_paced( + provider, + source_root, + current_paths, + cataloged_at_ms, + inventory_generation, + pace, + ) + }); + } + self.conn.execute( + "CREATE TEMP TABLE IF NOT EXISTS temp_catalog_current_paths(source_path TEXT PRIMARY KEY)", + [], + )?; + self.conn + .execute("DELETE FROM temp_catalog_current_paths", [])?; + { + let mut stmt = self.conn.prepare( + "INSERT OR IGNORE INTO temp_catalog_current_paths(source_path) VALUES (?1)", + )?; + for paths in current_paths.chunks(64) { + pace(paths.iter().fold(0u64, |bytes, path| { + bytes.saturating_add(path.len() as u64).saturating_add(64) + })); + for path in paths { + stmt.execute(params![path.as_str()])?; + } + } + } + let mut changed = 0usize; + loop { + let (batch_rows, batch_bytes) = self.conn.query_row( + r#" + SELECT COUNT(*), COALESCE(SUM(length(source_path) + 128), 0) + FROM ( + SELECT source_path + FROM catalog_sessions + WHERE provider = ?1 + AND source_root = ?2 + AND is_stale = 0 + AND EXISTS ( + SELECT 1 + FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 + AND inventory.source_root = ?2 + AND inventory.inventory_family = 'catalog_sessions' + AND inventory.current_generation = ?3 + ) + AND NOT EXISTS ( + SELECT 1 + FROM temp_catalog_current_paths current + WHERE current.source_path = catalog_sessions.source_path + ) + ORDER BY source_path + LIMIT 64 + ) + "#, + params![ + provider.as_str(), + source_root, + capped_i64(inventory_generation) + ], + |row| { + Ok(( + row.get::<_, usize>(0)?, + nonnegative_i64_to_u64(row.get(1)?)?, + )) + }, + )?; + if batch_rows == 0 { + break; + } + pace(batch_bytes); + let batch_changed = self.conn.execute( + r#" + UPDATE catalog_sessions + SET is_stale = 1, cataloged_at_ms = ?3 + WHERE rowid IN ( + SELECT rowid + FROM catalog_sessions + WHERE provider = ?1 + AND source_root = ?2 + AND is_stale = 0 + AND EXISTS ( + SELECT 1 + FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 + AND inventory.source_root = ?2 + AND inventory.inventory_family = 'catalog_sessions' + AND inventory.current_generation = ?4 + ) + AND NOT EXISTS ( + SELECT 1 + FROM temp_catalog_current_paths current + WHERE current.source_path = catalog_sessions.source_path + ) + ORDER BY source_path + LIMIT 64 + ) + "#, + params![ + provider.as_str(), + source_root, + cataloged_at_ms, + capped_i64(inventory_generation) + ], + )?; + changed = changed.saturating_add(batch_changed); + if batch_changed == 0 { + break; + } + } + self.conn + .execute("DELETE FROM temp_catalog_current_paths", [])?; + Ok(changed) + } +} diff --git a/crates/ctx-history-store/src/catalog/pending_work.rs b/crates/ctx-history-store/src/catalog/pending_work.rs index 55d18bdd..d9fc7205 100644 --- a/crates/ctx-history-store/src/catalog/pending_work.rs +++ b/crates/ctx-history-store/src/catalog/pending_work.rs @@ -25,6 +25,452 @@ impl Store { collect_rows(rows) } + pub fn repair_import_pending_reasons( + &self, + limit: usize, + ) -> Result { + with_immediate_transaction(&self.conn, || { + self.conn.execute_batch( + r#" + INSERT OR IGNORE INTO import_pending_reason_repairs ( + inventory_family, completed + ) + SELECT 'catalog_sessions', NOT EXISTS (SELECT 1 FROM catalog_sessions) + UNION ALL + SELECT 'source_import_files', NOT EXISTS (SELECT 1 FROM source_import_files); + "#, + )?; + + let mut progress = ImportPendingReasonRepairProgress::default(); + let mut remaining = limit; + for family in ImportPendingReasonRepairFamily::ALL { + let (cursor_provider, cursor_source_root, cursor_source_path, completed) = + self.conn.query_row( + r#" + SELECT cursor_provider, cursor_source_root, cursor_source_path, completed + FROM import_pending_reason_repairs + WHERE inventory_family = ?1 + "#, + [family.as_str()], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, bool>(3)?, + )) + }, + )?; + if completed || remaining == 0 { + continue; + } + + let batch_limit = remaining; + let rows = self.list_import_pending_reason_repair_rows( + family, + cursor_provider.as_deref(), + cursor_source_root.as_deref(), + cursor_source_path.as_deref(), + batch_limit, + )?; + if rows.is_empty() { + self.conn.execute( + "UPDATE import_pending_reason_repairs SET completed = 1 \ + WHERE inventory_family = ?1", + [family.as_str()], + )?; + continue; + } + + let processed = rows.len(); + for row in &rows { + if row.requires_work { + progress.classified_rows += + self.classify_legacy_pending_reason_row(family, row)?; + } + } + progress.processed_rows += processed; + remaining -= processed; + + let last = rows.last().expect("non-empty repair batch"); + let family_completed = processed < batch_limit + || !self.import_pending_reason_repair_rows_remain(family, last)?; + self.conn.execute( + r#" + UPDATE import_pending_reason_repairs + SET cursor_provider = ?2, cursor_source_root = ?3, + cursor_source_path = ?4, completed = ?5 + WHERE inventory_family = ?1 + "#, + params![ + family.as_str(), + &last.provider, + &last.source_root, + &last.source_path, + family_completed, + ], + )?; + } + + progress.completed_families = self.conn.query_row( + "SELECT COUNT(*) FROM import_pending_reason_repairs WHERE completed = 1", + [], + |row| row.get(0), + )?; + progress.complete = + progress.completed_families == ImportPendingReasonRepairFamily::ALL.len(); + Ok(progress) + }) + } + + fn list_import_pending_reason_repair_rows( + &self, + family: ImportPendingReasonRepairFamily, + cursor_provider: Option<&str>, + cursor_source_root: Option<&str>, + cursor_source_path: Option<&str>, + limit: usize, + ) -> Result> { + let (sql, query_params): (String, Vec) = match family { + ImportPendingReasonRepairFamily::CatalogSessions => { + let material_exists = catalog_material_exists_sql("catalog"); + ( + format!( + r#" + SELECT provider, source_root, source_path, + pending_reason IS NULL AND is_stale = 0 AND ( + indexed_status IN ('pending', 'failed') + OR ( + indexed_status IN ('indexed', 'completed_with_rejections') + AND ( + indexed_file_size_bytes IS NULL + OR indexed_file_modified_at_ms IS NULL + OR indexed_file_size_bytes != file_size_bytes + OR indexed_file_modified_at_ms != file_modified_at_ms + OR indexed_import_revision IS NULL + OR indexed_import_revision != import_revision + OR NOT ({material_exists}) + ) + ) + ) + FROM catalog_sessions AS catalog + WHERE ?1 IS NULL OR source_path > ?1 + ORDER BY source_path + LIMIT ?2 + "# + ), + vec![ + cursor_source_path.map(str::to_owned).into(), + capped_i64(limit as u64).into(), + ], + ) + } + ImportPendingReasonRepairFamily::SourceImportFiles => { + let material_exists = source_import_material_exists_sql("source_file"); + ( + format!( + r#" + SELECT provider, source_root, source_path, + pending_reason IS NULL AND is_stale = 0 AND ( + indexed_status IN ('pending', 'failed') + OR ( + indexed_status IN ('indexed', 'completed_with_rejections') + AND ( + indexed_file_size_bytes IS NULL + OR indexed_file_modified_at_ms IS NULL + OR indexed_file_size_bytes != file_size_bytes + OR indexed_file_modified_at_ms != file_modified_at_ms + OR indexed_import_revision IS NULL + OR indexed_import_revision != import_revision + OR NOT ({material_exists}) + ) + ) + ) + FROM source_import_files AS source_file + WHERE ( + ?1 IS NULL + OR provider > ?1 + OR (provider = ?1 AND source_root > ?2) + OR ( + provider = ?1 AND source_root = ?2 + AND source_path > ?3 + ) + ) + ORDER BY provider, source_root, source_path + LIMIT ?4 + "# + ), + vec![ + cursor_provider.map(str::to_owned).into(), + cursor_source_root.map(str::to_owned).into(), + cursor_source_path.map(str::to_owned).into(), + capped_i64(limit as u64).into(), + ], + ) + } + }; + let mut stmt = self.conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(query_params), |row| { + Ok(ImportPendingReasonRepairRow { + provider: row.get(0)?, + source_root: row.get(1)?, + source_path: row.get(2)?, + requires_work: row.get(3)?, + }) + })?; + collect_rows(rows) + } + + fn import_pending_reason_repair_rows_remain( + &self, + family: ImportPendingReasonRepairFamily, + cursor: &ImportPendingReasonRepairRow, + ) -> Result { + match family { + ImportPendingReasonRepairFamily::CatalogSessions => self + .conn + .query_row( + "SELECT EXISTS (SELECT 1 FROM catalog_sessions WHERE source_path > ?1)", + [&cursor.source_path], + |row| row.get(0), + ) + .map_err(Into::into), + ImportPendingReasonRepairFamily::SourceImportFiles => self + .conn + .query_row( + r#" + SELECT EXISTS ( + SELECT 1 FROM source_import_files + WHERE provider > ?1 + OR (provider = ?1 AND source_root > ?2) + OR ( + provider = ?1 AND source_root = ?2 + AND source_path > ?3 + ) + ) + "#, + params![&cursor.provider, &cursor.source_root, &cursor.source_path], + |row| row.get(0), + ) + .map_err(Into::into), + } + } + + fn classify_legacy_pending_reason_row( + &self, + family: ImportPendingReasonRepairFamily, + row: &ImportPendingReasonRepairRow, + ) -> Result { + let changed = match family { + ImportPendingReasonRepairFamily::CatalogSessions => self.conn.execute( + r#" + UPDATE catalog_sessions SET pending_reason = 'legacy' + WHERE source_path = ?1 AND pending_reason IS NULL + "#, + [&row.source_path], + )?, + ImportPendingReasonRepairFamily::SourceImportFiles => self.conn.execute( + r#" + UPDATE source_import_files SET pending_reason = 'legacy' + WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 + AND pending_reason IS NULL + "#, + params![&row.provider, &row.source_root, &row.source_path], + )?, + }; + Ok(changed) + } + + pub fn list_catalog_import_work( + &self, + provider: CaptureProvider, + source_root: &str, + class: ImportWorkClass, + limit: usize, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let predicate = import_work_class_predicate("catalog", class); + let inventory_published = catalog_inventory_material_published_predicate("catalog"); + let active_publication = + crate::provider_files::catalog_candidate_is_global_publication("catalog"); + let order = import_work_order("catalog", class); + let active_path = self + .effective_provider_file_publication_inventory_owner()? + .filter(|owner| { + owner.inventory_family == ProviderFileInventoryFamily::Catalog + && owner.provider == provider + && owner.source_root == source_root + }) + .map(|owner| owner.source_path); + let mut work = Vec::with_capacity(limit); + if let Some(active_path) = active_path.as_deref() { + let mut active_stmt = self.conn.prepare(&format!( + r#" + SELECT source_path, provider, source_format, source_root, + external_session_id, parent_external_session_id, agent_type, role_hint, + external_agent_id, cwd, session_started_at_ms, file_size_bytes, + file_modified_at_ms, import_revision, cataloged_at_ms, metadata_json, + pending_reason, + CASE + WHEN pending_reason = 'fresh_append' THEN MAX( + file_size_bytes - COALESCE(( + SELECT checkpoint.committed_byte_offset + FROM provider_file_checkpoints AS checkpoint + WHERE checkpoint.provider = catalog.provider + AND checkpoint.source_format = catalog.source_format + AND checkpoint.source_root = catalog.source_root + AND checkpoint.source_path = catalog.source_path + ), 0), + 0 + ) + ELSE file_size_bytes + END, + indexed_at_ms, 1 AS has_active_publication + FROM catalog_sessions AS catalog + WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 + AND is_stale = 0 AND {predicate} AND ({inventory_published}) + AND ({active_publication}) + LIMIT 1 + "# + ))?; + work.extend( + active_stmt + .query_map( + params![provider.as_str(), source_root, active_path], + catalog_import_work_from_row, + )? + .collect::>>()?, + ); + } + if work.len() >= limit { + return Ok(work); + } + let ordinary_limit = limit.saturating_add(usize::from(!work.is_empty())); + let mut stmt = self.conn.prepare(&format!( + r#" + SELECT source_path, provider, source_format, source_root, + external_session_id, parent_external_session_id, agent_type, role_hint, + external_agent_id, cwd, session_started_at_ms, file_size_bytes, + file_modified_at_ms, import_revision, cataloged_at_ms, metadata_json, + pending_reason, + CASE + WHEN pending_reason = 'fresh_append' THEN MAX( + file_size_bytes - COALESCE(( + SELECT checkpoint.committed_byte_offset + FROM provider_file_checkpoints AS checkpoint + WHERE checkpoint.provider = catalog.provider + AND checkpoint.source_format = catalog.source_format + AND checkpoint.source_root = catalog.source_root + AND checkpoint.source_path = catalog.source_path + ), 0), + 0 + ) + ELSE file_size_bytes + END, + indexed_at_ms, 0 AS has_active_publication + FROM catalog_sessions AS catalog + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 + AND {predicate} AND ({inventory_published}) + ORDER BY {order} + LIMIT ?3 + "# + ))?; + let rows = stmt.query_map( + params![ + provider.as_str(), + source_root, + capped_i64(ordinary_limit as u64) + ], + catalog_import_work_from_row, + )?; + let ordinary = collect_rows(rows)?; + let remaining = limit.saturating_sub(work.len()); + work.extend( + ordinary + .into_iter() + .filter(|candidate| { + active_path + .as_deref() + .is_none_or(|path| candidate.session.source_path != path) + }) + .take(remaining), + ); + Ok(work) + } + + pub fn catalog_import_work_count( + &self, + provider: CaptureProvider, + source_root: &str, + class: ImportWorkClass, + ) -> Result { + let predicate = import_work_class_predicate("catalog", class); + let inventory_published = catalog_inventory_material_published_predicate("catalog"); + self.conn + .query_row( + &format!( + "SELECT COUNT(*) FROM catalog_sessions AS catalog \ + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 \ + AND {predicate} AND ({inventory_published})" + ), + params![provider.as_str(), source_root], + |row| row.get(0), + ) + .map_err(Into::into) + } + + pub fn schedule_catalog_source_explicit_rescan( + &self, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, + ) -> Result { + with_immediate_transaction(&self.conn, || { + let legacy = self.conn.execute( + r#" + UPDATE catalog_sessions + SET pending_reason = 'legacy' + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 + AND indexed_status IN ('pending', 'failed') AND pending_reason IS NULL + AND EXISTS ( + SELECT 1 FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 AND inventory.source_root = ?2 + AND inventory.inventory_family = 'catalog_sessions' + AND inventory.current_generation = ?3 + ) + "#, + params![ + provider.as_str(), + source_root, + capped_i64(inventory_generation) + ], + )?; + let explicit = self.conn.execute( + r#" + UPDATE catalog_sessions + SET pending_reason = 'explicit_rescan' + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 + AND indexed_status = 'indexed' + AND pending_reason IS NULL + AND EXISTS ( + SELECT 1 FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 AND inventory.source_root = ?2 + AND inventory.inventory_family = 'catalog_sessions' + AND inventory.current_generation = ?3 + ) + "#, + params![ + provider.as_str(), + source_root, + capped_i64(inventory_generation) + ], + )?; + Ok(legacy.saturating_add(explicit)) + }) + } + pub fn list_active_catalog_sessions_for_source( &self, provider: CaptureProvider, @@ -49,6 +495,8 @@ impl Store { collect_rows(rows) } + /// Compatibility helper for catalog rows created before observation tokens. + #[doc(hidden)] pub fn mark_catalog_source_indexed( &self, provider: CaptureProvider, @@ -62,10 +510,35 @@ impl Store { ) } - pub fn record_catalog_source_import_result( + pub(crate) fn record_catalog_source_import_result( + &self, + provider: CaptureProvider, + update: CatalogSourceIndexUpdate<'_>, + status: CatalogIndexedStatus, + error: Option<&str>, + ) -> Result { + self.with_provider_file_inventory_result_write( + provider, + update.source_root, + update.source_path, + || { + self.record_catalog_source_import_result_inner( + provider, + update, + None, + status, + error, + status.preserves_native_resume_checkpoint(), + ) + }, + ) + } + + pub fn record_observed_catalog_source_import_result( &self, provider: CaptureProvider, update: CatalogSourceIndexUpdate<'_>, + metadata: &serde_json::Value, status: CatalogIndexedStatus, error: Option<&str>, ) -> Result { @@ -77,6 +550,7 @@ impl Store { self.record_catalog_source_import_result_inner( provider, update, + Some(metadata), status, error, status.preserves_native_resume_checkpoint(), @@ -85,10 +559,11 @@ impl Store { ) } - pub(crate) fn record_catalog_source_import_result_preserving_legacy_cursor( + pub(crate) fn record_observed_catalog_source_import_result_preserving_legacy_cursor( &self, provider: CaptureProvider, update: CatalogSourceIndexUpdate<'_>, + metadata: &serde_json::Value, status: CatalogIndexedStatus, error: Option<&str>, ) -> Result { @@ -98,7 +573,12 @@ impl Store { update.source_path, || { self.record_catalog_source_import_result_inner( - provider, update, status, error, false, + provider, + update, + Some(metadata), + status, + error, + false, ) }, ) @@ -108,10 +588,21 @@ impl Store { &self, provider: CaptureProvider, update: CatalogSourceIndexUpdate<'_>, + metadata: Option<&serde_json::Value>, status: CatalogIndexedStatus, error: Option<&str>, advance_legacy_cursor: bool, ) -> Result { + if metadata.is_some_and(|metadata| { + metadata + .get("file_observation_token_v1") + .and_then(serde_json::Value::as_str) + .is_none_or(str::is_empty) + }) { + return Err(StoreError::InvalidProviderFileCheckpoint( + "catalog observation token is required", + )); + } let changed = self.conn.execute( r#" UPDATE catalog_sessions @@ -122,6 +613,15 @@ impl Store { indexed_error = ?10, indexed_event_count = ?7, indexed_import_revision = ?12, + pending_reason = CASE + WHEN ?8 = 'failed' THEN CASE + WHEN pending_reason IN ('fresh_append', 'recovery_retry') + THEN 'recovery_retry' + ELSE 'recovery_replacement' + END + WHEN ?8 = 'pending' THEN COALESCE(pending_reason, 'legacy') + ELSE NULL + END, last_imported_at_ms = CASE WHEN ?11 THEN ?4 ELSE last_imported_at_ms END, last_imported_file_size_bytes = CASE WHEN ?11 THEN ?5 ELSE last_imported_file_size_bytes END, last_imported_file_modified_at_ms = CASE WHEN ?11 THEN ?6 ELSE last_imported_file_modified_at_ms END, @@ -134,6 +634,11 @@ impl Store { AND file_size_bytes = ?5 AND file_modified_at_ms = ?6 AND import_revision = ?12 + AND ((?14 IS NULL AND json_extract( + metadata_json, + '$.file_observation_token_v1' + ) IS NULL) + OR metadata_json IS ?14) AND EXISTS ( SELECT 1 FROM import_inventory_generations AS inventory @@ -157,6 +662,7 @@ impl Store { advance_legacy_cursor, i64::from(update.import_revision), capped_i64(update.inventory_generation), + metadata.map(serde_json::to_string).transpose()?, ], )?; Ok(changed) diff --git a/crates/ctx-history-store/src/catalog/source_imports.rs b/crates/ctx-history-store/src/catalog/source_imports.rs index 590f411d..f3584001 100644 --- a/crates/ctx-history-store/src/catalog/source_imports.rs +++ b/crates/ctx-history-store/src/catalog/source_imports.rs @@ -1,8 +1,339 @@ +const SOURCE_IMPORT_PERSIST_BATCH_ROWS: usize = 64; +const SOURCE_IMPORT_PERSIST_BATCH_BYTES: u64 = 8 * 1024 * 1024; +const SOURCE_IMPORT_PERSIST_ROW_OVERHEAD_BYTES: u64 = 256; + +fn source_import_persist_row_bytes(file: &SourceImportFile, metadata_json: &str) -> u64 { + [ + file.provider.as_str(), + file.source_format.as_str(), + file.source_root.as_str(), + file.source_path.as_str(), + metadata_json, + ] + .into_iter() + .fold(SOURCE_IMPORT_PERSIST_ROW_OVERHEAD_BYTES, |bytes, value| { + bytes.saturating_add(value.len() as u64) + }) +} + +fn source_import_current_path_batch(current_paths: &[String]) -> (usize, u64) { + let mut count = 0; + let mut bytes = 0_u64; + for source_path in current_paths.iter().take(SOURCE_IMPORT_PERSIST_BATCH_ROWS) { + let row_bytes = + SOURCE_IMPORT_PERSIST_ROW_OVERHEAD_BYTES.saturating_add(source_path.len() as u64); + if count > 0 && bytes.saturating_add(row_bytes) > SOURCE_IMPORT_PERSIST_BATCH_BYTES { + break; + } + count += 1; + bytes = bytes.saturating_add(row_bytes); + if bytes >= SOURCE_IMPORT_PERSIST_BATCH_BYTES { + break; + } + } + (count, bytes) +} + impl Store { + fn classify_source_import_pending_reason( + &self, + file: &SourceImportFile, + metadata_json: &str, + ) -> Result> { + let prior = self + .conn + .query_row( + r#" + SELECT source_format, file_size_bytes, file_modified_at_ms, + import_revision, is_stale, + indexed_file_size_bytes, indexed_file_modified_at_ms, + indexed_status, indexed_import_revision, pending_reason, metadata_json + FROM source_import_files + WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 + "#, + params![file.provider.as_str(), &file.source_root, &file.source_path], + |row| { + Ok(SourceImportPendingState { + source_format: row.get(0)?, + file_size_bytes: nonnegative_i64_to_u64(row.get(1)?)?, + file_modified_at_ms: row.get(2)?, + import_revision: nonnegative_i64_to_u32(row.get(3)?)?, + is_stale: row.get(4)?, + indexed_file_size_bytes: row + .get::<_, Option>(5)? + .map(nonnegative_i64_to_u64) + .transpose()?, + indexed_file_modified_at_ms: row.get(6)?, + indexed_status: parse_text_enum(row.get(7)?)?, + indexed_import_revision: row + .get::<_, Option>(8)? + .map(nonnegative_i64_to_u32) + .transpose()?, + pending_reason: row + .get::<_, Option>(9)? + .map(parse_text_enum) + .transpose()?, + metadata_json: row.get(10)?, + }) + }, + ) + .optional()?; + let Some(prior) = prior else { + return Ok(Some(ImportPendingReason::FreshNew)); + }; + if self.provider_file_publication_was_abandoned( + file.provider, + "source_import_files", + &prior.source_format, + &file.source_root, + &file.source_path, + )? { + return Ok(Some(ImportPendingReason::AbandonedPublication)); + } + let same_identity = prior.source_format == file.source_format; + let same_fingerprint = same_identity + && prior.file_size_bytes == file.file_size_bytes + && prior.file_modified_at_ms == file.file_modified_at_ms + && prior.import_revision == file.import_revision + && prior.metadata_json == metadata_json + && !prior.is_stale; + if same_fingerprint && prior.pending_reason == Some(ImportPendingReason::ExplicitRescan) { + return Ok(prior.pending_reason); + } + if !same_fingerprint { + if let Some(reason) = prior + .pending_reason + .filter(|reason| reason.requires_replacement()) + { + return Ok(Some(reason)); + } + let parser_revision_only = same_identity + && prior.file_size_bytes == file.file_size_bytes + && prior.file_modified_at_ms == file.file_modified_at_ms + && prior.metadata_json == metadata_json + && prior.import_revision != file.import_revision + && !prior.is_stale; + if parser_revision_only { + return Ok(Some(ImportPendingReason::ParserRevision)); + } + let grew_in_place = same_identity + && prior.import_revision == file.import_revision + && source_import_metadata_matches_owner_growth( + &prior.metadata_json, + &file.metadata, + ) + && !prior.is_stale + && file.file_size_bytes > prior.file_size_bytes; + if grew_in_place + && matches!( + prior.pending_reason, + Some(ImportPendingReason::FreshAppend | ImportPendingReason::RecoveryRetry) + ) + && self.source_import_incremental_material_is_supported(&prior, file)? + { + return Ok(prior.pending_reason); + } + if self.source_import_observation_is_append(&prior, file)? { + return Ok(Some(ImportPendingReason::FreshAppend)); + } + return Ok(Some(ImportPendingReason::FreshChanged)); + } + match prior.indexed_status { + CatalogIndexedStatus::Failed => Ok(Some(ImportPendingReason::retry_after_failure( + prior.pending_reason, + ))), + CatalogIndexedStatus::Pending => Ok(Some( + prior.pending_reason.unwrap_or(ImportPendingReason::Legacy), + )), + CatalogIndexedStatus::Indexed | CatalogIndexedStatus::CompletedWithRejections => { + let indexed_matches = prior.indexed_file_size_bytes == Some(file.file_size_bytes) + && prior.indexed_file_modified_at_ms == Some(file.file_modified_at_ms); + if prior.indexed_import_revision != Some(file.import_revision) { + Ok(Some(ImportPendingReason::ParserRevision)) + } else if !indexed_matches { + Ok(Some( + prior.pending_reason.unwrap_or(ImportPendingReason::Legacy), + )) + } else if !self.source_import_material_exists(file)? { + Ok(Some(ImportPendingReason::MissingMaterial)) + } else { + Ok(None) + } + } + CatalogIndexedStatus::Rejected => Ok(None), + } + } + + fn source_import_observation_is_append( + &self, + prior: &SourceImportPendingState, + file: &SourceImportFile, + ) -> Result { + if prior.source_format != file.source_format + || prior.import_revision != file.import_revision + || prior.is_stale + || file.file_size_bytes <= prior.file_size_bytes + || !matches!( + prior.indexed_status, + CatalogIndexedStatus::Indexed | CatalogIndexedStatus::CompletedWithRejections + ) + || prior.indexed_file_size_bytes != Some(prior.file_size_bytes) + || prior.indexed_file_modified_at_ms != Some(prior.file_modified_at_ms) + || prior.indexed_import_revision != Some(prior.import_revision) + { + return Ok(false); + } + Ok(self.provider_file_checkpoint_matches_prior_observation( + file.provider, + &file.source_format, + &file.source_root, + &file.source_path, + file.import_revision, + prior.file_size_bytes, + )? && self.source_import_material_exists(file)?) + } + + fn source_import_incremental_material_is_supported( + &self, + prior: &SourceImportPendingState, + file: &SourceImportFile, + ) -> Result { + let checkpoint_precedes_growth = self + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider: file.provider, + source_format: &file.source_format, + source_root: &file.source_root, + source_path: &file.source_path, + })? + .is_some_and(|checkpoint| { + checkpoint.import_revision == file.import_revision + && checkpoint.committed_byte_offset <= prior.file_size_bytes + }); + Ok(checkpoint_precedes_growth && self.source_import_material_exists(file)?) + } + + fn source_import_material_exists(&self, file: &SourceImportFile) -> Result { + let metadata_json = serde_json::to_string(&file.metadata)?; + let material_source_format = + expected_material_source_format(file.provider, &file.source_format); + self.conn + .query_row( + r#" + SELECT EXISTS ( + SELECT 1 + FROM capture_sources AS source + WHERE source.provider = ?1 + AND source.source_format = ?2 + AND ( + ( + json_extract(?5, '$.inventory_unit') = 'source_root' + AND source.source_root = ?3 + ) + OR ( + json_extract(?5, '$.inventory_unit') IS NOT 'source_root' + AND source.raw_source_path = ?4 + AND ( + source.source_root = ?3 + OR source.source_root = source.raw_source_path + OR source.source_root IS NULL + ) + ) + ) + LIMIT 1 + ) + "#, + params![ + file.provider.as_str(), + material_source_format, + &file.source_root, + &file.source_path, + metadata_json + ], + |row| row.get(0), + ) + .map_err(Into::into) + } + + fn provider_file_checkpoint_matches_prior_observation( + &self, + provider: CaptureProvider, + source_format: &str, + source_root: &str, + source_path: &str, + import_revision: u32, + prior_size_bytes: u64, + ) -> Result { + self.conn + .query_row( + r#" + SELECT EXISTS ( + SELECT 1 + FROM provider_file_checkpoints + WHERE provider = ?1 AND source_format = ?2 + AND source_root = ?3 AND source_path = ?4 + AND import_revision = ?5 + AND committed_byte_offset <= ?6 + ) + "#, + params![ + provider.as_str(), + source_format, + source_root, + source_path, + i64::from(import_revision), + capped_i64(prior_size_bytes) + ], + |row| row.get(0), + ) + .map_err(Into::into) + } + + fn provider_file_publication_was_abandoned( + &self, + provider: CaptureProvider, + inventory_family: &str, + source_format: &str, + source_root: &str, + source_path: &str, + ) -> Result { + self.conn + .query_row( + r#" + SELECT EXISTS ( + SELECT 1 + FROM provider_file_publications + WHERE provider = ?1 AND inventory_family = ?2 + AND inventory_source_format = ?3 + AND inventory_source_root = ?4 AND source_path = ?5 + AND mutation_started = 1 + ) + "#, + params![ + provider.as_str(), + inventory_family, + source_format, + source_root, + source_path + ], + |row| row.get(0), + ) + .map_err(Into::into) + } + pub fn upsert_source_import_files( &self, inventory_generation: u64, files: &[SourceImportFile], + ) -> Result { + self.upsert_source_import_files_with_pacing(inventory_generation, files, |_| {}) + } + + #[doc(hidden)] + pub fn upsert_source_import_files_with_pacing( + &self, + inventory_generation: u64, + files: &[SourceImportFile], + mut pace: impl FnMut(u64), ) -> Result { if files.is_empty() { return Ok(0); @@ -12,16 +343,16 @@ impl Store { INSERT INTO source_import_files ( provider, source_format, source_root, source_path, file_size_bytes, file_modified_at_ms, import_revision, observed_at_ms, is_stale, - metadata_json + pending_reason, metadata_json ) - SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 0, ?9 + SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 0, ?9, ?10 WHERE EXISTS ( SELECT 1 FROM import_inventory_generations AS inventory WHERE inventory.provider = ?1 AND inventory.source_root = ?3 AND inventory.inventory_family = 'source_import_files' - AND inventory.current_generation = ?10 + AND inventory.current_generation = ?11 ) ON CONFLICT(provider, source_root, source_path) DO UPDATE SET source_format = excluded.source_format, @@ -112,6 +443,7 @@ impl Store { THEN source_import_files.indexed_import_revision ELSE NULL END, + pending_reason = excluded.pending_reason, metadata_json = excluded.metadata_json WHERE EXISTS ( SELECT 1 @@ -119,7 +451,7 @@ impl Store { WHERE inventory.provider = excluded.provider AND inventory.source_root = excluded.source_root AND inventory.inventory_family = 'source_import_files' - AND inventory.current_generation = ?10 + AND inventory.current_generation = ?11 ) AND ( source_import_files.source_format IS NOT excluded.source_format @@ -128,23 +460,52 @@ impl Store { OR source_import_files.import_revision != excluded.import_revision OR source_import_files.is_stale != 0 OR source_import_files.metadata_json IS NOT excluded.metadata_json + OR source_import_files.pending_reason IS NOT excluded.pending_reason ) "#, )?; let mut changed = 0; - for file in files { - changed += stmt.execute(params![ - file.provider.as_str(), - file.source_format.as_str(), - file.source_root.as_str(), - file.source_path.as_str(), - capped_i64(file.file_size_bytes), - file.file_modified_at_ms, - i64::from(file.import_revision), - file.observed_at_ms, - serde_json::to_string(&file.metadata)?, - capped_i64(inventory_generation), - ])?; + let mut start = 0; + while start < files.len() { + let mut batch = Vec::with_capacity( + SOURCE_IMPORT_PERSIST_BATCH_ROWS.min(files.len().saturating_sub(start)), + ); + let mut batch_bytes = 0_u64; + for file in files[start..].iter().take(SOURCE_IMPORT_PERSIST_BATCH_ROWS) { + let metadata_json = serde_json::to_string(&file.metadata)?; + let row_bytes = source_import_persist_row_bytes(file, &metadata_json); + if !batch.is_empty() + && batch_bytes.saturating_add(row_bytes) > SOURCE_IMPORT_PERSIST_BATCH_BYTES + { + break; + } + batch_bytes = batch_bytes.saturating_add(row_bytes); + batch.push((file, metadata_json)); + if batch_bytes >= SOURCE_IMPORT_PERSIST_BATCH_BYTES { + break; + } + } + + pace(batch_bytes); + let batch_len = batch.len(); + for (file, metadata_json) in batch { + let pending_reason = + self.classify_source_import_pending_reason(file, &metadata_json)?; + changed += stmt.execute(params![ + file.provider.as_str(), + file.source_format.as_str(), + file.source_root.as_str(), + file.source_path.as_str(), + capped_i64(file.file_size_bytes), + file.file_modified_at_ms, + i64::from(file.import_revision), + file.observed_at_ms, + pending_reason.map(ImportPendingReason::as_str), + metadata_json, + capped_i64(inventory_generation), + ])?; + } + start = start.saturating_add(batch_len); } Ok(changed) } @@ -157,6 +518,38 @@ impl Store { observed_at_ms: i64, inventory_generation: u64, ) -> Result { + self.mark_source_import_missing_paths_stale_with_pacing( + provider, + source_root, + current_paths, + observed_at_ms, + inventory_generation, + |_| {}, + ) + } + + #[doc(hidden)] + pub fn mark_source_import_missing_paths_stale_with_pacing( + &self, + provider: CaptureProvider, + source_root: &str, + current_paths: &[String], + observed_at_ms: i64, + inventory_generation: u64, + mut pace: impl FnMut(u64), + ) -> Result { + if self.conn.is_autocommit() { + return with_immediate_transaction(&self.conn, || { + self.mark_source_import_missing_paths_stale_with_pacing( + provider, + source_root, + current_paths, + observed_at_ms, + inventory_generation, + pace, + ) + }); + } self.conn.execute_batch( "CREATE TEMP TABLE IF NOT EXISTS temp_source_import_current_paths (source_path TEXT PRIMARY KEY)", )?; @@ -166,38 +559,104 @@ impl Store { let mut stmt = self.conn.prepare( "INSERT OR IGNORE INTO temp_source_import_current_paths (source_path) VALUES (?1)", )?; - for source_path in current_paths { - stmt.execute(params![source_path])?; + let mut start = 0; + while start < current_paths.len() { + let (batch_len, batch_bytes) = + source_import_current_path_batch(¤t_paths[start..]); + pace(batch_bytes); + for source_path in ¤t_paths[start..start + batch_len] { + stmt.execute(params![source_path])?; + } + start = start.saturating_add(batch_len); } } - let changed = self.conn.execute( - r#" + let mut changed = 0usize; + loop { + let (batch_rows, batch_bytes) = self.conn.query_row( + r#" + SELECT COUNT(*), COALESCE(SUM( + length(provider) + length(source_format) + length(source_root) + + length(source_path) + length(metadata_json) + ?4 + ), 0) + FROM ( + SELECT provider, source_format, source_root, source_path, metadata_json + FROM source_import_files + WHERE provider = ?1 + AND source_root = ?2 + AND is_stale = 0 + AND EXISTS ( + SELECT 1 + FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 + AND inventory.source_root = ?2 + AND inventory.inventory_family = 'source_import_files' + AND inventory.current_generation = ?3 + ) + AND NOT EXISTS ( + SELECT 1 + FROM temp_source_import_current_paths AS current + WHERE current.source_path = source_import_files.source_path + ) + ORDER BY source_path + LIMIT 64 + ) + "#, + params![ + provider.as_str(), + source_root, + capped_i64(inventory_generation), + capped_i64(SOURCE_IMPORT_PERSIST_ROW_OVERHEAD_BYTES), + ], + |row| { + Ok(( + row.get::<_, usize>(0)?, + nonnegative_i64_to_u64(row.get(1)?)?, + )) + }, + )?; + if batch_rows == 0 { + break; + } + pace(batch_bytes); + let batch_changed = self.conn.execute( + r#" UPDATE source_import_files SET is_stale = 1, observed_at_ms = ?3 - WHERE provider = ?1 - AND source_root = ?2 - AND is_stale = 0 - AND EXISTS ( - SELECT 1 - FROM import_inventory_generations AS inventory - WHERE inventory.provider = ?1 - AND inventory.source_root = ?2 - AND inventory.inventory_family = 'source_import_files' - AND inventory.current_generation = ?4 - ) - AND NOT EXISTS ( - SELECT 1 - FROM temp_source_import_current_paths AS current - WHERE current.source_path = source_import_files.source_path + WHERE rowid IN ( + SELECT rowid + FROM source_import_files + WHERE provider = ?1 + AND source_root = ?2 + AND is_stale = 0 + AND EXISTS ( + SELECT 1 + FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 + AND inventory.source_root = ?2 + AND inventory.inventory_family = 'source_import_files' + AND inventory.current_generation = ?4 + ) + AND NOT EXISTS ( + SELECT 1 + FROM temp_source_import_current_paths AS current + WHERE current.source_path = source_import_files.source_path + ) + ORDER BY source_path + LIMIT 64 ) "#, - params![ - provider.as_str(), - source_root, - observed_at_ms, - capped_i64(inventory_generation) - ], - )?; + params![ + provider.as_str(), + source_root, + observed_at_ms, + capped_i64(inventory_generation) + ], + )?; + changed = changed.saturating_add(batch_changed); + if batch_changed == 0 { + break; + } + } self.conn .execute("DELETE FROM temp_source_import_current_paths", [])?; Ok(changed) @@ -231,6 +690,189 @@ impl Store { collect_rows(rows) } + pub fn list_source_import_file_work( + &self, + provider: CaptureProvider, + source_root: &str, + class: ImportWorkClass, + limit: usize, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let predicate = import_work_class_predicate("source_file", class); + let active_publication = + crate::provider_files::source_file_candidate_is_global_publication("source_file"); + let order = import_work_order("source_file", class); + let active_path = self + .effective_provider_file_publication_inventory_owner()? + .filter(|owner| { + owner.inventory_family == ProviderFileInventoryFamily::SourceImport + && owner.provider == provider + && owner.source_root == source_root + }) + .map(|owner| owner.source_path); + let mut work = Vec::with_capacity(limit); + if let Some(active_path) = active_path.as_deref() { + let mut active_stmt = self.conn.prepare(&format!( + r#" + SELECT provider, source_format, source_root, source_path, + file_size_bytes, file_modified_at_ms, import_revision, + observed_at_ms, metadata_json, pending_reason, + CASE + WHEN pending_reason = 'fresh_append' THEN MAX( + file_size_bytes - COALESCE(( + SELECT checkpoint.committed_byte_offset + FROM provider_file_checkpoints AS checkpoint + WHERE checkpoint.provider = source_file.provider + AND checkpoint.source_format = source_file.source_format + AND checkpoint.source_root = source_file.source_root + AND checkpoint.source_path = source_file.source_path + ), 0), + 0 + ) + ELSE file_size_bytes + END, + indexed_at_ms, 1 AS has_active_publication + FROM source_import_files AS source_file + WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 + AND is_stale = 0 AND {predicate} AND ({active_publication}) + LIMIT 1 + "# + ))?; + work.extend( + active_stmt + .query_map( + params![provider.as_str(), source_root, active_path], + source_import_file_work_from_row, + )? + .collect::>>()?, + ); + } + if work.len() >= limit { + return Ok(work); + } + let ordinary_limit = limit.saturating_add(usize::from(!work.is_empty())); + let mut stmt = self.conn.prepare(&format!( + r#" + SELECT provider, source_format, source_root, source_path, + file_size_bytes, file_modified_at_ms, import_revision, + observed_at_ms, metadata_json, pending_reason, + CASE + WHEN pending_reason = 'fresh_append' THEN MAX( + file_size_bytes - COALESCE(( + SELECT checkpoint.committed_byte_offset + FROM provider_file_checkpoints AS checkpoint + WHERE checkpoint.provider = source_file.provider + AND checkpoint.source_format = source_file.source_format + AND checkpoint.source_root = source_file.source_root + AND checkpoint.source_path = source_file.source_path + ), 0), + 0 + ) + ELSE file_size_bytes + END, + indexed_at_ms, 0 AS has_active_publication + FROM source_import_files AS source_file + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 + AND {predicate} + ORDER BY {order} + LIMIT ?3 + "# + ))?; + let rows = stmt.query_map( + params![ + provider.as_str(), + source_root, + capped_i64(ordinary_limit as u64) + ], + source_import_file_work_from_row, + )?; + let ordinary = collect_rows(rows)?; + let remaining = limit.saturating_sub(work.len()); + work.extend( + ordinary + .into_iter() + .filter(|candidate| { + active_path + .as_deref() + .is_none_or(|path| candidate.file.source_path != path) + }) + .take(remaining), + ); + Ok(work) + } + + pub fn source_import_file_work_count( + &self, + provider: CaptureProvider, + source_root: &str, + class: ImportWorkClass, + ) -> Result { + let predicate = import_work_class_predicate("source_file", class); + self.conn + .query_row( + &format!( + "SELECT COUNT(*) FROM source_import_files AS source_file \ + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 \ + AND {predicate}" + ), + params![provider.as_str(), source_root], + |row| row.get(0), + ) + .map_err(Into::into) + } + + pub fn schedule_source_import_explicit_rescan( + &self, + provider: CaptureProvider, + source_root: &str, + inventory_generation: u64, + ) -> Result { + with_immediate_transaction(&self.conn, || { + let legacy = self.conn.execute( + r#" + UPDATE source_import_files + SET pending_reason = 'legacy' + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 + AND indexed_status IN ('pending', 'failed') AND pending_reason IS NULL + AND EXISTS ( + SELECT 1 FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 AND inventory.source_root = ?2 + AND inventory.inventory_family = 'source_import_files' + AND inventory.current_generation = ?3 + ) + "#, + params![ + provider.as_str(), + source_root, + capped_i64(inventory_generation) + ], + )?; + let explicit = self.conn.execute( + r#" + UPDATE source_import_files + SET pending_reason = 'explicit_rescan' + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 + AND indexed_status = 'indexed' + AND pending_reason IS NULL + AND EXISTS ( + SELECT 1 FROM import_inventory_generations AS inventory + WHERE inventory.provider = ?1 AND inventory.source_root = ?2 + AND inventory.inventory_family = 'source_import_files' + AND inventory.current_generation = ?3 + ) + "#, + params![ + provider.as_str(), + source_root, + capped_i64(inventory_generation) + ], + )?; + Ok(legacy.saturating_add(explicit)) + }) + } + pub fn mark_source_import_file_indexed( &self, provider: CaptureProvider, @@ -269,7 +911,16 @@ impl Store { indexed_file_modified_at_ms = ?6, indexed_status = ?7, indexed_error = ?8, - indexed_import_revision = ?9 + indexed_import_revision = ?9, + pending_reason = CASE + WHEN ?7 = 'failed' THEN CASE + WHEN pending_reason IN ('fresh_append', 'recovery_retry') + THEN 'recovery_retry' + ELSE 'recovery_replacement' + END + WHEN ?7 = 'pending' THEN COALESCE(pending_reason, 'legacy') + ELSE NULL + END WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 diff --git a/crates/ctx-history-store/src/catalog/tests.rs b/crates/ctx-history-store/src/catalog/tests.rs index 72f07c60..db9f09b4 100644 --- a/crates/ctx-history-store/src/catalog/tests.rs +++ b/crates/ctx-history-store/src/catalog/tests.rs @@ -5,12 +5,14 @@ use ctx_history_core::{ new_id, AgentType, Artifact, ArtifactKind, CaptureProvider, CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, EntityTimestamps, Event, EventRole, EventType, Fidelity, Session, SessionStatus, SyncMetadata, SyncState, Visibility, + PROVIDER_MATERIAL_SOURCE_FORMATS, }; use rusqlite::{ffi::ErrorCode, params}; use uuid::Uuid; use crate::catalog::{ - CatalogIndexedStatus, CatalogSession, CatalogSourceIndexUpdate, SourceImportFile, + expected_material_source_format, CatalogIndexedStatus, CatalogSession, + CatalogSourceIndexUpdate, ImportPendingReason, ImportWorkClass, SourceImportFile, SourceImportFileIndexUpdate, }; use crate::connection::timestamp_ms; @@ -186,6 +188,33 @@ fn current_source_generation(store: &Store, provider: CaptureProvider, source_ro current_inventory_generation(store, provider, source_root, "source_import_files") } +fn insert_matching_checkpoint(store: &Store, file: &SourceImportFile) { + store + .conn + .execute( + r#" + INSERT INTO provider_file_checkpoints ( + provider, source_format, source_root, source_path, import_revision, + checkpoint_version, stable_file_identity, committed_byte_offset, + committed_complete_line_count, head_sha256, boundary_sha256, updated_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5, 1, 'test-file', ?6, 0, ?7, ?8, ?9) + "#, + params![ + file.provider.as_str(), + &file.source_format, + &file.source_root, + &file.source_path, + i64::from(file.import_revision), + file.file_size_bytes, + "a".repeat(64), + "b".repeat(64), + file.observed_at_ms, + ], + ) + .unwrap(); +} + +include!("tests/pending_reasons.rs"); include!("tests/catalog_imports.rs"); include!("tests/source_imports.rs"); include!("tests/queries.rs"); diff --git a/crates/ctx-history-store/src/catalog/tests/catalog_imports.rs b/crates/ctx-history-store/src/catalog/tests/catalog_imports.rs index 2035795b..efc2f15b 100644 --- a/crates/ctx-history-store/src/catalog/tests/catalog_imports.rs +++ b/crates/ctx-history-store/src/catalog/tests/catalog_imports.rs @@ -48,6 +48,33 @@ fn imported_source(source_id: Uuid, source_root: &str, external_session_id: &str } } +fn upsert_catalog_material( + store: &Store, + source_root: &str, + source_path: &str, + external_session_id: &str, +) { + let source_id = new_id(); + let mut source = imported_source(source_id, source_root, external_session_id); + source.descriptor.raw_source_path = Some(source_path.to_owned()); + store.upsert_capture_source(&source).unwrap(); + store + .upsert_session(&source_scoped_imported_session( + external_session_id, + source_id, + )) + .unwrap(); +} + +fn upsert_source_material(store: &Store, file: &SourceImportFile) { + let mut source = imported_source(new_id(), &file.source_root, "source-material"); + source.descriptor.provider = file.provider; + source.descriptor.raw_source_path = Some(file.source_path.clone()); + source.descriptor.source_format = + Some(expected_material_source_format(file.provider, &file.source_format).to_owned()); + store.upsert_capture_source(&source).unwrap(); +} + fn session_event(session_id: Uuid, index: u64) -> Event { Event { id: new_id(), @@ -237,9 +264,12 @@ fn catalog_sessions_count_indexed_and_stale_rows() { 1 ); - store - .upsert_session(&imported_session("codex-session-1")) - .unwrap(); + upsert_catalog_material( + &store, + "/home/user/.codex/sessions", + "/home/user/.codex/sessions/2026/06/24/rollout.jsonl", + "codex-session-1", + ); store .mark_catalog_source_indexed( CaptureProvider::Codex, @@ -327,9 +357,12 @@ fn catalog_import_planning_requires_current_index_state_and_matching_session() { assert_eq!(pending.len(), 1); assert_eq!(store.catalog_session_counts().unwrap().indexed, 0); - store - .upsert_session(&imported_session("codex-session-1")) - .unwrap(); + upsert_catalog_material( + &store, + "/home/user/.codex/sessions", + "/home/user/.codex/sessions/2026/06/24/rollout.jsonl", + "codex-session-1", + ); let pending = store .list_pending_catalog_sessions(CaptureProvider::Codex, "/home/user/.codex/sessions") .unwrap(); @@ -384,20 +417,7 @@ fn catalog_import_planning_scopes_matching_sessions_by_source_root() { .unwrap(); } - let first_source_id = new_id(); - store - .upsert_capture_source(&imported_source( - first_source_id, - first_root, - external_session_id, - )) - .unwrap(); - store - .upsert_session(&source_scoped_imported_session( - external_session_id, - first_source_id, - )) - .unwrap(); + upsert_catalog_material(&store, first_root, first_path, external_session_id); assert!(store .list_pending_catalog_sessions(CaptureProvider::Codex, first_root) @@ -413,6 +433,81 @@ fn catalog_import_planning_scopes_matching_sessions_by_source_root() { assert_eq!(counts.pending, 1); } +#[test] +fn catalog_material_requires_the_exact_owned_codex_tuple() { + for scenario in ["unowned", "wrong_format", "wrong_root", "wrong_external"] { + let temp = tempdir(); + let store = Store::open(temp.path().join(format!("{scenario}.sqlite"))).unwrap(); + let root = "/fixture/codex/sessions"; + let path = "/fixture/codex/sessions/session.jsonl"; + let external_session_id = "exact-owner"; + let observed_at_ms = timestamp_ms(fixed_time()); + let mut catalog = catalog_session_for_root(root, path, external_session_id, observed_at_ms); + upsert_catalog_inventory(&store, std::slice::from_ref(&catalog)); + store + .mark_catalog_source_indexed( + catalog.provider, + CatalogSourceIndexUpdate { + source_root: root, + source_path: path, + file_size_bytes: catalog.file_size_bytes, + file_modified_at_ms: catalog.file_modified_at_ms, + import_revision: catalog.import_revision, + inventory_generation: current_catalog_generation( + &store, + catalog.provider, + root, + ), + file_sha256: None, + event_count: Some(1), + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + + if scenario == "unowned" { + store + .upsert_session(&imported_session(external_session_id)) + .unwrap(); + } else { + let source_id = new_id(); + let mut source = imported_source(source_id, root, external_session_id); + source.descriptor.raw_source_path = Some(path.to_owned()); + match scenario { + "wrong_format" => { + source.descriptor.source_format = Some("codex_session_jsonl_tree".into()) + } + "wrong_root" => source.descriptor.source_root = Some("/fixture/other".into()), + "wrong_external" => { + source.descriptor.external_session_id = Some("other-session".into()) + } + _ => unreachable!(), + } + store.upsert_capture_source(&source).unwrap(); + store + .upsert_session(&source_scoped_imported_session( + external_session_id, + source_id, + )) + .unwrap(); + } + + catalog.cataloged_at_ms += 1; + upsert_catalog_inventory(&store, std::slice::from_ref(&catalog)); + let recovery = store + .list_catalog_import_work(catalog.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!( + recovery[0].reason, + ImportPendingReason::MissingMaterial, + "{scenario}" + ); + let counts = store.catalog_session_counts().unwrap(); + assert_eq!(counts.indexed, 0, "{scenario}"); + assert_eq!(counts.pending, 1, "{scenario}"); + } +} + #[test] fn catalog_import_mark_failed_records_error_and_remains_pending() { let temp = tempdir(); @@ -481,9 +576,12 @@ fn catalog_upsert_clears_completion_metadata_but_preserves_append_checkpoint() { cataloged_at_ms, )], ); - store - .upsert_session(&imported_session("codex-session-1")) - .unwrap(); + upsert_catalog_material( + &store, + "/home/user/.codex/sessions", + source_path, + "codex-session-1", + ); store .mark_catalog_source_indexed( CaptureProvider::Codex, @@ -581,16 +679,28 @@ fn catalog_upsert_clears_completion_metadata_but_preserves_append_checkpoint() { } #[test] -fn completed_with_rejections_converges_without_advancing_safe_resume_checkpoint() { +fn completed_with_rejections_self_rooted_material_resumes_from_safe_tail() { let temp = tempdir(); let store = Store::open(temp.path().join("work.sqlite")).unwrap(); let observed_at_ms = timestamp_ms(fixed_time()); let source_path = "/home/user/.codex/sessions/2026/06/24/mixed-tail.jsonl"; let initial = catalog_session(source_path, "mixed-tail", observed_at_ms); - upsert_catalog_inventory(&store, &[initial]); + upsert_catalog_inventory(&store, &[initial.clone()]); + upsert_catalog_material( + &store, + "/home/user/.codex/sessions", + source_path, + "mixed-tail", + ); store - .upsert_session(&imported_session("mixed-tail")) + .conn + .execute( + "UPDATE capture_sources SET source_root = raw_source_path \ + WHERE provider = 'codex' AND external_session_id = 'mixed-tail'", + [], + ) .unwrap(); + assert!(store.catalog_session_material_exists(&initial).unwrap()); store .mark_catalog_source_indexed( CaptureProvider::Codex, @@ -615,28 +725,31 @@ fn completed_with_rejections_converges_without_advancing_safe_resume_checkpoint( let mut appended = catalog_session(source_path, "mixed-tail", observed_at_ms + 1); appended.file_size_bytes = 64; upsert_catalog_inventory(&store, &[appended]); - store - .record_catalog_source_import_result( - CaptureProvider::Codex, - CatalogSourceIndexUpdate { - source_root: "/home/user/.codex/sessions", - source_path, - file_size_bytes: 64, - file_modified_at_ms: observed_at_ms + 1, - import_revision: 1, - inventory_generation: current_catalog_generation( - &store, - CaptureProvider::Codex, - "/home/user/.codex/sessions", - ), - file_sha256: None, - event_count: Some(4), - indexed_at_ms: observed_at_ms + 20, - }, - CatalogIndexedStatus::CompletedWithRejections, - Some("line 4: malformed record"), - ) - .unwrap(); + assert_eq!( + store + .record_catalog_source_import_result( + CaptureProvider::Codex, + CatalogSourceIndexUpdate { + source_root: "/home/user/.codex/sessions", + source_path, + file_size_bytes: 64, + file_modified_at_ms: observed_at_ms + 1, + import_revision: 1, + inventory_generation: current_catalog_generation( + &store, + CaptureProvider::Codex, + "/home/user/.codex/sessions", + ), + file_sha256: None, + event_count: Some(4), + indexed_at_ms: observed_at_ms + 20, + }, + CatalogIndexedStatus::CompletedWithRejections, + Some("line 4: malformed record"), + ) + .unwrap(), + 1 + ); assert!(store .list_pending_catalog_sessions(CaptureProvider::Codex, "/home/user/.codex/sessions") @@ -947,6 +1060,7 @@ fn source_outcomes_converge_and_revision_invalidation_is_provider_scoped() { ) .unwrap(); } + upsert_source_material(&store, &claude); assert!(store .list_pending_source_import_files(CaptureProvider::Claude, claude_root) diff --git a/crates/ctx-history-store/src/catalog/tests/pending_reasons.rs b/crates/ctx-history-store/src/catalog/tests/pending_reasons.rs new file mode 100644 index 00000000..9507f405 --- /dev/null +++ b/crates/ctx-history-store/src/catalog/tests/pending_reasons.rs @@ -0,0 +1,1077 @@ +#[test] +fn fresh_schema_stages_both_pending_reason_repairs() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let repairs = store + .conn + .prepare( + "SELECT inventory_family, cursor_provider, cursor_source_root, \ + cursor_source_path, completed \ + FROM import_pending_reason_repairs ORDER BY inventory_family", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, bool>(4)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + repairs, + vec![ + ("catalog_sessions".into(), None, None, None, true), + ("source_import_files".into(), None, None, None, true), + ] + ); +} + +#[test] +fn pending_reason_repair_is_bounded_resumable_idempotent_and_conservative() { + let temp = tempdir(); + let path = temp.path().join("catalog.sqlite"); + let store = Store::open(&path).unwrap(); + store + .conn + .execute_batch( + r#" + INSERT INTO catalog_sessions ( + source_path, provider, source_format, source_root, external_session_id, + agent_type, file_size_bytes, file_modified_at_ms, import_revision, + cataloged_at_ms, indexed_at_ms, indexed_file_size_bytes, + indexed_file_modified_at_ms, indexed_status, indexed_import_revision, + pending_reason + ) VALUES + ('/repair/catalog/01-clean.jsonl', 'codex', 'codex_session_jsonl', + '/repair/catalog', 'clean', 'primary', 10, 10, 1, 10, + 20, 10, 10, 'indexed', 1, NULL), + ('/repair/catalog/02-pending.jsonl', 'codex', 'codex_session_jsonl', + '/repair/catalog', 'pending', 'primary', 10, 10, 1, 10, + NULL, NULL, NULL, 'pending', NULL, NULL), + ('/repair/catalog/03-rejected.jsonl', 'codex', 'codex_session_jsonl', + '/repair/catalog', 'rejected', 'primary', 10, 10, 1, 10, + 20, NULL, NULL, 'rejected', NULL, NULL), + ('/repair/catalog/04-runtime.jsonl', 'codex', 'codex_session_jsonl', + '/repair/catalog', 'runtime', 'primary', 10, 10, 1, 10, + NULL, NULL, NULL, 'pending', NULL, 'fresh_changed'), + ('/repair/catalog/05-missing.jsonl', 'codex', 'codex_session_jsonl', + '/repair/catalog', 'missing', 'primary', 10, 10, 1, 10, + 20, 10, 10, 'indexed', 1, NULL); + + INSERT INTO source_import_files ( + provider, source_format, source_root, source_path, file_size_bytes, + file_modified_at_ms, import_revision, observed_at_ms, is_stale, + indexed_at_ms, indexed_file_size_bytes, indexed_file_modified_at_ms, + indexed_status, indexed_import_revision, pending_reason + ) VALUES + ('pi', 'pi_session_jsonl', '/repair/source', + '/repair/source/01-clean.jsonl', 10, 10, 1, 10, 0, + 20, 10, 10, 'indexed', 1, NULL), + ('pi', 'pi_session_jsonl', '/repair/source', + '/repair/source/02-failed.jsonl', 10, 10, 1, 10, 0, + 20, 10, 10, 'failed', 1, NULL), + ('pi', 'pi_session_jsonl', '/repair/source', + '/repair/source/03-stale.jsonl', 10, 10, 1, 10, 1, + NULL, NULL, NULL, 'pending', NULL, NULL), + ('pi', 'pi_session_jsonl', '/repair/source', + '/repair/source/04-runtime.jsonl', 10, 10, 1, 10, 0, + NULL, NULL, NULL, 'pending', NULL, 'missing_material'), + ('pi', 'pi_session_jsonl', '/repair/source', + '/repair/source/05-missing.jsonl', 10, 10, 1, 10, 0, + 20, 10, 10, 'indexed', 1, NULL); + + INSERT INTO capture_sources ( + id, kind, provider, machine_id, raw_source_path, source_format, + source_root, external_session_id, started_at_ms, fidelity + ) VALUES + ('repair-catalog-clean', 'provider_import', 'codex', 'fixture', + '/repair/catalog/01-clean.jsonl', 'codex_session_jsonl', + '/repair/catalog', 'clean', 1, 'imported'), + ('repair-source-clean', 'provider_import', 'pi', 'fixture', + '/repair/source/01-clean.jsonl', 'pi_session_jsonl', + '/repair/source', NULL, 1, 'imported'); + + INSERT INTO sessions ( + id, capture_source_id, provider, external_session_id, agent_type, + status, fidelity, started_at_ms, created_at_ms, updated_at_ms + ) VALUES ( + 'repair-session-clean', 'repair-catalog-clean', 'codex', 'clean', + 'primary', 'imported', 'imported', 1, 1, 1 + ); + "#, + ) + .unwrap(); + store + .conn + .execute( + "UPDATE import_pending_reason_repairs \ + SET cursor_provider = NULL, cursor_source_root = NULL, \ + cursor_source_path = NULL, completed = 0", + [], + ) + .unwrap(); + + let first = store.repair_import_pending_reasons(2).unwrap(); + assert_eq!(first.processed_rows, 2); + assert_eq!(first.classified_rows, 1); + assert!(!first.complete); + let first_cursor: String = store + .conn + .query_row( + "SELECT cursor_source_path FROM import_pending_reason_repairs \ + WHERE inventory_family = 'catalog_sessions'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(first_cursor, "/repair/catalog/02-pending.jsonl"); + drop(store); + + let store = Store::open(&path).unwrap(); + let mut processed_rows = first.processed_rows; + let mut classified_rows = first.classified_rows; + let mut completed = false; + for _ in 0..5 { + let progress = store.repair_import_pending_reasons(2).unwrap(); + assert!(progress.processed_rows <= 2); + processed_rows += progress.processed_rows; + classified_rows += progress.classified_rows; + if progress.complete { + completed = true; + break; + } + } + assert!(completed); + assert_eq!(processed_rows, 10); + assert_eq!(classified_rows, 4); + + let catalog_reasons = store + .conn + .prepare("SELECT source_path, pending_reason FROM catalog_sessions ORDER BY source_path") + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + catalog_reasons, + vec![ + ("/repair/catalog/01-clean.jsonl".into(), None), + ( + "/repair/catalog/02-pending.jsonl".into(), + Some("legacy".into()) + ), + ("/repair/catalog/03-rejected.jsonl".into(), None), + ( + "/repair/catalog/04-runtime.jsonl".into(), + Some("fresh_changed".into()) + ), + ( + "/repair/catalog/05-missing.jsonl".into(), + Some("legacy".into()) + ), + ] + ); + let source_reasons = store + .conn + .prepare("SELECT source_path, pending_reason FROM source_import_files ORDER BY source_path") + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + source_reasons, + vec![ + ("/repair/source/01-clean.jsonl".into(), None), + ( + "/repair/source/02-failed.jsonl".into(), + Some("legacy".into()) + ), + ("/repair/source/03-stale.jsonl".into(), None), + ( + "/repair/source/04-runtime.jsonl".into(), + Some("missing_material".into()) + ), + ( + "/repair/source/05-missing.jsonl".into(), + Some("legacy".into()) + ), + ] + ); + + let idempotent = store.repair_import_pending_reasons(2).unwrap(); + assert_eq!(idempotent.processed_rows, 0); + assert_eq!(idempotent.classified_rows, 0); + assert_eq!(idempotent.completed_families, 2); + assert!(idempotent.complete); +} + +#[test] +fn pending_work_orders_never_attempted_then_oldest_attempt_across_reopen() { + let temp = tempdir(); + let path = temp.path().join("catalog.sqlite"); + let store = Store::open(&path).unwrap(); + for (family, reason) in [ + ("fresh", "fresh_changed"), + ("recovery", "recovery_replacement"), + ] { + for (attempt, name) in [ + (None, "never-a"), + (None, "never-b"), + (Some(100), "oldest"), + (Some(200), "newest"), + ] { + store + .conn + .execute( + r#" + INSERT INTO catalog_sessions ( + source_path, provider, source_format, source_root, agent_type, + file_size_bytes, file_modified_at_ms, cataloged_at_ms, + indexed_at_ms, indexed_status, pending_reason + ) VALUES (?1, 'pi', 'pi_session_jsonl', ?2, 'primary', + 1, 1, 1, ?3, 'pending', ?4) + "#, + params![ + format!("/order/catalog/{family}/{name}.jsonl"), + format!("/order/catalog/{family}"), + attempt, + reason, + ], + ) + .unwrap(); + store + .conn + .execute( + r#" + INSERT INTO source_import_files ( + provider, source_format, source_root, source_path, + file_size_bytes, file_modified_at_ms, observed_at_ms, + indexed_at_ms, indexed_status, pending_reason + ) VALUES ('pi', 'pi_session_jsonl', ?1, ?2, + 1, 1, 1, ?3, 'pending', ?4) + "#, + params![ + format!("/order/source/{family}"), + format!("/order/source/{family}/{name}.jsonl"), + attempt, + reason, + ], + ) + .unwrap(); + } + } + drop(store); + + let store = Store::open(&path).unwrap(); + for (family, class) in [ + ("fresh", ImportWorkClass::Fresh), + ("recovery", ImportWorkClass::Recovery), + ] { + let catalog = store + .list_catalog_import_work( + CaptureProvider::Pi, + &format!("/order/catalog/{family}"), + class, + 4, + ) + .unwrap(); + assert_eq!( + catalog + .iter() + .map(|work| { + ( + work.session + .source_path + .rsplit('/') + .next() + .unwrap() + .to_owned(), + work.last_attempt_at_ms, + ) + }) + .collect::>(), + vec![ + ("never-a.jsonl".into(), None), + ("never-b.jsonl".into(), None), + ("oldest.jsonl".into(), Some(100)), + ("newest.jsonl".into(), Some(200)), + ] + ); + + let source = store + .list_source_import_file_work( + CaptureProvider::Pi, + &format!("/order/source/{family}"), + class, + 4, + ) + .unwrap(); + assert_eq!( + source + .iter() + .map(|work| { + ( + work.file.source_path.rsplit('/').next().unwrap().to_owned(), + work.last_attempt_at_ms, + ) + }) + .collect::>(), + vec![ + ("never-a.jsonl".into(), None), + ("never-b.jsonl".into(), None), + ("oldest.jsonl".into(), Some(100)), + ("newest.jsonl".into(), Some(200)), + ] + ); + } +} + +#[test] +fn source_import_work_is_selectable_by_freshness_class_within_one_source() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let root = "/fixture/mixed"; + let failed = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/mixed/failed.jsonl", + 1000, + ); + let fresh = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/mixed/fresh.jsonl", + 1001, + ); + upsert_source_inventory(&store, &[failed.clone(), fresh.clone()]); + let generation = current_source_generation(&store, CaptureProvider::Pi, root); + store + .record_source_import_file_result( + CaptureProvider::Pi, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &failed.source_path, + file_size_bytes: failed.file_size_bytes, + file_modified_at_ms: failed.file_modified_at_ms, + import_revision: failed.import_revision, + inventory_generation: generation, + metadata: &failed.metadata, + indexed_at_ms: 2000, + }, + CatalogIndexedStatus::Failed, + Some("retry me"), + ) + .unwrap(); + + let fresh_work = store + .list_source_import_file_work(CaptureProvider::Pi, root, ImportWorkClass::Fresh, 10) + .unwrap(); + assert_eq!(fresh_work.len(), 1); + assert_eq!(fresh_work[0].file.source_path, fresh.source_path); + assert_eq!(fresh_work[0].reason, ImportPendingReason::FreshNew); + + let recovery_work = store + .list_source_import_file_work(CaptureProvider::Pi, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(recovery_work.len(), 1); + assert_eq!(recovery_work[0].file.source_path, failed.source_path); + assert_eq!( + recovery_work[0].reason, + ImportPendingReason::RecoveryReplacement + ); + assert_eq!(recovery_work[0].reason.class(), ImportWorkClass::Recovery); + assert!(recovery_work[0].reason.requires_replacement()); + + upsert_source_inventory(&store, &[failed, fresh]); + let reclassified = store + .list_source_import_file_work(CaptureProvider::Pi, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!( + reclassified[0].reason, + ImportPendingReason::RecoveryReplacement + ); +} + +#[test] +fn explicit_source_rescan_survives_same_fingerprint_reobservation() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let root = "/fixture/explicit-rescan"; + let file = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/explicit-rescan/session.jsonl", + 1000, + ); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: 1001, + }, + ) + .unwrap(); + let mut material_source = imported_source(new_id(), root, "explicit-rescan-session"); + material_source.descriptor.provider = file.provider; + material_source.descriptor.raw_source_path = Some(file.source_path.clone()); + material_source.descriptor.source_format = Some(file.source_format.clone()); + store.upsert_capture_source(&material_source).unwrap(); + + store + .schedule_source_import_explicit_rescan( + file.provider, + root, + current_source_generation(&store, file.provider, root), + ) + .unwrap(); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + + let recovery = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(recovery.len(), 1); + assert_eq!(recovery[0].reason, ImportPendingReason::ExplicitRescan); +} + +#[test] +fn explicit_source_rescan_repairs_failures_and_reopens_healthy_mixed_rows() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let root = "/fixture/explicit-mixed-source"; + let files = [ + ("failed.jsonl", "failed"), + ("indexed.jsonl", "indexed"), + ("mixed.jsonl", "completed_with_rejections"), + ("rejected.jsonl", "rejected"), + ] + .map(|(name, _)| { + source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + &format!("{root}/{name}"), + 1000, + ) + }); + upsert_source_inventory(&store, &files); + for ((_, status), file) in [ + ("failed.jsonl", "failed"), + ("indexed.jsonl", "indexed"), + ("mixed.jsonl", "completed_with_rejections"), + ("rejected.jsonl", "rejected"), + ] + .into_iter() + .zip(&files) + { + store + .conn + .execute( + "UPDATE source_import_files SET indexed_status = ?2, pending_reason = NULL \ + WHERE source_path = ?1", + params![&file.source_path, status], + ) + .unwrap(); + } + + assert_eq!( + store + .schedule_source_import_explicit_rescan( + CaptureProvider::Pi, + root, + current_source_generation(&store, CaptureProvider::Pi, root), + ) + .unwrap(), + 2 + ); + let states = store + .conn + .prepare( + "SELECT indexed_status, pending_reason FROM source_import_files \ + WHERE source_root = ?1 ORDER BY source_path", + ) + .unwrap() + .query_map([root], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + states, + vec![ + ("failed".into(), Some("legacy".into())), + ("indexed".into(), Some("explicit_rescan".into())), + ("completed_with_rejections".into(), None), + ("rejected".into(), None), + ] + ); + let work = store + .list_source_import_file_work(CaptureProvider::Pi, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(work.len(), 2); + assert!(work + .iter() + .any(|work| work.reason == ImportPendingReason::Legacy)); + assert!(work + .iter() + .any(|work| work.reason == ImportPendingReason::ExplicitRescan)); +} + +#[test] +fn explicit_catalog_rescan_repairs_failures_and_reopens_healthy_mixed_rows() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let root = "/fixture/explicit-mixed-catalog"; + let sessions = [ + ("failed.jsonl", "failed"), + ("indexed.jsonl", "indexed"), + ("mixed.jsonl", "completed_with_rejections"), + ("rejected.jsonl", "rejected"), + ] + .map(|(name, _)| catalog_session_for_root(root, &format!("{root}/{name}"), name, 1000)); + upsert_catalog_inventory(&store, &sessions); + for ((_, status), session) in [ + ("failed.jsonl", "failed"), + ("indexed.jsonl", "indexed"), + ("mixed.jsonl", "completed_with_rejections"), + ("rejected.jsonl", "rejected"), + ] + .into_iter() + .zip(&sessions) + { + store + .conn + .execute( + "UPDATE catalog_sessions SET indexed_status = ?2, pending_reason = NULL \ + WHERE source_path = ?1", + params![&session.source_path, status], + ) + .unwrap(); + } + let generation = current_catalog_generation(&store, CaptureProvider::Codex, root); + + assert_eq!( + store + .schedule_catalog_source_explicit_rescan(CaptureProvider::Codex, root, generation,) + .unwrap(), + 2 + ); + let states = store + .conn + .prepare( + "SELECT indexed_status, pending_reason FROM catalog_sessions \ + WHERE source_root = ?1 ORDER BY source_path", + ) + .unwrap() + .query_map([root], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + states, + vec![ + ("failed".into(), Some("legacy".into())), + ("indexed".into(), Some("explicit_rescan".into())), + ("completed_with_rejections".into(), None), + ("rejected".into(), None), + ] + ); + let work = store + .list_catalog_import_work(CaptureProvider::Codex, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(work.len(), 2); + assert!(work + .iter() + .any(|work| work.reason == ImportPendingReason::Legacy)); + assert!(work + .iter() + .any(|work| work.reason == ImportPendingReason::ExplicitRescan)); +} + +#[test] +fn source_import_append_failure_retries_incrementally() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let root = "/fixture/append"; + let mut file = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/append/session.jsonl", + 1000, + ); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: 1001, + }, + ) + .unwrap(); + let mut material_source = imported_source(new_id(), root, "append-session"); + material_source.descriptor.provider = file.provider; + material_source.descriptor.raw_source_path = Some(file.source_path.clone()); + material_source.descriptor.source_format = Some(file.source_format.clone()); + store.upsert_capture_source(&material_source).unwrap(); + insert_matching_checkpoint(&store, &file); + + file.file_size_bytes = 64; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let fresh = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Fresh, 10) + .unwrap(); + assert_eq!(fresh[0].reason, ImportPendingReason::FreshAppend); + + file.file_size_bytes = 72; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let repeated_growth = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Fresh, 10) + .unwrap(); + assert_eq!(repeated_growth[0].reason, ImportPendingReason::FreshAppend); + assert!(!repeated_growth[0].reason.requires_replacement()); + + store + .record_source_import_file_result( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: 1002, + }, + CatalogIndexedStatus::Failed, + Some("retry append"), + ) + .unwrap(); + let recovery = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(recovery[0].reason, ImportPendingReason::RecoveryRetry); + assert_eq!(recovery[0].reason.class(), ImportWorkClass::Recovery); + assert!(!recovery[0].reason.requires_replacement()); + + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let reclassified = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(reclassified[0].reason, ImportPendingReason::RecoveryRetry); + + file.file_size_bytes = 80; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let after_growth = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(after_growth[0].reason, ImportPendingReason::RecoveryRetry); + assert!(!after_growth[0].reason.requires_replacement()); +} + +#[test] +fn source_import_pending_growth_ignores_only_owner_change_token() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + let root = "/fixture/append-token"; + let mut file = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/append-token/session.jsonl", + 1000, + ); + file.metadata = serde_json::json!({ + "dependencies": [{"path": "companion.json", "exists": true}], + "change_token_v1": "first", + }); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: 1001, + }, + ) + .unwrap(); + upsert_source_material(&store, &file); + insert_matching_checkpoint(&store, &file); + + file.file_size_bytes += 10; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + file.metadata["change_token_v1"] = serde_json::json!("second"); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let fresh = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Fresh, 10) + .unwrap(); + assert_eq!(fresh[0].reason, ImportPendingReason::FreshAppend); + + file.file_size_bytes += 10; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + file.metadata["dependencies"] = + serde_json::json!([{"path": "companion.json", "exists": false}]); + file.metadata["change_token_v1"] = serde_json::json!("third"); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let changed_dependency = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Fresh, 10) + .unwrap(); + assert_eq!( + changed_dependency[0].reason, + ImportPendingReason::FreshChanged + ); + assert!(changed_dependency[0].reason.requires_replacement()); +} + +#[test] +fn source_growth_requires_exact_material_and_checkpoint_for_fresh_append() { + for missing in ["material", "checkpoint"] { + let temp = tempdir(); + let store = Store::open(temp.path().join(format!("missing-{missing}.sqlite"))).unwrap(); + let root = "/fixture/append-guard"; + let mut file = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/append-guard/session.jsonl", + 1000, + ); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: 1001, + }, + ) + .unwrap(); + if missing != "material" { + upsert_source_material(&store, &file); + } + if missing != "checkpoint" { + insert_matching_checkpoint(&store, &file); + } + + file.file_size_bytes += 10; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + + let fresh = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Fresh, 10) + .unwrap(); + assert_eq!( + fresh[0].reason, + ImportPendingReason::FreshChanged, + "missing {missing}" + ); + assert!(fresh[0].reason.requires_replacement()); + } +} + +#[test] +fn catalog_changed_truncated_and_revision_failures_require_replacement() { + for scenario in ["changed", "truncated", "revision"] { + let temp = tempdir(); + let store = Store::open(temp.path().join(format!("{scenario}.sqlite"))).unwrap(); + let root = "/fixture/catalog-replacement"; + let source_path = format!("{root}/{scenario}.jsonl"); + let mut session = catalog_session_for_root(root, &source_path, scenario, 1000); + upsert_catalog_inventory(&store, std::slice::from_ref(&session)); + store + .mark_catalog_source_indexed( + session.provider, + CatalogSourceIndexUpdate { + source_root: root, + source_path: &source_path, + file_size_bytes: session.file_size_bytes, + file_modified_at_ms: session.file_modified_at_ms, + import_revision: session.import_revision, + inventory_generation: current_catalog_generation( + &store, + session.provider, + root, + ), + file_sha256: None, + event_count: Some(1), + indexed_at_ms: 1001, + }, + ) + .unwrap(); + + let expected_initial_reason = match scenario { + "changed" => { + session.file_modified_at_ms += 1; + ImportPendingReason::FreshChanged + } + "truncated" => { + session.file_size_bytes /= 2; + session.file_modified_at_ms += 1; + ImportPendingReason::FreshChanged + } + "revision" => { + session.import_revision += 1; + ImportPendingReason::ParserRevision + } + _ => unreachable!(), + }; + session.cataloged_at_ms += 1; + upsert_catalog_inventory(&store, std::slice::from_ref(&session)); + let initial_class = expected_initial_reason.class(); + let pending = store + .list_catalog_import_work(session.provider, root, initial_class, 10) + .unwrap(); + assert_eq!(pending[0].reason, expected_initial_reason, "{scenario}"); + + store + .record_catalog_source_import_result( + session.provider, + CatalogSourceIndexUpdate { + source_root: root, + source_path: &source_path, + file_size_bytes: session.file_size_bytes, + file_modified_at_ms: session.file_modified_at_ms, + import_revision: session.import_revision, + inventory_generation: current_catalog_generation( + &store, + session.provider, + root, + ), + file_sha256: None, + event_count: None, + indexed_at_ms: 1002, + }, + CatalogIndexedStatus::Failed, + Some("retry replacement"), + ) + .unwrap(); + let recovery = store + .list_catalog_import_work(session.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!( + recovery[0].reason, + ImportPendingReason::RecoveryReplacement, + "{scenario}" + ); + assert_eq!(recovery[0].reason.class(), ImportWorkClass::Recovery); + assert!(recovery[0].reason.requires_replacement()); + + upsert_catalog_inventory(&store, std::slice::from_ref(&session)); + let reclassified = store + .list_catalog_import_work(session.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!( + reclassified[0].reason, + ImportPendingReason::RecoveryReplacement, + "{scenario}" + ); + + session.file_size_bytes += 10; + session.file_modified_at_ms += 1; + session.cataloged_at_ms += 1; + upsert_catalog_inventory(&store, std::slice::from_ref(&session)); + let after_growth = store + .list_catalog_import_work(session.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!( + after_growth[0].reason, + ImportPendingReason::RecoveryReplacement, + "{scenario} growth" + ); + } +} + +#[test] +fn source_growth_preserves_every_replacement_required_pending_reason() { + for reason in [ + ImportPendingReason::FreshNew, + ImportPendingReason::FreshChanged, + ImportPendingReason::RecoveryReplacement, + ImportPendingReason::ParserRevision, + ImportPendingReason::MissingMaterial, + ImportPendingReason::AbandonedPublication, + ImportPendingReason::Legacy, + ImportPendingReason::ExplicitRescan, + ] { + let temp = tempdir(); + let store = Store::open(temp.path().join(format!("{}.sqlite", reason.as_str()))).unwrap(); + let root = "/fixture/preserve-replacement"; + let mut file = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/preserve-replacement/session.jsonl", + 1000, + ); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .conn + .execute( + "UPDATE source_import_files SET pending_reason = ?1 WHERE provider = ?2 AND source_root = ?3 AND source_path = ?4", + params![reason.as_str(), file.provider.as_str(), root, &file.source_path], + ) + .unwrap(); + + file.file_size_bytes += 10; + file.file_modified_at_ms += 1; + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + + let work = store + .list_source_import_file_work(file.provider, root, reason.class(), 10) + .unwrap(); + assert_eq!(work[0].reason, reason, "{}", reason.as_str()); + assert!(work[0].reason.requires_replacement()); + } +} + +#[test] +fn pending_work_queries_use_partial_reason_indexes() { + let temp = tempdir(); + let store = Store::open(temp.path().join("catalog.sqlite")).unwrap(); + store + .conn + .execute_batch( + r#" + WITH RECURSIVE rows(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM rows WHERE value < 4096 + ) + INSERT INTO catalog_sessions ( + source_path, provider, source_format, source_root, agent_type, + file_size_bytes, file_modified_at_ms, cataloged_at_ms, + indexed_status, pending_reason + ) + SELECT printf('/fixture/catalog-%04d.jsonl', value), 'pi', 'pi_session_jsonl', + '/fixture', 'primary', 1, 1, 1, 'indexed', + CASE + WHEN value > 4092 THEN 'fresh_changed' + WHEN value > 4088 THEN 'recovery_replacement' + END + FROM rows; + + WITH RECURSIVE rows(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM rows WHERE value < 4096 + ) + INSERT INTO source_import_files ( + provider, source_format, source_root, source_path, + file_size_bytes, file_modified_at_ms, observed_at_ms, + indexed_status, pending_reason + ) + SELECT 'pi', 'pi_session_jsonl', '/fixture', + printf('/fixture/source-%04d.jsonl', value), + 1, 1, 1, 'indexed', + CASE + WHEN value > 4092 THEN 'fresh_changed' + WHEN value > 4088 THEN 'recovery_replacement' + END + FROM rows; + + ANALYZE; + "#, + ) + .unwrap(); + for (table, fresh_index, recovery_index) in [ + ( + "catalog_sessions", + "idx_catalog_sessions_pending_fresh_attempt", + "idx_catalog_sessions_pending_recovery_attempt", + ), + ( + "source_import_files", + "idx_source_import_files_pending_fresh_attempt", + "idx_source_import_files_pending_recovery_attempt", + ), + ] { + for (reasons, order, index) in [ + ( + "'fresh_new', 'fresh_changed', 'fresh_append'", + "indexed_at_ms, source_path", + fresh_index, + ), + ( + "'recovery_retry', 'recovery_replacement', 'parser_revision', \ + 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan'", + "indexed_at_ms, source_path", + recovery_index, + ), + ] { + let sql = format!( + "EXPLAIN QUERY PLAN SELECT source_path FROM {table} \ + WHERE provider = ?1 AND source_root = ?2 AND is_stale = 0 \ + AND pending_reason IN ({reasons}) \ + ORDER BY {order} LIMIT 64" + ); + let mut stmt = store.conn.prepare(&sql).unwrap(); + let details = stmt + .query_map(params![CaptureProvider::Pi.as_str(), "/fixture"], |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert!( + details.iter().any(|detail| detail.contains(index)), + "query plan for {table} did not use {index}: {details:?}" + ); + assert!( + details + .iter() + .all(|detail| !detail.contains("USE TEMP B-TREE")), + "query plan for {table} used a temp sort: {details:?}" + ); + } + } +} diff --git a/crates/ctx-history-store/src/catalog/tests/source_imports.rs b/crates/ctx-history-store/src/catalog/tests/source_imports.rs index c4369caa..42765203 100644 --- a/crates/ctx-history-store/src/catalog/tests/source_imports.rs +++ b/crates/ctx-history-store/src/catalog/tests/source_imports.rs @@ -107,6 +107,440 @@ fn catalog_index_checkpoint_event_count_can_be_unknown() { ); } +#[test] +fn catalog_observation_token_refreshes_legacy_row_once() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let observed_at_ms = timestamp_ms(fixed_time()); + let source_root = "/home/user/.codex/sessions"; + let source_path = "/home/user/.codex/sessions/legacy-token.jsonl"; + let legacy = catalog_session(source_path, source_path, observed_at_ms); + upsert_catalog_inventory(&store, std::slice::from_ref(&legacy)); + store + .upsert_session(&imported_session(source_path)) + .unwrap(); + store + .mark_catalog_source_indexed( + CaptureProvider::Codex, + CatalogSourceIndexUpdate { + source_root, + source_path, + file_size_bytes: legacy.file_size_bytes, + file_modified_at_ms: legacy.file_modified_at_ms, + import_revision: legacy.import_revision, + inventory_generation: current_catalog_generation( + &store, + CaptureProvider::Codex, + source_root, + ), + file_sha256: None, + event_count: Some(1), + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + + let mut observed = legacy.clone(); + observed.cataloged_at_ms += 2; + observed.metadata["file_observation_token_v1"] = serde_json::json!("token-a"); + upsert_catalog_inventory(&store, std::slice::from_ref(&observed)); + let pending = store + .list_catalog_import_work( + CaptureProvider::Codex, + source_root, + ImportWorkClass::Fresh, + 10, + ) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].reason, ImportPendingReason::FreshChanged); + + let generation = current_catalog_generation(&store, CaptureProvider::Codex, source_root); + let update = CatalogSourceIndexUpdate { + source_root, + source_path, + file_size_bytes: observed.file_size_bytes, + file_modified_at_ms: observed.file_modified_at_ms, + import_revision: observed.import_revision, + inventory_generation: generation, + file_sha256: None, + event_count: Some(1), + indexed_at_ms: observed_at_ms + 3, + }; + assert_eq!( + store + .mark_catalog_source_indexed(CaptureProvider::Codex, update) + .unwrap(), + 0 + ); + assert_eq!( + store + .list_catalog_import_work( + CaptureProvider::Codex, + source_root, + ImportWorkClass::Fresh, + 10, + ) + .unwrap() + .len(), + 1 + ); + assert_eq!( + store + .record_observed_catalog_source_import_result( + CaptureProvider::Codex, + update, + &observed.metadata, + CatalogIndexedStatus::Indexed, + None, + ) + .unwrap(), + 1 + ); + observed.cataloged_at_ms += 1; + upsert_catalog_inventory(&store, &[observed]); + assert!(store + .list_catalog_import_work( + CaptureProvider::Codex, + source_root, + ImportWorkClass::Fresh, + 10 + ) + .unwrap() + .is_empty()); +} + +#[test] +fn source_import_inventory_pacing_precharges_exact_bounded_write_chunks() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let source_root = "/home/user/.claude/projects"; + let files = (0..65) + .map(|index| { + let source_path = format!("{source_root}/{index:03}.jsonl"); + let mut file = source_import_file( + CaptureProvider::Claude, + "claude_projects_jsonl_tree", + source_root, + &source_path, + 1_000 + index, + ); + file.metadata = serde_json::json!({"inventory_index": index}); + file + }) + .collect::>(); + let current_paths = files + .iter() + .map(|file| file.source_path.clone()) + .collect::>(); + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Claude, source_root) + .unwrap(); + let row_bytes = files + .iter() + .map(|file| { + let metadata_json = serde_json::to_string(&file.metadata).unwrap(); + [ + file.provider.as_str(), + file.source_format.as_str(), + file.source_root.as_str(), + file.source_path.as_str(), + metadata_json.as_str(), + ] + .into_iter() + .fold( + super::SOURCE_IMPORT_PERSIST_ROW_OVERHEAD_BYTES, + |bytes, value| bytes.saturating_add(value.len() as u64), + ) + }) + .collect::>(); + let expected_upsert_charges = vec![ + row_bytes[..64].iter().sum::(), + row_bytes[64..].iter().sum::(), + ]; + let path_bytes = current_paths + .iter() + .map(|path| { + super::SOURCE_IMPORT_PERSIST_ROW_OVERHEAD_BYTES.saturating_add(path.len() as u64) + }) + .collect::>(); + let expected_path_charges = vec![ + path_bytes[..64].iter().sum::(), + path_bytes[64..].iter().sum::(), + ]; + + let mut upsert_charges = Vec::new(); + let mut rows_before_charge = Vec::new(); + assert_eq!( + store + .upsert_source_import_files_with_pacing(generation, &files, |bytes| { + upsert_charges.push(bytes); + rows_before_charge.push( + store + .conn + .query_row("SELECT COUNT(*) FROM source_import_files", [], |row| { + row.get::<_, usize>(0) + }) + .unwrap(), + ); + }) + .unwrap(), + files.len() + ); + assert_eq!(upsert_charges, expected_upsert_charges); + assert_eq!(rows_before_charge, vec![0, 64]); + + let mut path_charges = Vec::new(); + let mut paths_before_charge = Vec::new(); + assert_eq!( + store + .mark_source_import_missing_paths_stale_with_pacing( + CaptureProvider::Claude, + source_root, + ¤t_paths, + 2_000, + generation, + |bytes| { + path_charges.push(bytes); + paths_before_charge.push( + store + .conn + .query_row( + "SELECT COUNT(*) FROM temp_source_import_current_paths", + [], + |row| row.get::<_, usize>(0), + ) + .unwrap(), + ); + }, + ) + .unwrap(), + 0 + ); + assert_eq!(path_charges, expected_path_charges); + assert_eq!(paths_before_charge, vec![0, 64]); + + let mut repeated_upsert_charges = Vec::new(); + assert_eq!( + store + .upsert_source_import_files_with_pacing(generation, &files, |bytes| { + repeated_upsert_charges.push(bytes); + }) + .unwrap(), + 0 + ); + assert_eq!(repeated_upsert_charges, expected_upsert_charges); + + let mut repeated_path_charges = Vec::new(); + assert_eq!( + store + .mark_source_import_missing_paths_stale_with_pacing( + CaptureProvider::Claude, + source_root, + ¤t_paths, + 3_000, + generation, + |bytes| repeated_path_charges.push(bytes), + ) + .unwrap(), + 0 + ); + assert_eq!(repeated_path_charges, expected_path_charges); + + let mut stale_observations = Vec::new(); + assert_eq!( + store + .mark_source_import_missing_paths_stale_with_pacing( + CaptureProvider::Claude, + source_root, + ¤t_paths[..64], + 4_000, + generation, + |bytes| { + let temp_paths = store + .conn + .query_row( + "SELECT COUNT(*) FROM temp_source_import_current_paths", + [], + |row| row.get::<_, usize>(0), + ) + .unwrap(); + let omitted_is_stale = store + .conn + .query_row( + "SELECT is_stale FROM source_import_files WHERE source_path = ?1", + params![¤t_paths[64]], + |row| row.get::<_, bool>(0), + ) + .unwrap(); + stale_observations.push((bytes, temp_paths, omitted_is_stale)); + }, + ) + .unwrap(), + 1 + ); + assert_eq!( + stale_observations, + vec![ + (path_bytes[..64].iter().sum::(), 0, false), + (row_bytes[64], 64, false), + ] + ); + + store + .conn + .execute("UPDATE source_import_files SET is_stale = 0", []) + .unwrap(); + let mut stale_batch_charges = Vec::new(); + let mut stale_rows_before_charge = Vec::new(); + assert_eq!( + store + .mark_source_import_missing_paths_stale_with_pacing( + CaptureProvider::Claude, + source_root, + &[], + 5_000, + generation, + |bytes| { + stale_batch_charges.push(bytes); + stale_rows_before_charge.push( + store + .conn + .query_row( + "SELECT COUNT(*) FROM source_import_files WHERE is_stale != 0", + [], + |row| row.get::<_, usize>(0), + ) + .unwrap(), + ); + }, + ) + .unwrap(), + files.len() + ); + assert_eq!(stale_batch_charges, expected_upsert_charges); + assert_eq!(stale_rows_before_charge, vec![0, 64]); + assert!(store + .conn + .query_row( + "SELECT is_stale FROM source_import_files WHERE source_path = ?1", + params![¤t_paths[64]], + |row| row.get::<_, bool>(0), + ) + .unwrap()); +} + +#[test] +fn direct_batched_stale_marking_rolls_back_all_rows_on_a_late_error() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let source_root = "/home/user/.claude/projects"; + let files = (0..65) + .map(|index| { + let path = format!("{source_root}/{index:03}.jsonl"); + source_import_file( + CaptureProvider::Claude, + "claude_projects_jsonl_tree", + source_root, + &path, + 1_000 + index, + ) + }) + .collect::>(); + let source_generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Claude, source_root) + .unwrap(); + store + .upsert_source_import_files(source_generation, &files) + .unwrap(); + store + .conn + .execute_batch( + r#" + CREATE TRIGGER fail_late_source_stale + BEFORE UPDATE OF is_stale ON source_import_files + WHEN NEW.is_stale = 1 + AND OLD.source_path = '/home/user/.claude/projects/064.jsonl' + BEGIN + SELECT RAISE(ABORT, 'late source stale failure'); + END; + "#, + ) + .unwrap(); + + assert!(store + .mark_source_import_missing_paths_stale( + CaptureProvider::Claude, + source_root, + &[], + 2_000, + source_generation, + ) + .is_err()); + assert!(store.conn.is_autocommit()); + assert_eq!( + store + .conn + .query_row( + "SELECT COUNT(*) FROM source_import_files WHERE is_stale != 0", + [], + |row| row.get::<_, usize>(0), + ) + .unwrap(), + 0 + ); + + let catalog_root = "/home/user/.codex/sessions"; + let sessions = (0..65) + .map(|index| { + let path = format!("{catalog_root}/{index:03}.jsonl"); + catalog_session(&path, &format!("session-{index:03}"), 3_000 + index) + }) + .collect::>(); + let catalog_generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Codex, catalog_root) + .unwrap(); + store + .upsert_catalog_sessions(catalog_generation, &sessions) + .unwrap(); + store + .conn + .execute_batch( + r#" + CREATE TRIGGER fail_late_catalog_stale + BEFORE UPDATE OF is_stale ON catalog_sessions + WHEN NEW.is_stale = 1 + AND OLD.source_path = '/home/user/.codex/sessions/064.jsonl' + BEGIN + SELECT RAISE(ABORT, 'late catalog stale failure'); + END; + "#, + ) + .unwrap(); + + assert!(store + .mark_catalog_source_missing_paths_stale( + CaptureProvider::Codex, + catalog_root, + &[], + 4_000, + catalog_generation, + ) + .is_err()); + assert!(store.conn.is_autocommit()); + assert_eq!( + store + .conn + .query_row( + "SELECT COUNT(*) FROM catalog_sessions WHERE is_stale != 0", + [], + |row| row.get::<_, usize>(0), + ) + .unwrap(), + 0 + ); +} + #[test] fn source_import_manifest_upsert_ignores_observed_at_for_unchanged_files() { let temp = tempdir(); @@ -144,6 +578,11 @@ fn source_import_manifest_upsert_ignores_observed_at_for_unchanged_files() { }, ) .unwrap(); + let mut material_source = imported_source(new_id(), &file.source_root, "claude-session"); + material_source.descriptor.provider = CaptureProvider::Claude; + material_source.descriptor.raw_source_path = Some(file.source_path.clone()); + material_source.descriptor.source_format = Some(file.source_format.clone()); + store.upsert_capture_source(&material_source).unwrap(); let after_indexed: i64 = store .conn .query_row("SELECT total_changes()", [], |row| row.get(0)) @@ -164,6 +603,272 @@ fn source_import_manifest_upsert_ignores_observed_at_for_unchanged_files() { .is_empty()); } +#[test] +fn manifested_inventory_formats_map_to_their_canonical_material_formats() { + for (index, mapping) in PROVIDER_MATERIAL_SOURCE_FORMATS.iter().enumerate() { + let provider = mapping.provider; + let inventory = mapping.inventory_source_format; + let material = mapping.material_source_format; + assert_eq!( + expected_material_source_format(provider, inventory), + material + ); + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let root = format!("/fixture/material-format/{index}"); + let path = format!("{root}/session.jsonl"); + let observed_at_ms = timestamp_ms(fixed_time()); + let mut file = source_import_file(provider, inventory, &root, &path, observed_at_ms); + file.metadata = serde_json::json!({"inventory_unit": "logical_import_unit"}); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + provider, + SourceImportFileIndexUpdate { + source_root: &root, + source_path: &path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, provider, &root), + metadata: &file.metadata, + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + let mut source = imported_source(new_id(), &path, "material-format-session"); + source.descriptor.provider = provider; + source.descriptor.raw_source_path = Some(path.clone()); + source.descriptor.source_format = Some(material.into()); + store.upsert_capture_source(&source).unwrap(); + assert!(store.source_import_material_exists(&file).unwrap()); + + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + assert!( + store + .list_pending_source_import_files(provider, &root) + .unwrap() + .is_empty(), + "SQL material mapping disagreed for {}:{inventory}", + provider.as_str() + ); + } + assert_eq!( + expected_material_source_format(CaptureProvider::Trae, "trae_state_vscdb"), + "trae_state_vscdb" + ); +} + +#[test] +fn file_owned_source_import_material_does_not_match_a_sibling_capture_source() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let observed_at_ms = timestamp_ms(fixed_time()); + let root = "/home/user/.claude/projects"; + let mut file = source_import_file( + CaptureProvider::Claude, + "claude_projects_jsonl_tree", + root, + "/home/user/.claude/projects/owned.jsonl", + observed_at_ms, + ); + file.metadata = serde_json::json!({"inventory_unit": "logical_import_unit"}); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + let mut sibling = imported_source(new_id(), root, "sibling-session"); + sibling.descriptor.provider = file.provider; + sibling.descriptor.raw_source_path = Some(format!("{root}/sibling.jsonl")); + sibling.descriptor.source_format = Some(file.source_format.clone()); + store.upsert_capture_source(&sibling).unwrap(); + + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let recovery = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!(recovery.len(), 1); + assert_eq!(recovery[0].reason, ImportPendingReason::MissingMaterial); +} + +#[test] +fn file_owned_source_import_material_accepts_an_exact_self_rooted_capture_source() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let observed_at_ms = timestamp_ms(fixed_time()); + let root = "/home/user/.mistral-vibe/logs"; + let mut file = source_import_file( + CaptureProvider::MistralVibe, + "mistral_vibe_session_jsonl_tree", + root, + "/home/user/.mistral-vibe/logs/session/messages.jsonl", + observed_at_ms, + ); + file.metadata = serde_json::json!({"inventory_unit": "logical_import_unit"}); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + let mut source = imported_source(new_id(), &file.source_path, "mistral-session"); + source.descriptor.provider = file.provider; + source.descriptor.raw_source_path = Some(file.source_path.clone()); + source.descriptor.source_format = Some("mistral_vibe_session_jsonl".into()); + store.upsert_capture_source(&source).unwrap(); + let material: (String, String, Option, Option) = store + .conn + .query_row( + "SELECT provider, source_format, source_root, raw_source_path FROM capture_sources WHERE id = ?1", + params![source.id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!( + material, + ( + file.provider.as_str().to_owned(), + "mistral_vibe_session_jsonl".to_owned(), + Some(file.source_path.clone()), + Some(file.source_path.clone()), + ) + ); + assert!(store.source_import_material_exists(&file).unwrap()); + + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + assert!(store + .list_pending_source_import_files(file.provider, root) + .unwrap() + .is_empty()); +} + +#[test] +fn source_import_material_requires_expected_format_and_exact_root() { + for scenario in ["wrong_format", "wrong_root"] { + let temp = tempdir(); + let store = Store::open(temp.path().join(format!("{scenario}.sqlite"))).unwrap(); + let observed_at_ms = timestamp_ms(fixed_time()); + let root = "/fixture/pi"; + let mut file = source_import_file( + CaptureProvider::Pi, + "pi_session_jsonl", + root, + "/fixture/pi/session.jsonl", + observed_at_ms, + ); + file.metadata = serde_json::json!({"inventory_unit": "logical_import_unit"}); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + let mut source = imported_source(new_id(), root, "pi-session"); + source.descriptor.provider = file.provider; + source.descriptor.raw_source_path = Some(file.source_path.clone()); + source.descriptor.source_format = Some(file.source_format.clone()); + if scenario == "wrong_format" { + source.descriptor.source_format = Some("pi_session_json".into()); + } else { + source.descriptor.source_root = Some("/fixture/pi-other".into()); + } + store.upsert_capture_source(&source).unwrap(); + + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + let recovery = store + .list_source_import_file_work(file.provider, root, ImportWorkClass::Recovery, 10) + .unwrap(); + assert_eq!( + recovery[0].reason, + ImportPendingReason::MissingMaterial, + "{scenario}" + ); + let counts = store.source_import_file_counts().unwrap(); + assert_eq!(counts.indexed, 0, "{scenario}"); + assert_eq!(counts.pending, 1, "{scenario}"); + } +} + +#[test] +fn source_root_import_material_accepts_a_capture_source_for_the_same_root() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let observed_at_ms = timestamp_ms(fixed_time()); + let root = "/home/user/.hermes"; + let mut file = source_import_file( + CaptureProvider::Hermes, + "hermes_state_sqlite", + root, + "/home/user/.hermes/state.db", + observed_at_ms, + ); + file.metadata = serde_json::json!({"inventory_unit": "source_root"}); + upsert_source_inventory(&store, std::slice::from_ref(&file)); + store + .mark_source_import_file_indexed( + file.provider, + SourceImportFileIndexUpdate { + source_root: root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: current_source_generation(&store, file.provider, root), + metadata: &file.metadata, + indexed_at_ms: observed_at_ms + 1, + }, + ) + .unwrap(); + let mut root_source = imported_source(new_id(), root, "root-session"); + root_source.descriptor.provider = file.provider; + root_source.descriptor.raw_source_path = Some(format!("{root}/sibling.db")); + root_source.descriptor.source_format = Some(file.source_format.clone()); + store.upsert_capture_source(&root_source).unwrap(); + + file.observed_at_ms += 1; + upsert_source_inventory(&store, std::slice::from_ref(&file)); + assert!(store + .list_pending_source_import_files(file.provider, root) + .unwrap() + .is_empty()); +} + #[test] fn source_root_inventory_change_token_marks_same_stat_source_pending() { let temp = tempdir(); @@ -205,6 +910,7 @@ fn source_root_inventory_change_token_marks_same_stat_source_pending() { }, ) .unwrap(); + upsert_source_material(&store, &file); assert!(store .list_pending_source_import_files(CaptureProvider::Hermes, root) .unwrap() @@ -263,6 +969,7 @@ fn logical_import_unit_change_token_marks_same_owner_stat_pending() { }, ) .unwrap(); + upsert_source_material(&store, &file); assert!(store .list_pending_source_import_files(CaptureProvider::OpenCode, root) .unwrap() @@ -371,6 +1078,7 @@ fn source_import_file_counts_track_pending_indexed_failed_and_stale() { }, ) .unwrap(); + upsert_source_material(&store, &files[0]); store .record_source_import_file_result( CaptureProvider::Claude, diff --git a/crates/ctx-history-store/src/connection_tests.rs b/crates/ctx-history-store/src/connection_tests.rs index 5b8a3d35..198d3f2c 100644 --- a/crates/ctx-history-store/src/connection_tests.rs +++ b/crates/ctx-history-store/src/connection_tests.rs @@ -2,7 +2,7 @@ use std::time::Duration; use rusqlite::{params, Connection, OptionalExtension}; -use crate::{Store, StoreError}; +use crate::{EventSearchBulkMaintenanceOutcome, Store, StoreError}; fn tempdir() -> tempfile::TempDir { tempfile::Builder::new() @@ -75,7 +75,7 @@ fn strict_truncating_checkpoint_reports_pinned_reader() { } #[test] -fn bulk_search_mode_recovers_on_reopen_and_restores_saved_config() { +fn bulk_search_mode_waits_for_paced_recovery_and_restores_saved_config() { let temp = tempdir(); let db_path = temp.path().join("work.sqlite"); let store = Store::open(&db_path).unwrap(); @@ -94,6 +94,17 @@ fn bulk_search_mode_recovers_on_reopen_and_restores_saved_config() { drop(guard); let reopened = Store::open(&db_path).unwrap(); + assert_eq!(bulk_mode_marker(&reopened), Some(1)); + for table in ["event_search", "event_search_scriptgram"] { + assert_eq!(fts_config(&reopened, table, "automerge", 4), 0); + assert_eq!(fts_config(&reopened, table, "crisismerge", 16), 1_000_000); + } + + let _pacing = crate::install_event_search_maintenance_pacer(|_| {}); + assert!(reopened + .advance_event_search_bulk_maintenance() + .unwrap() + .is_complete()); assert_eq!(bulk_mode_marker(&reopened), None); for table in ["event_search", "event_search_scriptgram"] { assert_eq!(fts_config(&reopened, table, "automerge", 4), 8); @@ -151,7 +162,10 @@ fn nested_bulk_search_mode_finishes_only_at_outer_scope() { let nested = first.begin_event_search_bulk_mode().unwrap(); let second = Store::open_with_busy_timeout(&db_path, Duration::from_millis(10)).unwrap(); - first.finish_event_search_bulk_mode(&nested).unwrap(); + assert_eq!( + first.finish_event_search_bulk_mode(&nested).unwrap(), + EventSearchBulkMaintenanceOutcome::Complete + ); assert_eq!(bulk_mode_marker(&first), Some(1)); let error = first.finish_event_search_bulk_mode(&outer).unwrap_err(); assert!(matches!(error, StoreError::InvalidBulkSearchGuard)); @@ -304,7 +318,7 @@ fn bulk_search_finish_preserves_preexisting_optimized_segment() { } #[test] -fn bulk_search_recovery_resumes_legacy_in_progress_full_merge() { +fn paced_bulk_search_recovery_resumes_legacy_in_progress_full_merge() { let temp = tempdir(); let db_path = temp.path().join("work.sqlite"); let store = Store::open(&db_path).unwrap(); @@ -330,6 +344,7 @@ fn bulk_search_recovery_resumes_legacy_in_progress_full_merge() { drop(store); drop(guard); + let _pacing = crate::install_event_search_maintenance_pacer(|_| {}); let reopened = Store::open(&db_path).unwrap(); assert_eq!(bulk_mode_marker(&reopened), None); assert_eq!( @@ -390,7 +405,7 @@ fn event_search_segment_count(store: &Store) -> i64 { } #[test] -fn interrupted_bounded_merge_resumes_after_reopen() { +fn interrupted_bounded_merge_resumes_after_paced_reopen() { let temp = tempdir(); let db_path = temp.path().join("work.sqlite"); let store = Store::open_with_busy_timeout(&db_path, Duration::from_millis(10)).unwrap(); @@ -429,6 +444,7 @@ fn interrupted_bounded_merge_resumes_after_reopen() { drop(store); drop(guard); + let _pacing = crate::install_event_search_maintenance_pacer(|_| {}); let reopened = Store::open(&db_path).unwrap(); assert_eq!(bulk_mode_marker(&reopened), None); let segments = reopened diff --git a/crates/ctx-history-store/src/error.rs b/crates/ctx-history-store/src/error.rs index b1c5762f..435d54a1 100644 --- a/crates/ctx-history-store/src/error.rs +++ b/crates/ctx-history-store/src/error.rs @@ -19,6 +19,8 @@ pub enum StoreError { NotFound(Uuid), #[error("unsupported history store schema version: {0}")] UnsupportedSchemaVersion(i64), + #[error("invalid import pending reason: {0}")] + InvalidImportPendingReason(String), #[error("unsafe or ambiguous history store identity")] UnsafeStoreIdentity, #[error("unsupported session history archive version: {0}")] diff --git a/crates/ctx-history-store/src/events.rs b/crates/ctx-history-store/src/events.rs index 63167ef1..573d207d 100644 --- a/crates/ctx-history-store/src/events.rs +++ b/crates/ctx-history-store/src/events.rs @@ -198,7 +198,7 @@ impl Store { } pub fn event_id_by_dedupe_key(&self, dedupe_key: &str) -> Result { - let visible = crate::provider_files::event_material_visible_predicate("events"); + let visible = self.importer_event_material_visible_predicate("events"); self.conn .query_row( &format!("SELECT id FROM events WHERE dedupe_key = ?1 AND {visible}"), @@ -209,7 +209,7 @@ impl Store { } pub fn event_id_by_seq(&self, seq: u64) -> Result { - let visible = crate::provider_files::event_material_visible_predicate("events"); + let visible = self.importer_event_material_visible_predicate("events"); self.conn .query_row( &format!("SELECT id FROM events WHERE seq = ?1 AND {visible}"), @@ -232,7 +232,7 @@ impl Store { pub fn get_event(&self, id: Uuid) -> Result { let tail = format!( "WHERE id = ?1 AND {}", - crate::provider_files::event_material_visible_predicate("events") + self.importer_event_material_visible_predicate("events") ); self.conn .query_row( @@ -257,7 +257,7 @@ impl Store { pub fn events_for_session(&self, session_id: Uuid) -> Result> { let tail = format!( "WHERE session_id = ?1 AND {} ORDER BY seq, occurred_at_ms", - crate::provider_files::event_material_visible_predicate("events") + self.importer_event_material_visible_predicate("events") ); let mut stmt = self.conn.prepare(event_select_sql(&tail).as_str())?; let rows = stmt.query_map(params![session_id.to_string()], event_from_row)?; @@ -267,7 +267,7 @@ impl Store { pub fn events_for_session_limited(&self, session_id: Uuid, limit: usize) -> Result> { let tail = format!( "WHERE session_id = ?1 AND {} ORDER BY seq, occurred_at_ms LIMIT ?2", - crate::provider_files::event_material_visible_predicate("events") + self.importer_event_material_visible_predicate("events") ); let mut stmt = self.conn.prepare(event_select_sql(&tail).as_str())?; let rows = stmt.query_map( @@ -300,7 +300,7 @@ impl Store { } else { let tail = format!( "WHERE session_id = ?1 AND seq < ?2 AND {} ORDER BY seq DESC, occurred_at_ms DESC LIMIT ?3", - crate::provider_files::event_material_visible_predicate("events") + self.importer_event_material_visible_predicate("events") ); let mut stmt = self.conn.prepare(event_select_sql(&tail).as_str())?; let rows = stmt.query_map( @@ -319,7 +319,7 @@ impl Store { if after > 0 { let tail = format!( "WHERE session_id = ?1 AND seq > ?2 AND {} ORDER BY seq, occurred_at_ms LIMIT ?3", - crate::provider_files::event_material_visible_predicate("events") + self.importer_event_material_visible_predicate("events") ); let mut stmt = self.conn.prepare(event_select_sql(&tail).as_str())?; let rows = stmt.query_map( diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index fbdd6133..da91c7da 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -22,22 +22,28 @@ mod sync; mod vcs; pub use archive::validate_archive_version; -pub use bulk_search::EventSearchBulkGuard; +#[doc(hidden)] +pub use bulk_search::{install_event_search_maintenance_pacer, EventSearchMaintenancePacingGuard}; +pub use bulk_search::{EventSearchBulkGuard, EventSearchBulkMaintenanceOutcome}; pub use catalog::{ - CatalogCounts, CatalogIndexedStatus, CatalogSession, CatalogSourceIndexState, - CatalogSourceIndexUpdate, IndexedHistoryCounts, SourceImportFile, SourceImportFileCounts, - SourceImportFileIndexUpdate, + CatalogCounts, CatalogImportWork, CatalogIndexedStatus, CatalogSession, + CatalogSourceIndexState, CatalogSourceIndexUpdate, ImportPendingReason, + ImportPendingReasonRepairProgress, ImportWorkClass, IndexedHistoryCounts, SourceImportFile, + SourceImportFileCounts, SourceImportFileIndexUpdate, SourceImportFileWork, }; pub use error::{Result, StoreError}; pub use files::FileTouchScope; pub use identity::{LocalDeviceIdentity, LocalWorkspaceIdentity}; pub use provider_files::{ ProviderFileCheckpoint, ProviderFileCheckpointKey, ProviderFileFinalizeOutcome, - ProviderFileImportOutcome, ProviderFileInventoryObservation, ProviderFileMaintenanceWarning, - ProviderFilePreparationProgress, ProviderFilePublicationCommit, ProviderFilePublicationKind, - ProviderFilePublicationScope, ProviderFileReconciliationCounts, - ProviderFileReconciliationProgress, PROVIDER_FILE_CHECKPOINT_RESUME_STATE_MAX_BYTES, - PROVIDER_FILE_PREPARATION_MAX_ROWS, PROVIDER_FILE_RECONCILIATION_MAX_ROWS, + ProviderFileImportOutcome, ProviderFileInventoryFamily, ProviderFileInventoryObservation, + ProviderFileMaintenanceWarning, ProviderFilePreparationProgress, ProviderFilePublicationCommit, + ProviderFilePublicationCompletion, ProviderFilePublicationInventoryOwner, + ProviderFilePublicationKind, ProviderFilePublicationPhase, + ProviderFilePublicationRetirementWork, ProviderFilePublicationScope, + ProviderFileReconciliationCounts, ProviderFileReconciliationProgress, + PROVIDER_FILE_CHECKPOINT_RESUME_STATE_MAX_BYTES, PROVIDER_FILE_PREPARATION_MAX_ROWS, + PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES, PROVIDER_FILE_RECONCILIATION_MAX_ROWS, }; pub use raw_sql::{ RawSqlColumn, RawSqlLimits, RawSqlOptions, RawSqlResult, RawSqlTruncation, RawSqlValue, @@ -57,7 +63,7 @@ use std::{ use rusqlite::Connection; -pub(crate) const SCHEMA_VERSION: i64 = 50; +pub(crate) const SCHEMA_VERSION: i64 = 54; pub struct Store { path: PathBuf, diff --git a/crates/ctx-history-store/src/provider_files.rs b/crates/ctx-history-store/src/provider_files.rs index 73b0f53c..b9706fa9 100644 --- a/crates/ctx-history-store/src/provider_files.rs +++ b/crates/ctx-history-store/src/provider_files.rs @@ -1,15 +1,17 @@ use std::cell::Cell; use std::collections::BTreeSet; use std::fs::{self, File}; +use std::ops::ControlFlow; use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; use ctx_history_core::{ - CaptureProvider, CaptureSource, CaptureSourceDescriptor, Event, FileTouched, HistoryRecordLink, - Run, Session, SessionEdge, SessionHistoryArchive, Summary, VcsChange, + utc_now, CaptureProvider, CaptureSource, CaptureSourceDescriptor, Event, FileTouched, + HistoryRecordLink, Run, Session, SessionEdge, SessionHistoryArchive, Summary, VcsChange, }; use fs2::FileExt; use rusqlite::{params, Connection, OptionalExtension}; diff --git a/crates/ctx-history-store/src/provider_files/checkpoints.rs b/crates/ctx-history-store/src/provider_files/checkpoints.rs index 30fc7c60..78bb92c1 100644 --- a/crates/ctx-history-store/src/provider_files/checkpoints.rs +++ b/crates/ctx-history-store/src/provider_files/checkpoints.rs @@ -166,4 +166,88 @@ impl Store { let rows = stmt.query_map(params![session_id.to_string()], event_from_row)?; crate::connection::collect_rows(rows) } + + /// Checks importer-owned record identity without exposing staged material to + /// ordinary readers. + pub fn provider_file_publication_history_record_exists( + &self, + scope: &ProviderFilePublicationScope, + record_id: Uuid, + ) -> Result { + self.validate_provider_file_import_read_scope(scope)?; + self.conn + .query_row( + "SELECT EXISTS (SELECT 1 FROM history_records WHERE id = ?1)", + params![record_id.to_string()], + |row| row.get(0), + ) + .map_err(StoreError::from) + } + + pub fn discard_provider_file_publication_orphan_record( + &self, + scope: &ProviderFilePublicationScope, + record_id: Uuid, + ) -> Result> { + self.ensure_active_provider_file_publication(scope)?; + self.with_provider_file_publication_writes_inner(scope, false, |store| { + store.with_atomic_provider_file_update(|| { + let replacement_id = scope.scope_id.to_string(); + let tracked: bool = store.conn.query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1 AND entity_kind = 'history_record' AND entity_id = ?2)" + ), + params![&replacement_id, record_id.to_string()], + |row| row.get(0), + )?; + if !tracked { + return Ok(None); + } + // Keep the seen row as durable staging evidence until the + // publication marker is finalized. Without it, adoption would + // conservatively reset a mutated marker that has no material. + store.delete_orphan_record_row(record_id).map(Some) + }) + }) + } + + pub fn discard_provider_file_publication_orphan_capture_sources( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result { + self.ensure_active_provider_file_publication(scope)?; + self.with_provider_file_publication_writes_inner(scope, false, |store| { + store.with_atomic_provider_file_update(|| { + store + .conn + .execute( + &format!( + r#" + DELETE FROM capture_sources + WHERE id IN ( + SELECT entity_id FROM {STAGING_SEEN_TABLE} + WHERE replacement_id = ?1 AND entity_kind = ?2 + ) + AND NOT EXISTS (SELECT 1 FROM vcs_workspaces WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM history_records WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM sessions WHERE capture_source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM session_edges WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM runs WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM events WHERE capture_source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM vcs_changes WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM history_record_links WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM summaries WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM files_touched WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM history_record_tags WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM record_edges WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM audit_log WHERE source_id = capture_sources.id) + "# + ), + params![scope.scope_id.to_string(), CURRENT_CAPTURE_SOURCE_KIND], + ) + .map_err(StoreError::from) + }) + }) + } } diff --git a/crates/ctx-history-store/src/provider_files/fencing_entity_writes.rs b/crates/ctx-history-store/src/provider_files/fencing_entity_writes.rs index 61acc4a5..1f2110bc 100644 --- a/crates/ctx-history-store/src/provider_files/fencing_entity_writes.rs +++ b/crates/ctx-history-store/src/provider_files/fencing_entity_writes.rs @@ -190,23 +190,19 @@ impl Store { ) -> Result<()> { let existing = self.stored_session_effective_source_id(session.id)?; let mut existing_sources = vec![existing]; - if let Some((parent, root, transcript, record)) = self + if let Some((transcript, record)) = self .conn .query_row( - "SELECT parent_session_id, root_session_id, transcript_blob_id, history_record_id FROM sessions WHERE id = ?1", + "SELECT transcript_blob_id, history_record_id FROM sessions WHERE id = ?1", params![session.id.to_string()], - four_optional_uuids_from_row, + two_optional_uuids_from_row, ) .optional()? { - existing_sources.extend(self.session_reference_source_ids( - parent, root, transcript, record, - )?); + existing_sources.extend(self.session_owned_reference_source_ids(transcript, record)?); } let mut incoming_sources = vec![session.capture_source_id]; - incoming_sources.extend(self.session_reference_source_ids( - session.parent_session_id, - session.root_session_id, + incoming_sources.extend(self.session_owned_reference_source_ids( session.transcript_blob_id, session.history_record_id, )?); @@ -314,27 +310,7 @@ impl Store { edge.from_session_id, edge.to_session_id, )?; - let mut existing_sources = vec![existing]; - if let Some((from, to)) = self - .conn - .query_row( - "SELECT from_session_id, to_session_id FROM session_edges WHERE id = ?1", - params![edge.id.to_string()], - two_uuids_from_row, - ) - .optional()? - { - existing_sources.extend([ - self.direct_entity_source_id("sessions", from)?, - self.direct_entity_source_id("sessions", to)?, - ]); - } - let incoming_sources = vec![ - incoming, - self.direct_entity_source_id("sessions", edge.from_session_id)?, - self.direct_entity_source_id("sessions", edge.to_session_id)?, - ]; - self.ensure_provider_file_source_ids_write_allowed(&existing_sources, &incoming_sources) + self.ensure_provider_file_source_ids_write_allowed(&[existing], &[incoming]) } pub(crate) fn ensure_provider_file_direct_source_write_allowed( @@ -383,7 +359,7 @@ impl Store { existing_sources.push(self.direct_entity_source_id("sessions", session)?); } if let Some(record) = record { - existing_sources.push(self.direct_entity_source_id("history_records", record)?); + self.push_history_record_reference_source(&mut existing_sources, record)?; } } let mut incoming_sources = vec![summary.source_id]; @@ -391,7 +367,7 @@ impl Store { incoming_sources.push(self.direct_entity_source_id("sessions", session)?); } if let Some(record) = summary.history_record_id { - incoming_sources.push(self.direct_entity_source_id("history_records", record)?); + self.push_history_record_reference_source(&mut incoming_sources, record)?; } self.ensure_provider_file_source_ids_write_allowed(&existing_sources, &incoming_sources) } @@ -418,16 +394,14 @@ impl Store { ) .optional()? { - existing_sources.extend([ - self.direct_entity_source_id("history_records", record)?, - self.link_target_source_id(&target_type, target)?, - ]); + self.push_history_record_reference_source(&mut existing_sources, record)?; + existing_sources.push(self.link_target_source_id(&target_type, target)?); } - let incoming_sources = vec![ + let mut incoming_sources = vec![ link.source_id, - self.direct_entity_source_id("history_records", link.history_record_id)?, self.link_target_source_id(link.target_type.as_str(), link.target_id)?, ]; + self.push_history_record_reference_source(&mut incoming_sources, link.history_record_id)?; self.ensure_provider_file_source_ids_write_allowed(&existing_sources, &incoming_sources) } @@ -463,6 +437,16 @@ impl Store { record_id: Uuid, ) -> Result<()> { if let Some(active) = self.provider_file_publication.borrow().as_ref() { + let exact_capability = active.lifecycle.load(Ordering::Acquire) + && self.provider_file_write_scope.get() == Some(active.scope_id); + let record_exists: bool = self.conn.query_row( + "SELECT EXISTS (SELECT 1 FROM history_records WHERE id = ?1)", + params![record_id.to_string()], + |row| row.get(0), + )?; + if exact_capability && !record_exists { + return Ok(()); + } return Err(active_owner_mismatch(active)); } let marker = self @@ -525,6 +509,57 @@ impl Store { self.ensure_provider_file_marker_write_allowed(marker) } + pub(crate) fn track_provider_file_publication_history_record( + &self, + record_id: Uuid, + ) -> Result<()> { + let active = self.provider_file_publication.borrow(); + let Some(active) = active.as_ref() else { + return Ok(()); + }; + if !active.lifecycle.load(Ordering::Acquire) + || self.provider_file_write_scope.get() != Some(active.scope_id) + || !active.attached + { + return Err(active_owner_mismatch(active)); + } + self.conn.execute( + &format!( + "INSERT OR IGNORE INTO {STAGING_SEEN_TABLE} (replacement_id, entity_kind, entity_id) VALUES (?1, 'history_record', ?2)" + ), + params![active.scope_id.to_string(), record_id.to_string()], + )?; + Ok(()) + } + + pub(crate) fn track_provider_file_publication_capture_source( + &self, + source_id: Uuid, + ) -> Result<()> { + let active = self.provider_file_publication.borrow(); + let Some(active) = active.as_ref() else { + return Ok(()); + }; + if !active.lifecycle.load(Ordering::Acquire) + || self.provider_file_write_scope.get() != Some(active.scope_id) + { + return Err(active_owner_mismatch(active)); + } + let replacement_id = active.scope_id.to_string(); + self.conn.execute( + &format!( + "INSERT OR IGNORE INTO {STAGING_SEEN_TABLE} \ + (replacement_id, entity_kind, entity_id) VALUES (?1, ?2, ?3)" + ), + params![ + &replacement_id, + CURRENT_CAPTURE_SOURCE_KIND, + source_id.to_string() + ], + )?; + Ok(()) + } + pub(crate) fn ensure_provider_file_session_assignment_write_allowed( &self, session_id: Uuid, @@ -662,42 +697,4 @@ impl Store { effective_source_predicate, ) } - - fn direct_entity_source_id(&self, table: &'static str, id: Uuid) -> Result> { - if table == "vcs_changes" { - return self - .conn - .query_row( - r#" - SELECT COALESCE(change.source_id, workspace.source_id) - FROM vcs_changes AS change - LEFT JOIN vcs_workspaces AS workspace ON workspace.id = change.vcs_workspace_id - WHERE change.id = ?1 - "#, - params![id.to_string()], - optional_uuid_from_first_column, - ) - .optional() - .map(Option::flatten) - .map_err(StoreError::from); - } - let source_column = match table { - "sessions" => "capture_source_id", - "artifacts" - | "history_records" - | "summaries" - | "history_record_links" - | "vcs_workspaces" => "source_id", - _ => unreachable!("unsupported referenced provider-owned table"), - }; - self.conn - .query_row( - &format!("SELECT {source_column} FROM {table} WHERE id = ?1"), - params![id.to_string()], - optional_uuid_from_first_column, - ) - .optional() - .map(|value| value.flatten()) - .map_err(StoreError::from) - } } diff --git a/crates/ctx-history-store/src/provider_files/fencing_reference_sources.rs b/crates/ctx-history-store/src/provider_files/fencing_reference_sources.rs index c60944d0..1f7fe1de 100644 --- a/crates/ctx-history-store/src/provider_files/fencing_reference_sources.rs +++ b/crates/ctx-history-store/src/provider_files/fencing_reference_sources.rs @@ -1,21 +1,53 @@ impl Store { - fn session_reference_source_ids( + fn direct_entity_source_id(&self, table: &'static str, id: Uuid) -> Result> { + if table == "vcs_changes" { + return self + .conn + .query_row( + r#" + SELECT COALESCE(change.source_id, workspace.source_id) + FROM vcs_changes AS change + LEFT JOIN vcs_workspaces AS workspace ON workspace.id = change.vcs_workspace_id + WHERE change.id = ?1 + "#, + params![id.to_string()], + optional_uuid_from_first_column, + ) + .optional() + .map(Option::flatten) + .map_err(StoreError::from); + } + let source_column = match table { + "sessions" => "capture_source_id", + "artifacts" + | "history_records" + | "summaries" + | "history_record_links" + | "vcs_workspaces" => "source_id", + _ => unreachable!("unsupported referenced provider-owned table"), + }; + self.conn + .query_row( + &format!("SELECT {source_column} FROM {table} WHERE id = ?1"), + params![id.to_string()], + optional_uuid_from_first_column, + ) + .optional() + .map(|value| value.flatten()) + .map_err(StoreError::from) + } + + fn session_owned_reference_source_ids( &self, - parent: Option, - root: Option, transcript: Option, record: Option, ) -> Result>> { let mut sources = Vec::new(); - for (table, id) in [ - ("sessions", parent), - ("sessions", root), - ("artifacts", transcript), - ("history_records", record), - ] { - if let Some(id) = id { - sources.push(self.direct_entity_source_id(table, id)?); - } + if let Some(transcript) = transcript { + sources.push(self.direct_entity_source_id("artifacts", transcript)?); + } + if let Some(record) = record { + self.push_history_record_reference_source(&mut sources, record)?; } Ok(sources) } @@ -38,7 +70,7 @@ impl Store { sources.push(self.direct_entity_source_id("artifacts", artifact)?); } if let Some(record) = record { - sources.push(self.direct_entity_source_id("history_records", record)?); + self.push_history_record_reference_source(&mut sources, record)?; } Ok(sources) } @@ -58,7 +90,7 @@ impl Store { sources.push(self.direct_entity_source_id("artifacts", artifact)?); } if let Some(record) = record { - sources.push(self.direct_entity_source_id("history_records", record)?); + self.push_history_record_reference_source(&mut sources, record)?; } Ok(sources) } @@ -81,11 +113,32 @@ impl Store { sources.push(self.direct_entity_source_id("vcs_workspaces", workspace)?); } if let Some(record) = record { - sources.push(self.direct_entity_source_id("history_records", record)?); + self.push_history_record_reference_source(&mut sources, record)?; } Ok(sources) } + fn push_history_record_reference_source( + &self, + sources: &mut Vec>, + record_id: Uuid, + ) -> Result<()> { + let source_id = self.direct_entity_source_id("history_records", record_id)?; + if source_id.is_some() { + sources.push(source_id); + return Ok(()); + } + let exists = self.conn.query_row( + "SELECT EXISTS (SELECT 1 FROM history_records WHERE id = ?1)", + params![record_id.to_string()], + |row| row.get::<_, bool>(0), + )?; + if !exists { + sources.push(None); + } + Ok(()) + } + fn link_target_source_id(&self, target_type: &str, target_id: Uuid) -> Result> { match target_type { "session" => self.direct_entity_source_id("sessions", target_id), @@ -213,22 +266,25 @@ impl Store { effective_source_predicate: &'static str, ) -> Result<()> { self.cleanup_abandoned_provider_file_publication()?; - if !self + let replacement_id = self .provider_file_publication .borrow() .as_ref() - .is_some_and(|active| active.attached) - { + .filter(|active| active.attached) + .map(|active| active.scope_id.to_string()); + let Some(replacement_id) = replacement_id else { return Ok(()); - } + }; let sql = format!( r#" - INSERT OR IGNORE INTO {STAGING_SCHEMA}.seen (entity_kind, entity_id) - SELECT ?1, entity.id + INSERT OR IGNORE INTO {STAGING_SEEN_TABLE} + (replacement_id, entity_kind, entity_id) + SELECT scope.replacement_id, ?2, entity.id FROM {table} AS entity JOIN capture_sources AS source ON ({effective_source_predicate}) - JOIN {STAGING_SCHEMA}.scope AS scope - ON scope.provider = source.provider + JOIN provider_file_publications AS scope + ON scope.replacement_id = ?1 + AND scope.provider = source.provider AND scope.material_source_format = source.source_format AND ( (source.raw_source_path = scope.source_path AND ( @@ -238,11 +294,13 @@ impl Store { )) OR (source.raw_source_path IS NULL AND source.source_root = scope.source_path) ) - WHERE entity.id = ?2 + WHERE entity.id = ?3 "# ); - self.conn - .execute(&sql, params![entity_kind, entity_id.to_string()])?; + self.conn.execute( + &sql, + params![replacement_id, entity_kind, entity_id.to_string()], + )?; Ok(()) } diff --git a/crates/ctx-history-store/src/provider_files/lifecycle_begin.rs b/crates/ctx-history-store/src/provider_files/lifecycle_begin.rs index 5c993a1c..acf2df39 100644 --- a/crates/ctx-history-store/src/provider_files/lifecycle_begin.rs +++ b/crates/ctx-history-store/src/provider_files/lifecycle_begin.rs @@ -39,7 +39,6 @@ impl Store { if self.provider_file_publication.borrow().is_some() { return Err(StoreError::InvalidProviderFilePublicationScope); } - let lifecycle = Arc::new(AtomicBool::new(true)); let owner_id = opaque_provider_file_owner_id( provider, @@ -64,15 +63,12 @@ impl Store { else { return Ok(None); }; - if !publication.mutation_started - || self.provider_file_owner_has_current_observation( - provider, - &publication.inventory_source_root, - &publication.source_path, - )? - { + if !publication.mutation_started { return Err(StoreError::InvalidProviderFilePublicationScope); } + if self.provider_file_publication_has_current_observation(publication.scope_id)? { + return Ok(None); + } let mut scope = ProviderFilePublicationScope { scope_id: publication.scope_id, @@ -89,16 +85,18 @@ impl Store { file_modified_at_ms: publication.file_modified_at_ms, import_revision: publication.import_revision, metadata_json: publication.metadata_json, - kind: ProviderFilePublicationKind::Replacement, + kind: publication.publication_kind, owner_id, staging_id: publication.staging_id, - tracks_prior_material: true, + tracks_prior_material: publication.tracks_prior_material, + reuse_staging_state: publication.staging_initialized, retires_observation: true, lifecycle: Arc::clone(&lifecycle), _owner_lock: owner_lock, _owner_lock_path: owner_lock_path.clone(), }; self.publish_provider_file_publication_marker(&mut scope, created_at_ms)?; + self.initialize_provider_file_publication_retirement(&mut scope)?; invalidate_semantic_searchable_item_stats(&self.conn)?; Ok(Some(scope)) })?; @@ -118,22 +116,18 @@ impl Store { retires_observation: true, _owner_lock_path: owner_lock_path, attached: false, - staging_dir_path: None, - staging_path: None, - #[cfg(test)] - staging_file_mode: None, - #[cfg(test)] - staging_dir_mode: None, })); if let Err(error) = self.reclaim_orphaned_provider_staging(&scope) { lifecycle.store(false, Ordering::Release); let _ = self.cleanup_active_provider_file_publication(scope.scope_id); return Err(error); } - if let Err(error) = self.attach_provider_file_publication_staging(&scope) { - lifecycle.store(false, Ordering::Release); - let _ = self.cleanup_active_provider_file_publication(scope.scope_id); - return Err(error); + if scope.kind == ProviderFilePublicationKind::Replacement && scope.tracks_prior_material { + if let Err(error) = self.attach_provider_file_publication_staging(&scope) { + lifecycle.store(false, Ordering::Release); + let _ = self.cleanup_active_provider_file_publication(scope.scope_id); + return Err(error); + } } Ok(Some(scope)) } @@ -191,13 +185,25 @@ impl Store { owner_id, staging_id, tracks_prior_material: false, + reuse_staging_state: false, retires_observation: false, lifecycle: Arc::clone(&lifecycle), _owner_lock: owner_lock, _owner_lock_path: owner_lock_path.clone(), }; self.with_atomic_provider_file_update(|| { - self.ensure_provider_file_observation_is_current(provider, observation)?; + if let Err(error) = + self.ensure_provider_file_observation_is_current(provider, observation) + { + if !self.provider_file_publication_matches_candidate( + provider, + observation, + material_source_format, + observation.source_root(), + )? { + return Err(error); + } + } scope.tracks_prior_material = self.provider_file_owner_has_prior_material( provider, material_source_format, @@ -219,19 +225,16 @@ impl Store { retires_observation: false, _owner_lock_path: owner_lock_path, attached: false, - staging_dir_path: None, - staging_path: None, - #[cfg(test)] - staging_file_mode: None, - #[cfg(test)] - staging_dir_mode: None, })); if let Err(error) = self.reclaim_orphaned_provider_staging(&scope) { lifecycle.store(false, Ordering::Release); let _ = self.cleanup_active_provider_file_publication(scope.scope_id); return Err(error); } - if scope.kind == ProviderFilePublicationKind::Replacement && scope.tracks_prior_material { + // Replacement staging also records material written by a first import. + // That durable seen-set is required if the process dies after mutation + // and the source observation later disappears or is revived. + if scope.kind == ProviderFilePublicationKind::Replacement { if let Err(error) = self.attach_provider_file_publication_staging(&scope) { lifecycle.store(false, Ordering::Release); let _ = self.cleanup_active_provider_file_publication(scope.scope_id); @@ -245,4 +248,75 @@ impl Store { } Ok(scope) } + + pub fn provider_file_publication_phase( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result { + self.ensure_active_provider_file_publication(scope)?; + let marker = self.load_replacement_marker(scope)?; + self.ensure_scope_observation_allows_progress(scope, &marker)?; + Ok(derive_provider_file_publication_phase(scope, &marker)) + } + + pub fn stage_provider_file_publication_completion( + &self, + scope: &ProviderFilePublicationScope, + completion: &ProviderFilePublicationCompletion, + ) -> Result<()> { + self.ensure_active_provider_file_publication(scope)?; + if scope.retires_observation || self.provider_file_write_scope.get().is_some() { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + let payload_json = serialize_provider_file_publication_completion(completion)?; + self.with_atomic_provider_file_update(|| { + let marker = self.load_replacement_marker(scope)?; + self.ensure_scope_observation_allows_progress(scope, &marker)?; + if scope.tracks_prior_material && !marker.preparation_complete { + return Err(StoreError::ProviderFileReconciliationIncomplete); + } + match marker.completion_payload_json.as_deref() { + Some(existing) if existing == payload_json.as_str() => return Ok(()), + Some(_) => return Err(StoreError::InvalidProviderFilePublicationScope), + None => {} + } + let changed = self.conn.execute( + r#" + UPDATE main.provider_file_publications + SET completion_payload_json = ?2 + WHERE replacement_id = ?1 AND completion_payload_json IS NULL + AND (preparation_complete = 1 OR ?3 = 0) + "#, + params![ + scope.scope_id.to_string(), + &payload_json, + scope.tracks_prior_material + ], + )?; + if changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + if self.take_provider_file_fault(ProviderFileFaultPoint::CompletionBeforeCommit) { + return Err(StoreError::ProviderFileStaging); + } + Ok(()) + }) + } + + pub fn load_provider_file_publication_completion( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result> { + self.ensure_active_provider_file_publication(scope)?; + if scope.retires_observation { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + let marker = self.load_replacement_marker(scope)?; + self.ensure_scope_observation_allows_progress(scope, &marker)?; + marker + .completion_payload_json + .as_deref() + .map(parse_provider_file_publication_completion) + .transpose() + } } diff --git a/crates/ctx-history-store/src/provider_files/lifecycle_finalize.rs b/crates/ctx-history-store/src/provider_files/lifecycle_finalize.rs index 29b67c41..c183f3d1 100644 --- a/crates/ctx-history-store/src/provider_files/lifecycle_finalize.rs +++ b/crates/ctx-history-store/src/provider_files/lifecycle_finalize.rs @@ -1,169 +1,117 @@ impl Store { - pub fn retire_provider_file_publication( - &self, - scope: ProviderFilePublicationScope, - ) -> Result { - self.validate_provider_file_publication_scope(&scope)?; - if !scope.retires_observation || scope.kind != ProviderFilePublicationKind::Replacement { - return Err(StoreError::InvalidProviderFilePublicationScope); - } - self.ensure_active_provider_file_publication(&scope)?; - - let durable_result = self.with_atomic_provider_file_update(|| { - let marker = self.load_replacement_marker(&scope)?; - self.validate_replacement_marker(&scope, &marker)?; - self.ensure_scope_observation_allows_progress(&scope, &marker)?; - if !marker.preparation_complete - || marker.cleanup_phase != CLEANUP_PHASE_COMPLETE - || !marker.mutation_started - { - return Err(StoreError::ProviderFileReconciliationIncomplete); - } - self.delete_provider_file_checkpoint_for_scope(&scope)?; - self.retire_stale_provider_file_observation(&scope)?; - let deleted = self.conn.execute( - "DELETE FROM provider_file_publications WHERE replacement_id = ?1", - params![scope.scope_id.to_string()], - )?; - if deleted != 1 { - return Err(StoreError::InvalidProviderFilePublicationScope); - } - invalidate_semantic_searchable_item_stats(&self.conn)?; - self.bump_semantic_replacement_revision()?; - if self.take_provider_file_fault(ProviderFileFaultPoint::FinalizeBeforeCommit) { - return Err(StoreError::ProviderFileStaging); - } - #[cfg(test)] - { - if self - .take_provider_file_fault(ProviderFileFaultPoint::RetirementFinalizeProcessExit) - { - std::process::exit(37); - } - } - Ok(marker.counts) - }); - - scope.lifecycle.store(false, Ordering::Release); - let counts = match durable_result { - Ok(counts) => counts, - Err(error) => { - let _ = self.cleanup_active_provider_file_publication(scope.scope_id); - return Err(error); - } - }; - let maintenance_warning = self - .cleanup_active_provider_file_publication(scope.scope_id) - .err(); - Ok(ProviderFileFinalizeOutcome { - reconciliation: counts, - maintenance_warning, - }) - } - pub fn finalize_provider_file_publication( &self, scope: ProviderFilePublicationScope, outcome: ProviderFileImportOutcome<'_>, commit: ProviderFilePublicationCommit<'_>, ) -> Result { - self.validate_provider_file_publication_scope(&scope)?; - if scope.retires_observation { - return Err(StoreError::InvalidProviderFilePublicationScope); - } - validate_successful_outcome(outcome)?; - validate_scope_matches_outcome(&scope, outcome)?; - let (completion_kind, checkpoint) = match commit { - ProviderFilePublicationCommit::Append(checkpoint) => { - if scope.kind != ProviderFilePublicationKind::Incremental { - return Err(StoreError::InvalidProviderFilePublicationScope); - } - (ProviderFileCompletionKind::AppendDelta, Some(checkpoint)) + let durable_result = (|| { + self.validate_provider_file_publication_scope(&scope)?; + if scope.retires_observation { + return Err(StoreError::InvalidProviderFilePublicationScope); } - ProviderFilePublicationCommit::RetainCheckpoint => { - if scope.kind != ProviderFilePublicationKind::Incremental { - return Err(StoreError::InvalidProviderFilePublicationScope); + validate_scope_matches_outcome(&scope, outcome)?; + let (completion_kind, checkpoint) = match commit { + ProviderFilePublicationCommit::Append(checkpoint) => { + if scope.kind != ProviderFilePublicationKind::Incremental { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + (ProviderFileCompletionKind::AppendDelta, Some(checkpoint)) } - (ProviderFileCompletionKind::RetainCheckpoint, None) - } - ProviderFilePublicationCommit::Replacement(checkpoint) => { - if scope.kind != ProviderFilePublicationKind::Replacement { - return Err(StoreError::InvalidProviderFilePublicationScope); + ProviderFilePublicationCommit::RetainCheckpoint => { + if scope.kind != ProviderFilePublicationKind::Incremental { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + (ProviderFileCompletionKind::RetainCheckpoint, None) } - (ProviderFileCompletionKind::Replacement, checkpoint) + ProviderFilePublicationCommit::Replacement(checkpoint) => { + if scope.kind != ProviderFilePublicationKind::Replacement { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + (ProviderFileCompletionKind::Replacement, checkpoint) + } + }; + validate_provider_file_completion_outcome( + outcome, + completion_kind, + checkpoint.is_some(), + scope.tracks_prior_material, + )?; + if let Some(checkpoint) = checkpoint { + validate_checkpoint_for_outcome(outcome, checkpoint)?; } - }; - if let Some(checkpoint) = checkpoint { - validate_checkpoint_for_outcome(outcome, checkpoint)?; - } - self.ensure_active_provider_file_publication(&scope)?; + self.ensure_active_provider_file_publication(&scope)?; - let durable_result = self.with_provider_file_publication_writes(&scope, |_| { - self.with_atomic_provider_file_update(|| { - self.ensure_provider_file_observation_is_current( - outcome.provider, - outcome.observation, - )?; - let marker = self.load_replacement_marker(&scope)?; - if marker.publication_kind != scope.kind { - return Err(StoreError::InvalidProviderFilePublicationScope); - } - let counts = if completion_kind == ProviderFileCompletionKind::Replacement { - self.validate_replacement_marker(&scope, &marker)?; - if !marker.preparation_complete - || (scope.tracks_prior_material - && marker.cleanup_phase != CLEANUP_PHASE_COMPLETE) - { - return Err(StoreError::ProviderFileReconciliationIncomplete); + self.with_provider_file_publication_writes_inner(&scope, true, |_| { + self.with_atomic_provider_file_update(|| { + self.ensure_scope_observation_is_current(&scope)?; + let marker = self.load_replacement_marker(&scope)?; + if marker.publication_kind != scope.kind { + return Err(StoreError::InvalidProviderFilePublicationScope); } - marker.counts - } else { - ProviderFileReconciliationCounts::default() - }; - self.record_matching_provider_file_outcome( - outcome, - completion_kind, - checkpoint.is_some(), - )?; - match completion_kind { - ProviderFileCompletionKind::Replacement => { - self.replace_provider_file_checkpoint(outcome, checkpoint)?; + let counts = if completion_kind == ProviderFileCompletionKind::Replacement { + self.validate_replacement_marker(&scope, &marker)?; + if !marker.preparation_complete + || (scope.tracks_prior_material + && marker.cleanup_phase != CLEANUP_PHASE_COMPLETE) + { + return Err(StoreError::ProviderFileReconciliationIncomplete); + } + marker.counts + } else { + ProviderFileReconciliationCounts::default() + }; + if outcome.status == CatalogIndexedStatus::Rejected { + self.ensure_rejected_publication_has_no_material(&scope)?; } - ProviderFileCompletionKind::AppendDelta => { - self.advance_provider_file_checkpoint( - checkpoint.ok_or(StoreError::InvalidProviderFilePublicationScope)?, - )?; + self.record_matching_provider_file_outcome( + outcome, + completion_kind, + checkpoint.is_some(), + )?; + match completion_kind { + ProviderFileCompletionKind::Replacement => { + self.replace_provider_file_checkpoint(outcome, checkpoint)?; + } + ProviderFileCompletionKind::AppendDelta => { + self.advance_provider_file_checkpoint( + checkpoint + .ok_or(StoreError::InvalidProviderFilePublicationScope)?, + )?; + } + ProviderFileCompletionKind::RetainCheckpoint => {} } - ProviderFileCompletionKind::RetainCheckpoint => {} - } - let deleted = self.conn.execute( - "DELETE FROM provider_file_publications WHERE replacement_id = ?1", - params![scope.scope_id.to_string()], - )?; - if deleted != 1 { - return Err(StoreError::InvalidProviderFilePublicationScope); - } - invalidate_semantic_searchable_item_stats(&self.conn)?; - if completion_kind == ProviderFileCompletionKind::Replacement - && scope.tracks_prior_material - { - self.bump_semantic_replacement_revision()?; - } - if self.take_provider_file_fault(ProviderFileFaultPoint::FinalizeBeforeCommit) { - return Err(StoreError::ProviderFileStaging); - } - Ok(counts) + let deleted = self.conn.execute( + "DELETE FROM provider_file_publications WHERE replacement_id = ?1", + params![scope.scope_id.to_string()], + )?; + if deleted != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + invalidate_semantic_searchable_item_stats(&self.conn)?; + if completion_kind == ProviderFileCompletionKind::Replacement + && scope.tracks_prior_material + { + self.bump_semantic_replacement_revision()?; + } + if self.take_provider_file_fault(ProviderFileFaultPoint::FinalizeBeforeCommit) { + return Err(StoreError::ProviderFileStaging); + } + Ok(counts) + }) }) - }); + })(); - scope.lifecycle.store(false, Ordering::Release); let counts = match durable_result { Ok(counts) => counts, Err(error) => { + let _ = self.abort_provider_file_publication_durable(&scope); + scope.lifecycle.store(false, Ordering::Release); let _ = self.cleanup_active_provider_file_publication(scope.scope_id); return Err(error); } }; + scope.lifecycle.store(false, Ordering::Release); let maintenance_warning = self .cleanup_active_provider_file_publication(scope.scope_id) .err(); @@ -186,4 +134,285 @@ impl Store { pub fn has_pending_provider_file_publications(&self) -> Result { has_fenced_provider_file_publications(&self.conn) } + + pub fn effective_provider_file_publication_inventory_owner( + &self, + ) -> Result> { + let global_id = global_provider_file_publication_id_sql(); + self.conn + .query_row( + &format!( + r#" + SELECT publication.provider, publication.inventory_family, + publication.inventory_source_format, + publication.inventory_source_root, publication.source_path, + publication.inventory_generation, publication.file_size_bytes, + publication.file_modified_at_ms, publication.import_revision, + publication.metadata_json + FROM provider_file_publications AS publication + WHERE publication.replacement_id = ({global_id}) + "# + ), + [], + |row| { + let provider = CaptureProvider::from_str(&row.get::<_, String>(0)?) + .map_err(|_| rusqlite::Error::InvalidQuery)?; + let inventory_family = match row.get::<_, String>(1)?.as_str() { + CATALOG_INVENTORY_FAMILY => ProviderFileInventoryFamily::Catalog, + SOURCE_IMPORT_INVENTORY_FAMILY => ProviderFileInventoryFamily::SourceImport, + _ => return Err(rusqlite::Error::InvalidQuery), + }; + Ok(ProviderFilePublicationInventoryOwner { + provider, + inventory_family, + source_format: row.get(2)?, + source_root: row.get(3)?, + source_path: row.get(4)?, + inventory_generation: nonnegative_i64_to_u64(row.get(5)?)?, + file_size_bytes: nonnegative_i64_to_u64(row.get(6)?)?, + file_modified_at_ms: row.get(7)?, + import_revision: nonnegative_i64_to_u32(row.get(8)?)?, + metadata_json: row.get(9)?, + }) + }, + ) + .optional() + .map_err(StoreError::from) + } + + pub fn effective_provider_file_publication_has_staged_completion(&self) -> Result { + let global_id = global_provider_file_publication_id_sql(); + self.conn + .query_row( + &format!( + "SELECT COALESCE((SELECT completion_payload_json IS NOT NULL \ + FROM provider_file_publications WHERE replacement_id = ({global_id})), 0)" + ), + [], + |row| row.get(0), + ) + .map_err(StoreError::from) + } + + pub fn invalidate_effective_provider_file_publication_observation( + &self, + owner: &ProviderFilePublicationInventoryOwner, + invalidated_at_ms: i64, + ) -> Result { + self.with_atomic_provider_file_update(|| { + if self + .effective_provider_file_publication_inventory_owner()? + .as_ref() + != Some(owner) + { + return Ok(false); + } + let global_id = global_provider_file_publication_id_sql(); + let (publication_id, mutation_started) = self.conn.query_row( + &format!( + "SELECT replacement_id, mutation_started FROM provider_file_publications \ + WHERE replacement_id = ({global_id})" + ), + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?)), + )?; + let changed = match owner.inventory_family { + ProviderFileInventoryFamily::Catalog => self.conn.execute( + r#" + UPDATE catalog_sessions + SET is_stale = 1, cataloged_at_ms = ?9 + WHERE provider = ?1 AND source_format = ?2 + AND source_root = ?3 AND source_path = ?4 + AND file_size_bytes = ?5 AND file_modified_at_ms = ?6 + AND import_revision = ?7 AND is_stale = 0 + "#, + params![ + owner.provider.as_str(), + &owner.source_format, + &owner.source_root, + &owner.source_path, + capped_i64(owner.file_size_bytes), + owner.file_modified_at_ms, + i64::from(owner.import_revision), + &owner.metadata_json, + invalidated_at_ms, + ], + )?, + ProviderFileInventoryFamily::SourceImport => self.conn.execute( + r#" + UPDATE source_import_files + SET is_stale = 1, observed_at_ms = ?9 + WHERE provider = ?1 AND source_format = ?2 + AND source_root = ?3 AND source_path = ?4 + AND file_size_bytes = ?5 AND file_modified_at_ms = ?6 + AND import_revision = ?7 AND metadata_json IS ?8 + AND is_stale = 0 + "#, + params![ + owner.provider.as_str(), + &owner.source_format, + &owner.source_root, + &owner.source_path, + capped_i64(owner.file_size_bytes), + owner.file_modified_at_ms, + i64::from(owner.import_revision), + &owner.metadata_json, + invalidated_at_ms, + ], + )?, + }; + let publication_changed = if mutation_started { + self.conn.execute( + "UPDATE provider_file_publications \ + SET inventory_observation_invalidated = 1, \ + updated_at_ms = MAX(updated_at_ms + 1, ?1) \ + WHERE replacement_id = ?2 AND mutation_started != 0", + params![invalidated_at_ms, publication_id], + )? + } else if changed == 1 { + self.conn.execute( + "DELETE FROM provider_file_publications \ + WHERE replacement_id = ?1 AND mutation_started = 0", + params![publication_id], + )? + } else { + return Ok(false); + }; + if publication_changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + Ok(true) + }) + } + + pub fn provider_file_publication_matches_candidate( + &self, + provider: CaptureProvider, + observation: ProviderFileInventoryObservation<'_>, + material_source_format: &str, + material_source_root: &str, + ) -> Result { + let owner_id = opaque_provider_file_owner_id( + provider, + material_source_format, + material_source_root, + observation.source_path(), + ); + let global_id = global_provider_file_publication_id_sql(); + self.conn + .query_row( + &format!( + r#" + SELECT EXISTS ( + SELECT 1 FROM provider_file_publications AS publication + WHERE publication.replacement_id = ({global_id}) + AND publication.owner_id = ?1 + AND publication.provider = ?2 + AND publication.inventory_family = ?3 + AND publication.inventory_source_format = ?4 + AND publication.inventory_source_root = ?5 + AND publication.source_path = ?6 + AND publication.material_source_format = ?7 + AND publication.material_source_root = ?8 + AND publication.file_size_bytes = ?9 + AND publication.file_modified_at_ms = ?10 + AND publication.import_revision = ?11 + AND publication.metadata_json IS ?12 + AND ({}) + ) + "#, + provider_file_retirement_observation_current_predicate("publication") + ), + params![ + owner_id, + provider.as_str(), + observation.inventory_family(), + observation.source_format(), + observation.source_root(), + observation.source_path(), + material_source_format, + material_source_root, + capped_i64(observation.file_size_bytes()), + observation.file_modified_at_ms(), + i64::from(observation.import_revision()), + observation.metadata_json()?, + ], + |row| row.get(0), + ) + .map_err(StoreError::from) + } + + pub fn provider_file_publication_retirement_work_count(&self) -> Result { + let current_observation = + provider_file_retirement_observation_current_predicate("publication"); + self.conn + .query_row( + &format!( + r#" + SELECT COUNT(*) + FROM provider_file_publications AS publication + WHERE publication.mutation_started != 0 + AND NOT ({current_observation}) + "# + ), + [], + |row| nonnegative_i64_to_usize(row.get(0)?), + ) + .map_err(StoreError::from) + } + + pub fn list_provider_file_publication_retirement_work( + &self, + limit: usize, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let current_observation = + provider_file_retirement_observation_current_predicate("publication"); + let mut stmt = self.conn.prepare(&format!( + r#" + SELECT publication.provider, publication.material_source_format, + publication.material_source_root, publication.source_path, + publication.file_size_bytes, publication.updated_at_ms + FROM provider_file_publications AS publication + WHERE publication.mutation_started != 0 + AND NOT ({current_observation}) + ORDER BY publication.updated_at_ms, publication.replacement_id + LIMIT ?1 + "# + ))?; + let rows = stmt.query_map(params![capped_i64(limit as u64)], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + nonnegative_i64_to_u64(row.get(4)?)?, + row.get::<_, i64>(5)?, + )) + })?; + let mut work = Vec::new(); + for row in rows { + let ( + provider, + material_source_format, + material_source_root, + source_path, + bytes, + last_attempt_at_ms, + ) = row?; + let provider = CaptureProvider::from_str(&provider) + .map_err(|_| StoreError::InvalidProviderFilePublicationScope)?; + work.push(ProviderFilePublicationRetirementWork { + provider, + material_source_format, + material_source_root, + source_path, + estimated_bytes: bytes, + last_attempt_at_ms, + }); + } + Ok(work) + } } diff --git a/crates/ctx-history-store/src/provider_files/lifecycle_progress.rs b/crates/ctx-history-store/src/provider_files/lifecycle_progress.rs index a4d45252..dab6ce0f 100644 --- a/crates/ctx-history-store/src/provider_files/lifecycle_progress.rs +++ b/crates/ctx-history-store/src/provider_files/lifecycle_progress.rs @@ -14,15 +14,22 @@ impl Store { if scope.kind != ProviderFilePublicationKind::Replacement { return Err(StoreError::InvalidProviderFilePublicationScope); } + self.ensure_active_provider_file_publication(scope)?; if !scope.tracks_prior_material { - self.ensure_active_provider_file_publication(scope)?; + let marker = self.load_replacement_marker(scope)?; + self.validate_replacement_marker(scope, &marker)?; + self.ensure_scope_observation_allows_progress(scope, &marker)?; + if (scope.tracks_prior_material && !marker.preparation_complete) + || (!scope.retires_observation && marker.completion_payload_json.is_none()) + { + return Err(StoreError::ProviderFileReconciliationIncomplete); + } return Ok(ProviderFileReconciliationProgress { rows_scanned: 0, complete: true, counts: ProviderFileReconciliationCounts::default(), }); } - self.ensure_active_provider_file_publication(scope)?; if !self .provider_file_publication .borrow() @@ -40,6 +47,9 @@ impl Store { if !marker.preparation_complete { return Err(StoreError::ProviderFileReconciliationIncomplete); } + if !scope.retires_observation && marker.completion_payload_json.is_none() { + return Err(StoreError::ProviderFileReconciliationIncomplete); + } marker.mutation_started = true; let mut rows_scanned = 0usize; while rows_scanned < max_rows && marker.cleanup_phase < CLEANUP_PHASE_COMPLETE { @@ -99,12 +109,14 @@ impl Store { if scope.kind == ProviderFilePublicationKind::Incremental { return Ok(ProviderFilePreparationProgress { source_ids_staged: 0, + rows_processed: 0, complete: true, }); } if !scope.tracks_prior_material { return Ok(ProviderFilePreparationProgress { source_ids_staged: 0, + rows_processed: 0, complete: true, }); } @@ -125,10 +137,34 @@ impl Store { if marker.preparation_complete { return Ok(ProviderFilePreparationProgress { source_ids_staged: 0, + rows_processed: 0, complete: true, }); } - let sqlite_limit = i64::try_from(max_rows + 1).map_err(|_| { + let reset_rows = if marker + .preparation_cursor + .as_deref() + .is_some_and(is_retirement_reset_cursor) + { + self.reset_provider_file_publication_staging_slice(scope, &mut marker, max_rows)? + } else { + 0 + }; + if marker + .preparation_cursor + .as_deref() + .is_some_and(is_retirement_reset_cursor) + || reset_rows == max_rows + { + self.update_replacement_marker(scope, &marker)?; + return Ok(ProviderFilePreparationProgress { + source_ids_staged: 0, + rows_processed: reset_rows, + complete: false, + }); + } + let remaining_rows = max_rows.saturating_sub(reset_rows); + let sqlite_limit = i64::try_from(remaining_rows + 1).map_err(|_| { StoreError::ProviderFileReconciliationLimitOutOfRange { value: max_rows, max: PROVIDER_FILE_PREPARATION_MAX_ROWS, @@ -158,24 +194,42 @@ impl Store { |row| row.get::<_, String>(0), )?; let mut ids = rows.collect::>>()?; - let complete = ids.len() <= max_rows; + let complete = ids.len() <= remaining_rows; if !complete { ids.pop(); } let mut insert = self.conn.prepare_cached(&format!( - "INSERT OR IGNORE INTO {STAGING_SCHEMA}.prior_sources (id) VALUES (?1)" + "INSERT OR IGNORE INTO {STAGING_PRIOR_SOURCES_TABLE} \ + (replacement_id, source_id) VALUES (?1, ?2)" ))?; + let replacement_id = scope.scope_id.to_string(); for id in &ids { - insert.execute(params![id])?; + insert.execute(params![&replacement_id, id])?; } marker.preparation_cursor = ids.last().cloned(); marker.preparation_complete = complete; + if complete && scope.retires_observation { + let staged_state_exists: bool = self.conn.query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1) \ + OR EXISTS (SELECT 1 FROM {STAGING_PRIOR_SOURCES_TABLE} WHERE replacement_id = ?1)" + ), + params![&replacement_id], + |row| row.get(0), + )?; + if !staged_state_exists { + marker.cleanup_phase = CLEANUP_PHASE_COMPLETE; + marker.source_cursor = None; + marker.entity_cursor = None; + } + } self.update_replacement_marker(scope, &marker)?; if self.take_provider_file_fault(ProviderFileFaultPoint::PreparationBeforeCommit) { return Err(StoreError::ProviderFileStaging); } Ok(ProviderFilePreparationProgress { source_ids_staged: ids.len(), + rows_processed: reset_rows.saturating_add(ids.len()), complete, }) })(); @@ -197,10 +251,139 @@ impl Store { pub fn abandon_provider_file_publication( &self, scope: ProviderFilePublicationScope, - ) -> Result<()> { + ) -> Result> { self.validate_provider_file_publication_scope(&scope)?; + let durable_result = if scope.retires_observation { + self.advance_provider_file_publication_attempt(&scope, utc_now().timestamp_millis()) + } else { + Ok(()) + }; + scope.lifecycle.store(false, Ordering::Release); + let maintenance_warning = self + .cleanup_active_provider_file_publication(scope.scope_id) + .err(); + durable_result?; + Ok(maintenance_warning) + } + + /// Releases a failed publication and removes its durable marker only when + /// importer writes have not started. `Continue` means the publication was + /// cancelled; `Break` means mutation started and the durable marker remains + /// fenced for replacement or retirement recovery. Either outcome carries + /// a non-fatal staging cleanup warning when local maintenance was deferred. + pub fn abort_provider_file_publication( + &self, + scope: ProviderFilePublicationScope, + ) -> Result< + ControlFlow, Option>, + > { + let durable_result = self.abort_provider_file_publication_durable(&scope); + scope.lifecycle.store(false, Ordering::Release); - self.cleanup_active_provider_file_publication(scope.scope_id) - .map_err(maintenance_warning_as_error) + let maintenance_warning = self + .cleanup_active_provider_file_publication(scope.scope_id) + .err(); + let cancelled = durable_result?; + Ok(if cancelled { + ControlFlow::Continue(maintenance_warning) + } else { + ControlFlow::Break(maintenance_warning) + }) + } + + fn abort_provider_file_publication_durable( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result { + self.validate_provider_file_publication_scope(scope)?; + if scope.retires_observation { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + self.ensure_active_provider_file_publication(scope)?; + + self.with_atomic_provider_file_update(|| { + let marker = self.load_replacement_marker(scope)?; + if marker.publication_kind != scope.kind { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + if marker.mutation_started { + return Ok(false); + } + let deleted = self.conn.execute( + "DELETE FROM provider_file_publications WHERE replacement_id = ?1 AND mutation_started = 0", + params![scope.scope_id.to_string()], + )?; + if deleted != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + invalidate_semantic_searchable_item_stats(&self.conn)?; + Ok(true) + }) + } + + pub fn retire_provider_file_publication( + &self, + scope: ProviderFilePublicationScope, + ) -> Result { + self.validate_provider_file_publication_scope(&scope)?; + if !scope.retires_observation { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + self.ensure_active_provider_file_publication(&scope)?; + + let durable_result = self.with_atomic_provider_file_update(|| { + let marker = self.load_replacement_marker(&scope)?; + if marker.publication_kind != scope.kind { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + self.ensure_scope_observation_allows_progress(&scope, &marker)?; + if (scope.kind == ProviderFilePublicationKind::Replacement + && (!marker.preparation_complete + || (scope.tracks_prior_material + && marker.cleanup_phase != CLEANUP_PHASE_COMPLETE))) + || !marker.mutation_started + { + return Err(StoreError::ProviderFileReconciliationIncomplete); + } + self.delete_provider_file_checkpoint_for_scope(&scope)?; + self.retire_stale_provider_file_observation(&scope)?; + let deleted = self.conn.execute( + "DELETE FROM provider_file_publications WHERE replacement_id = ?1", + params![scope.scope_id.to_string()], + )?; + if deleted != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + invalidate_semantic_searchable_item_stats(&self.conn)?; + self.bump_semantic_replacement_revision()?; + if self.take_provider_file_fault(ProviderFileFaultPoint::FinalizeBeforeCommit) { + return Err(StoreError::ProviderFileStaging); + } + #[cfg(test)] + { + if self + .take_provider_file_fault(ProviderFileFaultPoint::RetirementFinalizeProcessExit) + { + std::process::exit(37); + } + } + Ok(marker.counts) + }); + + scope.lifecycle.store(false, Ordering::Release); + let counts = match durable_result { + Ok(counts) => counts, + Err(error) => { + let _ = self.cleanup_active_provider_file_publication(scope.scope_id); + return Err(error); + } + }; + let maintenance_warning = self + .cleanup_active_provider_file_publication(scope.scope_id) + .err(); + Ok(ProviderFileFinalizeOutcome { + reconciliation: counts, + maintenance_warning, + }) } } diff --git a/crates/ctx-history-store/src/provider_files/locks.rs b/crates/ctx-history-store/src/provider_files/locks.rs index ad86b61f..174d5744 100644 --- a/crates/ctx-history-store/src/provider_files/locks.rs +++ b/crates/ctx-history-store/src/provider_files/locks.rs @@ -275,41 +275,6 @@ fn create_private_staging_dir(_path: &Path) -> std::io::Result<()> { )) } -#[cfg(unix)] -fn create_private_staging_file(path: &Path) -> std::io::Result { - use std::os::unix::fs::OpenOptionsExt; - - fs::OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .mode(0o600) - .open(path) -} - -#[cfg(unix)] -fn open_existing_private_staging_file(path: &Path) -> std::io::Result { - use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; - - let metadata = fs::symlink_metadata(path)?; - if !metadata.is_file() - || metadata.file_type().is_symlink() - || metadata.permissions().mode() & 0o077 != 0 - || metadata.uid() != unsafe { libc::geteuid() } - || metadata.nlink() != 1 - { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "unsafe provider staging file", - )); - } - fs::OpenOptions::new() - .read(true) - .write(true) - .custom_flags(libc::O_NOFOLLOW) - .open(path) -} - #[cfg(unix)] fn validate_existing_private_staging_file_for_removal(path: &Path) -> std::io::Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -340,29 +305,6 @@ fn validate_existing_private_staging_file_for_removal(_path: &Path) -> std::io:: Ok(()) } -#[cfg(windows)] -fn open_existing_private_staging_file(path: &Path) -> std::io::Result { - use std::os::windows::fs::MetadataExt; - use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT; - - let metadata = fs::symlink_metadata(path)?; - if !metadata.is_file() || metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "unsafe provider staging file", - )); - } - open_private_owner_lock_file(path) -} - -#[cfg(not(any(unix, windows)))] -fn open_existing_private_staging_file(_path: &Path) -> std::io::Result { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "private provider staging is unsupported on this platform", - )) -} - #[cfg(windows)] fn create_private_staging_file(path: &Path) -> std::io::Result { use std::{mem, os::windows::ffi::OsStrExt, os::windows::io::FromRawHandle, ptr}; @@ -409,14 +351,6 @@ fn create_private_staging_file(path: &Path) -> std::io::Result { Ok(file) } -#[cfg(not(any(unix, windows)))] -fn create_private_staging_file(_path: &Path) -> std::io::Result { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "private replacement staging is unsupported on this platform", - )) -} - #[cfg(windows)] fn validate_existing_private_windows_path(path: &Path, directory: bool) -> std::io::Result<()> { use std::os::windows::{ffi::OsStrExt, fs::MetadataExt}; @@ -549,25 +483,3 @@ fn private_windows_security_descriptor( } Ok(descriptor) } - -#[cfg(all(test, unix))] -fn staging_directory_mode(path: &Path) -> Result> { - use std::os::unix::fs::PermissionsExt; - Ok(Some(fs::metadata(path)?.permissions().mode() & 0o777)) -} - -#[cfg(all(test, not(unix)))] -fn staging_directory_mode(_path: &Path) -> Result> { - Ok(None) -} - -#[cfg(all(test, unix))] -fn staging_file_mode(path: &Path) -> Result> { - use std::os::unix::fs::PermissionsExt; - Ok(Some(fs::metadata(path)?.permissions().mode() & 0o777)) -} - -#[cfg(all(test, not(unix)))] -fn staging_file_mode(_path: &Path) -> Result> { - Ok(None) -} diff --git a/crates/ctx-history-store/src/provider_files/observation_validation.rs b/crates/ctx-history-store/src/provider_files/observation_validation.rs index f1097741..17758197 100644 --- a/crates/ctx-history-store/src/provider_files/observation_validation.rs +++ b/crates/ctx-history-store/src/provider_files/observation_validation.rs @@ -1,35 +1,103 @@ impl Store { + fn ensure_rejected_publication_has_no_material( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result<()> { + let has_material: bool = self.conn.query_row( + &format!( + r#" + SELECT EXISTS ( + SELECT 1 + FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = ?1 + AND ( + (seen.entity_kind = 'history_record' AND EXISTS (SELECT 1 FROM history_records WHERE id = seen.entity_id)) + OR (seen.entity_kind = '{PRIOR_HISTORY_RECORD_KIND}' AND EXISTS (SELECT 1 FROM history_records WHERE id = seen.entity_id)) + OR (seen.entity_kind = '{CURRENT_CAPTURE_SOURCE_KIND}' AND EXISTS (SELECT 1 FROM capture_sources WHERE id = seen.entity_id)) + OR (seen.entity_kind = '{PRIOR_CAPTURE_SOURCE_KIND}' AND EXISTS (SELECT 1 FROM capture_sources WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'session' AND EXISTS (SELECT 1 FROM sessions WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'event' AND EXISTS (SELECT 1 FROM events WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'run' AND EXISTS (SELECT 1 FROM runs WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'file_touched' AND EXISTS (SELECT 1 FROM files_touched WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'session_edge' AND EXISTS (SELECT 1 FROM session_edges WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'artifact' AND EXISTS (SELECT 1 FROM artifacts WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'history_record_link' AND EXISTS (SELECT 1 FROM history_record_links WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'summary' AND EXISTS (SELECT 1 FROM summaries WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'vcs_workspace' AND EXISTS (SELECT 1 FROM vcs_workspaces WHERE id = seen.entity_id)) + OR (seen.entity_kind = 'vcs_change' AND EXISTS (SELECT 1 FROM vcs_changes WHERE id = seen.entity_id)) + OR seen.entity_kind NOT IN ( + 'history_record', 'session', 'event', 'run', 'file_touched', + 'session_edge', 'artifact', 'history_record_link', 'summary', + 'vcs_workspace', 'vcs_change', '{CURRENT_CAPTURE_SOURCE_KIND}', + '{PRIOR_CAPTURE_SOURCE_KIND}', '{PRIOR_HISTORY_RECORD_KIND}' + ) + ) + ) + "# + ), + params![scope.scope_id.to_string()], + |row| row.get(0), + )?; + if has_material { + return Err(StoreError::InvalidProviderFileCheckpoint( + "rejected replacement cannot publish staged material", + )); + } + Ok(()) + } + fn ensure_provider_file_observation_is_current( &self, provider: CaptureProvider, observation: ProviderFileInventoryObservation<'_>, ) -> Result<()> { self.ensure_inventory_generation_is_current(provider, observation)?; + self.ensure_provider_file_observation_matches_persisted(provider, observation) + } + + fn ensure_provider_file_observation_matches_persisted( + &self, + provider: CaptureProvider, + observation: ProviderFileInventoryObservation<'_>, + ) -> Result<()> { let matches = match observation { - ProviderFileInventoryObservation::Catalog { + ProviderFileInventoryObservation::ObservedCatalog { source_format, update, - } => self.conn.query_row( - r#" + metadata, + } => { + if metadata + .get("file_observation_token_v1") + .and_then(serde_json::Value::as_str) + .is_none_or(str::is_empty) + { + return Err(StoreError::InvalidProviderFileCheckpoint( + "catalog observation token is required", + )); + } + self.conn.query_row( + r#" SELECT EXISTS ( SELECT 1 FROM catalog_sessions WHERE provider = ?1 AND source_format = ?2 AND source_root = ?3 AND source_path = ?4 AND is_stale = 0 AND file_size_bytes = ?5 AND file_modified_at_ms = ?6 - AND import_revision = ?7 + AND import_revision = ?7 AND metadata_json IS ?8 ) "#, - params![ - provider.as_str(), - source_format, - update.source_root, - update.source_path, - capped_i64(update.file_size_bytes), - update.file_modified_at_ms, - i64::from(update.import_revision), - ], - |row| row.get::<_, bool>(0), - )?, + params![ + provider.as_str(), + source_format, + update.source_root, + update.source_path, + capped_i64(update.file_size_bytes), + update.file_modified_at_ms, + i64::from(update.import_revision), + serde_json::to_string(metadata)?, + ], + |row| row.get::<_, bool>(0), + )? + } ProviderFileInventoryObservation::SourceImport { source_format, update, @@ -95,35 +163,29 @@ impl Store { Ok(()) } - fn ensure_scope_observation_is_current( + fn current_provider_file_inventory_generation( &self, - scope: &ProviderFilePublicationScope, - ) -> Result<()> { - let generation_is_current = self - .conn + provider: CaptureProvider, + source_root: &str, + inventory_family: &str, + ) -> Result { + self.conn .query_row( r#" - SELECT current_generation = ?4 + SELECT current_generation FROM import_inventory_generations WHERE provider = ?1 AND source_root = ?2 AND inventory_family = ?3 "#, - params![ - scope.provider.as_str(), - &scope.inventory_source_root, - scope.inventory_family, - capped_i64(scope.inventory_generation), - ], - |row| row.get::<_, bool>(0), + params![provider.as_str(), source_root, inventory_family], + |row| nonnegative_i64_to_u64(row.get(0)?), ) - .optional()? - .unwrap_or(false); - if !generation_is_current { - return Err(StoreError::ImportInventorySuperseded { - provider: scope.provider.as_str().to_owned(), - inventory_family: scope.inventory_family, - expected_generation: scope.inventory_generation, - }); - } + .map_err(StoreError::from) + } + + fn ensure_scope_observation_is_current( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result<()> { let matches: bool = if scope.inventory_family == CATALOG_INVENTORY_FAMILY { self.conn.query_row( r#" @@ -132,6 +194,7 @@ impl Store { WHERE provider = ?1 AND source_format = ?2 AND source_root = ?3 AND source_path = ?4 AND is_stale = 0 AND file_size_bytes = ?5 AND file_modified_at_ms = ?6 AND import_revision = ?7 + AND (?8 IS NULL OR metadata_json IS ?8) ) "#, params![ @@ -142,6 +205,7 @@ impl Store { capped_i64(scope.file_size_bytes), scope.file_modified_at_ms, i64::from(scope.import_revision), + &scope.metadata_json, ], |row| row.get(0), )? @@ -192,11 +256,7 @@ impl Store { return self.ensure_scope_observation_is_current(scope); } if !marker.mutation_started - || self.provider_file_owner_has_current_observation( - scope.provider, - &scope.inventory_source_root, - &scope.source_path, - )? + || self.provider_file_publication_has_current_observation(scope.scope_id)? { return Err(StoreError::ProviderFileObservationChanged { provider: scope.provider.as_str().to_owned(), @@ -219,27 +279,21 @@ impl Store { Ok(()) } - fn provider_file_owner_has_current_observation( - &self, - provider: CaptureProvider, - source_root: &str, - source_path: &str, - ) -> Result { + fn provider_file_publication_has_current_observation(&self, scope_id: Uuid) -> Result { + let current_observation = + provider_file_retirement_observation_current_predicate("publication"); self.conn .query_row( - r#" + &format!( + r#" SELECT EXISTS ( - SELECT 1 FROM catalog_sessions - WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 - AND is_stale = 0 - UNION ALL - SELECT 1 FROM source_import_files - WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 - AND is_stale = 0 - LIMIT 1 + SELECT 1 FROM provider_file_publications AS publication + WHERE publication.replacement_id = ?1 + AND ({current_observation}) ) - "#, - params![provider.as_str(), source_root, source_path], + "# + ), + params![scope_id.to_string()], |row| row.get(0), ) .map_err(StoreError::from) @@ -256,10 +310,12 @@ impl Store { self.conn .query_row( r#" - SELECT replacement_id, staging_id, inventory_family, + SELECT replacement_id, staging_id, publication_kind, inventory_family, inventory_source_format, inventory_source_root, source_path, inventory_generation, file_size_bytes, file_modified_at_ms, - import_revision, metadata_json, mutation_started + import_revision, metadata_json, mutation_started, + tracks_prior_material, + staging_initialized FROM provider_file_publications WHERE owner_id = ?1 AND provider = ?2 AND material_source_format = ?3 AND material_source_root = ?4 AND source_path = ?5 @@ -275,18 +331,23 @@ impl Store { Ok(DurableProviderFilePublication { scope_id: parse_uuid_text(row.get(0)?)?, staging_id: row.get(1)?, - inventory_family: parse_provider_file_inventory_family_sql( + publication_kind: parse_provider_file_publication_kind_sql( &row.get::<_, String>(2)?, )?, - inventory_source_format: row.get(3)?, - inventory_source_root: row.get(4)?, - source_path: row.get(5)?, - inventory_generation: nonnegative_i64_to_u64(row.get(6)?)?, - file_size_bytes: nonnegative_i64_to_u64(row.get(7)?)?, - file_modified_at_ms: row.get(8)?, - import_revision: nonnegative_i64_to_u32(row.get(9)?)?, - metadata_json: row.get(10)?, - mutation_started: row.get(11)?, + inventory_family: parse_provider_file_inventory_family_sql( + &row.get::<_, String>(3)?, + )?, + inventory_source_format: row.get(4)?, + inventory_source_root: row.get(5)?, + source_path: row.get(6)?, + inventory_generation: nonnegative_i64_to_u64(row.get(7)?)?, + file_size_bytes: nonnegative_i64_to_u64(row.get(8)?)?, + file_modified_at_ms: row.get(9)?, + import_revision: nonnegative_i64_to_u32(row.get(10)?)?, + metadata_json: row.get(11)?, + mutation_started: row.get(12)?, + tracks_prior_material: row.get(13)?, + staging_initialized: row.get(14)?, }) }, ) @@ -299,7 +360,6 @@ impl Store { let compatible = existing.import_revision == checkpoint.import_revision && existing.checkpoint_version == checkpoint.checkpoint_version && existing.stable_file_identity == checkpoint.stable_file_identity - && existing.head_sha256 == checkpoint.head_sha256 && existing.committed_byte_offset <= checkpoint.committed_byte_offset && existing.committed_complete_line_count <= checkpoint.committed_complete_line_count @@ -377,12 +437,13 @@ impl Store { self.conn.execute( &format!( "DELETE FROM {table} WHERE provider = ?1 AND source_root = ?2 \ - AND source_path = ?3 AND is_stale != 0" + AND source_path = ?3 AND source_format = ?4 AND is_stale != 0" ), params![ scope.provider.as_str(), &scope.inventory_source_root, &scope.source_path, + &scope.inventory_source_format, ], )?; Ok(()) diff --git a/crates/ctx-history-store/src/provider_files/publication_marker.rs b/crates/ctx-history-store/src/provider_files/publication_marker.rs index 5d80d333..5ad46faf 100644 --- a/crates/ctx-history-store/src/provider_files/publication_marker.rs +++ b/crates/ctx-history-store/src/provider_files/publication_marker.rs @@ -4,36 +4,139 @@ impl Store { scope: &mut ProviderFilePublicationScope, created_at_ms: i64, ) -> Result<()> { + let global_id = global_provider_file_publication_id_sql(); + let global_owner = self + .conn + .query_row( + &format!( + r#" + SELECT provider, owner_id, inventory_family, + inventory_source_format, inventory_source_root, source_path, + material_source_format, material_source_root + FROM provider_file_publications + WHERE replacement_id = ({global_id}) + "# + ), + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + )) + }, + ) + .optional()?; + if let Some(( + provider, + owner_id, + inventory_family, + inventory_source_format, + inventory_source_root, + source_path, + material_source_format, + material_source_root, + )) = global_owner + { + let exact_owner = owner_id == scope.owner_id + && inventory_family == scope.inventory_family + && inventory_source_format == scope.inventory_source_format + && inventory_source_root == scope.inventory_source_root + && source_path == scope.source_path + && material_source_format == scope.material_source_format + && material_source_root == scope.material_source_root; + if !exact_owner { + return Err(StoreError::ProviderFileReplacementBusy { provider, owner_id }); + } + } let prior = self .conn .query_row( r#" - SELECT replacement_id, staging_id, publication_kind, mutation_started + SELECT replacement_id, staging_id, publication_kind, mutation_started, + inventory_family = ?2 + AND inventory_source_format = ?3 + AND inventory_source_root = ?4 + AND source_path = ?5 + AND material_source_format = ?6 + AND material_source_root = ?7 + AND file_size_bytes = ?8 + AND file_modified_at_ms = ?9 + AND import_revision = ?10 + AND metadata_json IS ?11, + tracks_prior_material, staging_initialized, retirement_started, + completion_payload_json IS NOT NULL FROM provider_file_publications WHERE owner_id = ?1 "#, - params![&scope.owner_id], + params![ + &scope.owner_id, + scope.inventory_family, + &scope.inventory_source_format, + &scope.inventory_source_root, + &scope.source_path, + &scope.material_source_format, + &scope.material_source_root, + capped_i64(scope.file_size_bytes), + scope.file_modified_at_ms, + i64::from(scope.import_revision), + &scope.metadata_json, + ], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, bool>(3)?, + row.get::<_, bool>(4)?, + row.get::<_, bool>(5)?, + row.get::<_, bool>(6)?, + row.get::<_, bool>(7)?, + row.get::<_, bool>(8)?, )) }, ) .optional()?; - let changed = if let Some((publication_id, staging_id, prior_kind, mutation_started)) = - prior + let changed = if let Some(( + publication_id, + staging_id, + prior_kind, + mutation_started, + same_observation, + tracks_prior_material, + staging_initialized, + retirement_started, + has_staged_completion, + )) = prior { let prior_kind = parse_provider_file_publication_kind(&prior_kind)?; - scope.kind = match (prior_kind, mutation_started) { - (_, true) => ProviderFilePublicationKind::Replacement, - (_, false) => scope.kind, + scope.kind = match (prior_kind, mutation_started, scope.retires_observation) { + (prior_kind, true, true) => prior_kind, + (_, true, false) => ProviderFilePublicationKind::Replacement, + (_, false, _) => scope.kind, }; - scope.tracks_prior_material |= mutation_started; scope.scope_id = Uuid::parse_str(&publication_id)?; scope.staging_id = staging_id; + if mutation_started { + scope.tracks_prior_material = if same_observation && has_staged_completion { + tracks_prior_material + } else { + tracks_prior_material + || scope.tracks_prior_material + || self.provider_file_publication_has_staged_material(scope.scope_id)? + }; + } + let reuse_staging_state = same_observation + && prior_kind == scope.kind + && (scope.retires_observation || !retirement_started); + let reuse_staging_state = reuse_staging_state && staging_initialized; + let clear_observation_invalidation = !scope.retires_observation && !same_observation; + scope.reuse_staging_state = reuse_staging_state; self.conn.execute( r#" UPDATE provider_file_publications @@ -44,24 +147,43 @@ impl Store { inventory_generation = ?10, file_size_bytes = ?11, file_modified_at_ms = ?12, import_revision = ?13, metadata_json = ?14, mutation_started = ?15, - preparation_complete = CASE WHEN ?2 = 'incremental' THEN 1 ELSE 0 END, - preparation_cursor = NULL, cleanup_phase = 0, - cleanup_source_cursor = NULL, cleanup_entity_cursor = NULL, - removed_artifacts = CASE WHEN ?15 THEN removed_artifacts ELSE 0 END, - removed_summaries = CASE WHEN ?15 THEN removed_summaries ELSE 0 END, - removed_history_record_links = CASE WHEN ?15 THEN removed_history_record_links ELSE 0 END, - removed_history_records = CASE WHEN ?15 THEN removed_history_records ELSE 0 END, - removed_history_record_tags = CASE WHEN ?15 THEN removed_history_record_tags ELSE 0 END, - removed_record_edges = CASE WHEN ?15 THEN removed_record_edges ELSE 0 END, - removed_audit_log_entries = CASE WHEN ?15 THEN removed_audit_log_entries ELSE 0 END, - removed_vcs_workspaces = CASE WHEN ?15 THEN removed_vcs_workspaces ELSE 0 END, - removed_vcs_changes = CASE WHEN ?15 THEN removed_vcs_changes ELSE 0 END, - removed_events = CASE WHEN ?15 THEN removed_events ELSE 0 END, - removed_runs = CASE WHEN ?15 THEN removed_runs ELSE 0 END, - removed_files_touched = CASE WHEN ?15 THEN removed_files_touched ELSE 0 END, - removed_session_edges = CASE WHEN ?15 THEN removed_session_edges ELSE 0 END, - tombstoned_sessions = CASE WHEN ?15 THEN tombstoned_sessions ELSE 0 END, - started_at_ms = ?16, updated_at_ms = ?16 + tracks_prior_material = ?16, + inventory_observation_invalidated = CASE + WHEN ?19 THEN 0 ELSE inventory_observation_invalidated + END, + retirement_started = CASE + WHEN ?17 THEN retirement_started ELSE 0 + END, + staging_initialized = CASE + WHEN ?17 THEN staging_initialized ELSE 0 + END, + preparation_complete = CASE + WHEN ?17 THEN preparation_complete + WHEN ?2 = 'incremental' THEN 1 ELSE 0 + END, + preparation_cursor = CASE WHEN ?17 THEN preparation_cursor ELSE NULL END, + cleanup_phase = CASE WHEN ?17 THEN cleanup_phase ELSE 0 END, + cleanup_source_cursor = CASE WHEN ?17 THEN cleanup_source_cursor ELSE NULL END, + cleanup_entity_cursor = CASE WHEN ?17 THEN cleanup_entity_cursor ELSE NULL END, + removed_artifacts = CASE WHEN ?17 THEN removed_artifacts ELSE 0 END, + removed_summaries = CASE WHEN ?17 THEN removed_summaries ELSE 0 END, + removed_history_record_links = CASE WHEN ?17 THEN removed_history_record_links ELSE 0 END, + removed_history_records = CASE WHEN ?17 THEN removed_history_records ELSE 0 END, + removed_history_record_tags = CASE WHEN ?17 THEN removed_history_record_tags ELSE 0 END, + removed_record_edges = CASE WHEN ?17 THEN removed_record_edges ELSE 0 END, + removed_audit_log_entries = CASE WHEN ?17 THEN removed_audit_log_entries ELSE 0 END, + removed_vcs_workspaces = CASE WHEN ?17 THEN removed_vcs_workspaces ELSE 0 END, + removed_vcs_changes = CASE WHEN ?17 THEN removed_vcs_changes ELSE 0 END, + removed_events = CASE WHEN ?17 THEN removed_events ELSE 0 END, + removed_runs = CASE WHEN ?17 THEN removed_runs ELSE 0 END, + removed_files_touched = CASE WHEN ?17 THEN removed_files_touched ELSE 0 END, + removed_session_edges = CASE WHEN ?17 THEN removed_session_edges ELSE 0 END, + tombstoned_sessions = CASE WHEN ?17 THEN tombstoned_sessions ELSE 0 END, + completion_payload_json = CASE + WHEN ?17 THEN completion_payload_json ELSE NULL + END, + started_at_ms = ?18, + updated_at_ms = MAX(updated_at_ms + 1, ?18) WHERE owner_id = ?3 AND replacement_id = ?1 "#, params![ @@ -80,7 +202,10 @@ impl Store { i64::from(scope.import_revision), &scope.metadata_json, mutation_started, + scope.tracks_prior_material, + reuse_staging_state, created_at_ms, + clear_observation_invalidation, ], )? } else { @@ -91,7 +216,8 @@ impl Store { inventory_family, inventory_source_format, inventory_source_root, source_path, material_source_format, material_source_root, inventory_generation, file_size_bytes, file_modified_at_ms, - import_revision, metadata_json, mutation_started, + import_revision, metadata_json, mutation_started, tracks_prior_material, + staging_initialized, preparation_complete, preparation_cursor, cleanup_phase, cleanup_source_cursor, cleanup_entity_cursor, removed_artifacts, removed_summaries, removed_history_record_links, @@ -99,12 +225,12 @@ impl Store { removed_audit_log_entries, removed_vcs_workspaces, removed_vcs_changes, removed_events, removed_runs, removed_files_touched, removed_session_edges, tombstoned_sessions, - started_at_ms, updated_at_ms) + started_at_ms, updated_at_ms, completion_payload_json) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, - ?13, ?14, ?15, ?16, 0, + ?13, ?14, ?15, ?16, 0, ?17, 0, CASE WHEN ?3 = 'incremental' THEN 1 ELSE 0 END, NULL, 0, NULL, NULL, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ?17, ?17) + ?18, ?18, NULL) "#, params![ scope.scope_id.to_string(), @@ -123,6 +249,7 @@ impl Store { scope.file_modified_at_ms, i64::from(scope.import_revision), &scope.metadata_json, + scope.tracks_prior_material, created_at_ms, ], )? @@ -151,6 +278,7 @@ impl Store { r#" SELECT publication_kind, mutation_started, preparation_complete, preparation_cursor, cleanup_phase, cleanup_source_cursor, cleanup_entity_cursor, + completion_payload_json, removed_artifacts, removed_summaries, removed_history_record_links, removed_history_records, removed_history_record_tags, removed_record_edges, removed_audit_log_entries, removed_vcs_workspaces, removed_vcs_changes, @@ -191,21 +319,22 @@ impl Store { cleanup_phase: row.get(4)?, source_cursor: row.get(5)?, entity_cursor: row.get(6)?, + completion_payload_json: row.get(7)?, counts: ProviderFileReconciliationCounts { - artifacts: nonnegative_i64_to_usize(row.get(7)?)?, - summaries: nonnegative_i64_to_usize(row.get(8)?)?, - history_record_links: nonnegative_i64_to_usize(row.get(9)?)?, - history_records: nonnegative_i64_to_usize(row.get(10)?)?, - history_record_tags: nonnegative_i64_to_usize(row.get(11)?)?, - record_edges: nonnegative_i64_to_usize(row.get(12)?)?, - audit_log_entries: nonnegative_i64_to_usize(row.get(13)?)?, - vcs_workspaces: nonnegative_i64_to_usize(row.get(14)?)?, - vcs_changes: nonnegative_i64_to_usize(row.get(15)?)?, - events: nonnegative_i64_to_usize(row.get(16)?)?, - runs: nonnegative_i64_to_usize(row.get(17)?)?, - files_touched: nonnegative_i64_to_usize(row.get(18)?)?, - session_edges: nonnegative_i64_to_usize(row.get(19)?)?, - sessions_tombstoned: nonnegative_i64_to_usize(row.get(20)?)?, + artifacts: nonnegative_i64_to_usize(row.get(8)?)?, + summaries: nonnegative_i64_to_usize(row.get(9)?)?, + history_record_links: nonnegative_i64_to_usize(row.get(10)?)?, + history_records: nonnegative_i64_to_usize(row.get(11)?)?, + history_record_tags: nonnegative_i64_to_usize(row.get(12)?)?, + record_edges: nonnegative_i64_to_usize(row.get(13)?)?, + audit_log_entries: nonnegative_i64_to_usize(row.get(14)?)?, + vcs_workspaces: nonnegative_i64_to_usize(row.get(15)?)?, + vcs_changes: nonnegative_i64_to_usize(row.get(16)?)?, + events: nonnegative_i64_to_usize(row.get(17)?)?, + runs: nonnegative_i64_to_usize(row.get(18)?)?, + files_touched: nonnegative_i64_to_usize(row.get(19)?)?, + session_edges: nonnegative_i64_to_usize(row.get(20)?)?, + sessions_tombstoned: nonnegative_i64_to_usize(row.get(21)?)?, }, }) }, @@ -230,7 +359,7 @@ impl Store { removed_audit_log_entries = ?14, removed_vcs_workspaces = ?15, removed_vcs_changes = ?16, removed_events = ?17, removed_runs = ?18, removed_files_touched = ?19, removed_session_edges = ?20, - tombstoned_sessions = ?21, updated_at_ms = ?22 + tombstoned_sessions = ?21, updated_at_ms = MAX(updated_at_ms, ?22) WHERE replacement_id = ?1 AND publication_kind = ?23 "#, params![ @@ -297,115 +426,76 @@ impl Store { Ok((lock, lock_path)) } - fn attach_provider_file_publication_staging( + fn advance_provider_file_publication_attempt( &self, scope: &ProviderFilePublicationScope, + attempted_at_ms: i64, ) -> Result<()> { - let owner_id = provider_file_owner_lock_name( - self.store_identity.digest(), - scope.provider, - &scope.material_source_format, - &scope.material_source_root, - &scope.source_path, - ); - let staging_dir = self.store_identity.private_root().join(format!( - "{STAGING_DIR_PREFIX}-{owner_id}-{}", - scope.staging_id - )); - create_or_validate_private_lock_dir(&staging_dir)?; - #[cfg(test)] - let staging_dir_mode = staging_directory_mode(&staging_dir)?; - let staging_path = staging_dir.join("seen.sqlite"); - let file = match create_private_staging_file(&staging_path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - open_existing_private_staging_file(&staging_path)? - } - Err(error) => return Err(error.into()), - }; - drop(file); - #[cfg(test)] - let staging_file_mode = staging_file_mode(&staging_path)?; + self.ensure_active_provider_file_publication(scope)?; + let changed = self.conn.execute( + r#" + UPDATE provider_file_publications + SET updated_at_ms = MAX(updated_at_ms + 1, ?2) + WHERE replacement_id = ?1 AND mutation_started != 0 + "#, + params![scope.scope_id.to_string(), attempted_at_ms], + )?; + if changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + Ok(()) + } - let attach_result = (|| -> Result<()> { - self.conn.execute( - &format!("ATTACH DATABASE ?1 AS {STAGING_SCHEMA}"), - params![staging_path.to_string_lossy().as_ref()], - )?; - self.conn.execute_batch(&format!( - r#" - PRAGMA {STAGING_SCHEMA}.page_size = 4096; - PRAGMA {STAGING_SCHEMA}.cache_size = -8192; - PRAGMA {STAGING_SCHEMA}.journal_mode = OFF; - PRAGMA {STAGING_SCHEMA}.synchronous = OFF; - CREATE TABLE IF NOT EXISTS {STAGING_SCHEMA}.scope ( - scope_id TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL, - material_source_format TEXT NOT NULL, - material_source_root TEXT NOT NULL, - source_path TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS {STAGING_SCHEMA}.seen ( - entity_kind TEXT NOT NULL, - entity_id TEXT NOT NULL, - PRIMARY KEY (entity_kind, entity_id) - ); - CREATE TABLE IF NOT EXISTS {STAGING_SCHEMA}.prior_sources (id TEXT PRIMARY KEY NOT NULL); - CREATE TABLE IF NOT EXISTS {STAGING_SCHEMA}.batch ( - source_id TEXT NOT NULL, - entity_id TEXT NOT NULL, - PRIMARY KEY (source_id, entity_id) - ); - DELETE FROM {STAGING_SCHEMA}.scope; - DELETE FROM {STAGING_SCHEMA}.seen; - DELETE FROM {STAGING_SCHEMA}.prior_sources; - DELETE FROM {STAGING_SCHEMA}.batch; - "# - ))?; - self.conn.execute( - &format!( - "INSERT INTO {STAGING_SCHEMA}.scope - (scope_id, provider, material_source_format, material_source_root, source_path) - VALUES (?1, ?2, ?3, ?4, ?5)" - ), - params![ - scope.scope_id.to_string(), - scope.provider.as_str(), - &scope.material_source_format, - &scope.material_source_root, - &scope.source_path, - ], - )?; - Ok(()) - })(); - if let Err(error) = attach_result { - let _ = self - .conn - .execute_batch(&format!("DETACH DATABASE {STAGING_SCHEMA}")); - return Err(error); + fn attach_provider_file_publication_staging( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result<()> { + if !scope.retires_observation { + self.with_atomic_provider_file_update(|| { + let marker = self.load_replacement_marker(scope)?; + let replacement_id = scope.scope_id.to_string(); + let staged_state_exists: bool = self.conn.query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1) \ + OR EXISTS (SELECT 1 FROM {STAGING_PRIOR_SOURCES_TABLE} WHERE replacement_id = ?1)" + ), + params![&replacement_id], + |row| row.get(0), + )?; + let progress_without_state = !staged_state_exists + && (marker.mutation_started + || marker.preparation_cursor.is_some() + || marker.cleanup_phase != CLEANUP_PHASE_LINKS + || marker.source_cursor.is_some() + || marker.entity_cursor.is_some() + || (scope.tracks_prior_material + && marker.completion_payload_json.is_some()) + || marker.counts != ProviderFileReconciliationCounts::default()); + if !scope.reuse_staging_state || progress_without_state { + self.reset_provider_file_publication_staging(scope)?; + } else { + self.conn.execute( + &format!("DELETE FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1"), + params![&replacement_id], + )?; + let changed = self.conn.execute( + "UPDATE provider_file_publications SET staging_initialized = 1 \ + WHERE replacement_id = ?1", + params![&replacement_id], + )?; + if changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + } + Ok(()) + })?; } - #[cfg(unix)] - fs::remove_file(&staging_path)?; let mut active = self.provider_file_publication.borrow_mut(); let active = active .as_mut() .filter(|active| active.scope_id == scope.scope_id) .ok_or(StoreError::InvalidProviderFilePublicationScope)?; active.attached = true; - active.staging_dir_path = Some(staging_dir); - #[cfg(unix)] - { - active.staging_path = None; - } - #[cfg(not(unix))] - { - active.staging_path = Some(staging_path); - } - #[cfg(test)] - { - active.staging_file_mode = staging_file_mode; - active.staging_dir_mode = staging_dir_mode; - } Ok(()) } } diff --git a/crates/ctx-history-store/src/provider_files/reconciliation_delete.rs b/crates/ctx-history-store/src/provider_files/reconciliation_delete.rs index d2fcf6a6..e486c2e9 100644 --- a/crates/ctx-history-store/src/provider_files/reconciliation_delete.rs +++ b/crates/ctx-history-store/src/provider_files/reconciliation_delete.rs @@ -1,24 +1,33 @@ impl Store { - fn delete_unseen_batch(&self, entity_kind: &'static str, table: &'static str) -> Result { + fn delete_unseen_batch( + &self, + replacement_id: &str, + entity_kind: &'static str, + table: &'static str, + ) -> Result { self.conn .execute( &format!( r#" DELETE FROM {table} - WHERE id IN (SELECT entity_id FROM {STAGING_SCHEMA}.batch) + WHERE id IN ( + SELECT entity_id FROM {STAGING_BATCH_TABLE} + WHERE replacement_id = ?1 + ) AND NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = ?1 AND seen.entity_id = {table}.id + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = ?1 + AND seen.entity_kind = ?2 AND seen.entity_id = {table}.id ) "# ), - params![entity_kind], + params![replacement_id, entity_kind], ) .map_err(StoreError::from) } - fn delete_unseen_event_batch(&self) -> Result { - let stale_ids = self.unseen_batch_ids("event")?; + fn delete_unseen_event_batch(&self, replacement_id: &str) -> Result { + let stale_ids = self.unseen_batch_ids(replacement_id, "event")?; if stale_ids.is_empty() { return Ok(0); } @@ -28,17 +37,19 @@ impl Store { SELECT EXISTS ( SELECT 1 FROM files_touched AS file WHERE file.event_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'event' - AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = batch.replacement_id + AND seen.entity_kind = 'event' + AND seen.entity_id = batch.entity_id ) ) ) "# ), - [], + params![replacement_id], |row| row.get(0), )?; if surviving_reference { @@ -60,60 +71,66 @@ impl Store { self.conn.execute( &format!( "DELETE FROM {table} WHERE event_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'event' - AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = batch.replacement_id + AND seen.entity_kind = 'event' + AND seen.entity_id = batch.entity_id ) )" ), - [], + params![replacement_id], )?; } } - let removed = self.delete_unseen_batch("event", "events")?; + let removed = self.delete_unseen_batch(replacement_id, "event", "events")?; decrement_semantic_searchable_item_stats_if_cached(&self.conn, semantic_removed)?; Ok(removed) } - fn delete_unseen_run_batch(&self) -> Result { + fn delete_unseen_run_batch(&self, replacement_id: &str) -> Result { let surviving_reference: bool = self.conn.query_row( &format!( r#" SELECT EXISTS ( SELECT 1 FROM events AS event WHERE event.run_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'run' - AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = batch.replacement_id + AND seen.entity_kind = 'run' + AND seen.entity_id = batch.entity_id ) ) UNION ALL SELECT 1 FROM files_touched AS file WHERE file.run_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'run' - AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = batch.replacement_id + AND seen.entity_kind = 'run' + AND seen.entity_id = batch.entity_id ) ) ) "# ), - [], + params![replacement_id], |row| row.get(0), )?; if surviving_reference { return Err(StoreError::ProviderFileReconciliationInconsistent { entity: "run" }); } - self.delete_unseen_batch("run", "runs") + self.delete_unseen_batch(replacement_id, "run", "runs") } - fn delete_unseen_vcs_change_batch(&self) -> Result { + fn delete_unseen_vcs_change_batch(&self, replacement_id: &str) -> Result { let surviving_link: bool = self.conn.query_row( &format!( r#" @@ -121,17 +138,19 @@ impl Store { SELECT 1 FROM history_record_links AS link WHERE link.target_type = 'vcs_change' AND link.target_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'vcs_change' - AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = batch.replacement_id + AND seen.entity_kind = 'vcs_change' + AND seen.entity_id = batch.entity_id ) ) ) "# ), - [], + params![replacement_id], |row| row.get(0), )?; if surviving_link { @@ -139,16 +158,19 @@ impl Store { entity: "VCS change", }); } - self.delete_unseen_batch("vcs_change", "vcs_changes") + self.delete_unseen_batch(replacement_id, "vcs_change", "vcs_changes") } - fn delete_unseen_artifact_batch(&self) -> Result { + fn delete_unseen_artifact_batch(&self, replacement_id: &str) -> Result { let eligible = format!( r#" - id IN (SELECT entity_id FROM {STAGING_SCHEMA}.batch) + id IN ( + SELECT entity_id FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1 + ) AND NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'artifact' AND seen.entity_id = artifacts.id + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = ?1 + AND seen.entity_kind = 'artifact' AND seen.entity_id = artifacts.id ) AND NOT EXISTS (SELECT 1 FROM sessions WHERE transcript_blob_id = artifacts.id) AND NOT EXISTS ( @@ -169,47 +191,55 @@ impl Store { SELECT id FROM artifacts WHERE {eligible} )" ), - [], + params![replacement_id], )?; } self.conn - .execute(&format!("DELETE FROM artifacts WHERE {eligible}"), []) + .execute( + &format!("DELETE FROM artifacts WHERE {eligible}"), + params![replacement_id], + ) .map_err(StoreError::from) } - fn delete_unseen_vcs_workspace_batch(&self) -> Result { + fn delete_unseen_vcs_workspace_batch(&self, replacement_id: &str) -> Result { let surviving_reference: bool = self.conn.query_row( &format!( r#" SELECT EXISTS ( SELECT 1 FROM vcs_changes WHERE vcs_workspace_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS (SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen WHERE seen.entity_kind = 'vcs_workspace' AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id) + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen WHERE seen.replacement_id = batch.replacement_id AND seen.entity_kind = 'vcs_workspace' AND seen.entity_id = batch.entity_id) ) UNION ALL SELECT 1 FROM files_touched WHERE vcs_workspace_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS (SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen WHERE seen.entity_kind = 'vcs_workspace' AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id) + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen WHERE seen.replacement_id = batch.replacement_id AND seen.entity_kind = 'vcs_workspace' AND seen.entity_id = batch.entity_id) ) UNION ALL SELECT 1 FROM history_records WHERE primary_vcs_workspace_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS (SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen WHERE seen.entity_kind = 'vcs_workspace' AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id) + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen WHERE seen.replacement_id = batch.replacement_id AND seen.entity_kind = 'vcs_workspace' AND seen.entity_id = batch.entity_id) ) UNION ALL SELECT 1 FROM local_workspaces WHERE vcs_workspace_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS (SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen WHERE seen.entity_kind = 'vcs_workspace' AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id) + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen WHERE seen.replacement_id = batch.replacement_id AND seen.entity_kind = 'vcs_workspace' AND seen.entity_id = batch.entity_id) ) UNION ALL SELECT 1 FROM history_record_links WHERE target_type = 'vcs_workspace' AND target_id IN ( - SELECT entity_id FROM {STAGING_SCHEMA}.batch - WHERE NOT EXISTS (SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen WHERE seen.entity_kind = 'vcs_workspace' AND seen.entity_id = {STAGING_SCHEMA}.batch.entity_id) + SELECT batch.entity_id FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen WHERE seen.replacement_id = batch.replacement_id AND seen.entity_kind = 'vcs_workspace' AND seen.entity_id = batch.entity_id) ) ) "# ), - [], + params![replacement_id], |row| row.get(0), )?; if surviving_reference { @@ -217,29 +247,33 @@ impl Store { entity: "VCS workspace", }); } - self.delete_unseen_batch("vcs_workspace", "vcs_workspaces") + self.delete_unseen_batch(replacement_id, "vcs_workspace", "vcs_workspaces") } - fn delete_history_record_tag_batch(&self) -> Result { + fn delete_history_record_tag_batch(&self, replacement_id: &str) -> Result { self.conn .execute( &format!( "DELETE FROM history_record_tags WHERE rowid IN ( - SELECT CAST(entity_id AS INTEGER) FROM {STAGING_SCHEMA}.batch + SELECT CAST(entity_id AS INTEGER) FROM {STAGING_BATCH_TABLE} + WHERE replacement_id = ?1 )" ), - [], + params![replacement_id], ) .map_err(StoreError::from) } - fn delete_unseen_history_record_batch(&self) -> Result { + fn delete_unseen_history_record_batch(&self, replacement_id: &str) -> Result { let eligible = format!( r#" - id IN (SELECT entity_id FROM {STAGING_SCHEMA}.batch) + id IN ( + SELECT entity_id FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1 + ) AND NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'history_record' + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = ?1 + AND seen.entity_kind = 'history_record' AND seen.entity_id = history_records.id ) AND NOT EXISTS (SELECT 1 FROM sessions WHERE history_record_id = history_records.id) @@ -269,27 +303,73 @@ impl Store { SELECT id FROM history_records WHERE {eligible} )" ), - [], + params![replacement_id], )?; } } self.conn - .execute(&format!("DELETE FROM history_records WHERE {eligible}"), []) + .execute( + &format!("DELETE FROM history_records WHERE {eligible}"), + params![replacement_id], + ) + .map_err(StoreError::from) + } + + fn delete_unseen_capture_source_batch(&self, replacement_id: &str) -> Result { + self.conn + .execute( + &format!( + r#" + DELETE FROM capture_sources + WHERE id IN ( + SELECT entity_id FROM {STAGING_BATCH_TABLE} + WHERE replacement_id = ?1 + ) + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = ?1 + AND seen.entity_kind = ?2 + AND seen.entity_id = capture_sources.id + ) + AND NOT EXISTS (SELECT 1 FROM vcs_workspaces WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM history_records WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM sessions WHERE capture_source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM session_edges WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM runs WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM events WHERE capture_source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM vcs_changes WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM history_record_links WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM summaries WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM files_touched WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM history_record_tags WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM record_edges WHERE source_id = capture_sources.id) + AND NOT EXISTS (SELECT 1 FROM audit_log WHERE source_id = capture_sources.id) + "# + ), + params![replacement_id, CURRENT_CAPTURE_SOURCE_KIND], + ) .map_err(StoreError::from) } - fn unseen_batch_ids(&self, entity_kind: &'static str) -> Result> { + fn unseen_batch_ids( + &self, + replacement_id: &str, + entity_kind: &'static str, + ) -> Result> { let mut stmt = self.conn.prepare(&format!( r#" SELECT batch.entity_id - FROM {STAGING_SCHEMA}.batch AS batch - WHERE NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = ?1 AND seen.entity_id = batch.entity_id + FROM {STAGING_BATCH_TABLE} AS batch + WHERE batch.replacement_id = ?1 + AND NOT EXISTS ( + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = batch.replacement_id + AND seen.entity_kind = ?2 AND seen.entity_id = batch.entity_id ) "# ))?; - let rows = stmt.query_map(params![entity_kind], |row| row.get(0))?; + let rows = stmt.query_map(params![replacement_id, entity_kind], |row| row.get(0))?; rows.collect::>>() .map_err(StoreError::from) } @@ -305,11 +385,15 @@ impl Store { UPDATE sessions SET deleted_at_ms = ?1, updated_at_ms = max(updated_at_ms, ?1), transcript_blob_id = NULL - WHERE id IN (SELECT entity_id FROM {STAGING_SCHEMA}.batch) + WHERE id IN ( + SELECT entity_id FROM {STAGING_BATCH_TABLE} + WHERE replacement_id = ?2 + ) AND deleted_at_ms IS NULL AND NOT EXISTS ( - SELECT 1 FROM {STAGING_SCHEMA}.seen AS seen - WHERE seen.entity_kind = 'session' AND seen.entity_id = sessions.id + SELECT 1 FROM {STAGING_SEEN_TABLE} AS seen + WHERE seen.replacement_id = ?2 + AND seen.entity_kind = 'session' AND seen.entity_id = sessions.id ) AND NOT EXISTS (SELECT 1 FROM events WHERE events.session_id = sessions.id) AND NOT EXISTS (SELECT 1 FROM runs WHERE runs.session_id = sessions.id) @@ -339,7 +423,8 @@ impl Store { AND ( related.capture_source_id IS NULL OR related.capture_source_id NOT IN ( - SELECT id FROM {STAGING_SCHEMA}.prior_sources + SELECT source_id FROM {STAGING_PRIOR_SOURCES_TABLE} + WHERE replacement_id = ?2 ) ) AND ( @@ -349,7 +434,7 @@ impl Store { ) "# ), - params![scope.file_modified_at_ms], + params![scope.file_modified_at_ms, scope.scope_id.to_string()], ) .map_err(StoreError::from) } diff --git a/crates/ctx-history-store/src/provider_files/reconciliation_scan.rs b/crates/ctx-history-store/src/provider_files/reconciliation_scan.rs index aa96ce41..4a8b28b2 100644 --- a/crates/ctx-history-store/src/provider_files/reconciliation_scan.rs +++ b/crates/ctx-history-store/src/provider_files/reconciliation_scan.rs @@ -7,21 +7,45 @@ impl Store { entity_cursor: Option<&str>, limit: usize, ) -> Result { - let scan = self.reconciliation_batch_rows(phase, source_cursor, entity_cursor, limit)?; + let replacement_id = scope.scope_id.to_string(); + if phase == CLEANUP_PHASE_AUDIT_LOG + && matches!( + source_cursor, + Some(PRIOR_HISTORY_RECORD_CURSOR | PRIOR_CAPTURE_SOURCE_CURSOR) + ) + { + return self.reconcile_prior_seen_batch( + &replacement_id, + source_cursor.ok_or(StoreError::InvalidProviderFilePublicationScope)?, + entity_cursor, + limit, + ); + } + let scan = self.reconciliation_batch_rows( + &replacement_id, + phase, + source_cursor, + entity_cursor, + limit, + )?; if scan.owned_entity_ids.is_empty() { - return Ok(ReconciliationBatch { + let batch = ReconciliationBatch { visited: scan.visited, phase_complete: scan.phase_complete, source_cursor: scan.source_cursor, entity_cursor: scan.entity_cursor, removed: ProviderFileReconciliationCounts::default(), - }); + }; + return self.begin_prior_seen_cleanup_if_needed(phase, &replacement_id, batch); } - self.conn - .execute(&format!("DELETE FROM {STAGING_SCHEMA}.batch"), [])?; + self.conn.execute( + &format!("DELETE FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1"), + params![&replacement_id], + )?; { let mut insert = self.conn.prepare_cached(&format!( - "INSERT INTO {STAGING_SCHEMA}.batch (source_id, entity_id) VALUES (?1, ?2)" + "INSERT INTO {STAGING_BATCH_TABLE} \ + (replacement_id, source_id, entity_id) VALUES (?1, ?2, ?3)" ))?; let source_id = scan.batch_source_id.as_deref().ok_or( StoreError::ProviderFileReconciliationInconsistent { @@ -29,33 +53,44 @@ impl Store { }, )?; for entity_id in &scan.owned_entity_ids { - insert.execute(params![source_id, entity_id])?; + insert.execute(params![&replacement_id, source_id, entity_id])?; } } let removed = match phase { CLEANUP_PHASE_LINKS => ProviderFileReconciliationCounts { - history_record_links: self - .delete_unseen_batch("history_record_link", "history_record_links")?, + history_record_links: self.delete_unseen_batch( + &replacement_id, + "history_record_link", + "history_record_links", + )?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_FILES => ProviderFileReconciliationCounts { - files_touched: self.delete_unseen_batch("file_touched", "files_touched")?, + files_touched: self.delete_unseen_batch( + &replacement_id, + "file_touched", + "files_touched", + )?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_EDGES => ProviderFileReconciliationCounts { - session_edges: self.delete_unseen_batch("session_edge", "session_edges")?, + session_edges: self.delete_unseen_batch( + &replacement_id, + "session_edge", + "session_edges", + )?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_SUMMARIES => ProviderFileReconciliationCounts { - summaries: self.delete_unseen_batch("summary", "summaries")?, + summaries: self.delete_unseen_batch(&replacement_id, "summary", "summaries")?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_EVENTS => ProviderFileReconciliationCounts { - events: self.delete_unseen_event_batch()?, + events: self.delete_unseen_event_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_RUNS => ProviderFileReconciliationCounts { - runs: self.delete_unseen_run_batch()?, + runs: self.delete_unseen_run_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_SESSIONS => ProviderFileReconciliationCounts { @@ -63,46 +98,211 @@ impl Store { ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_VCS_CHANGES => ProviderFileReconciliationCounts { - vcs_changes: self.delete_unseen_vcs_change_batch()?, + vcs_changes: self.delete_unseen_vcs_change_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_ARTIFACTS => ProviderFileReconciliationCounts { - artifacts: self.delete_unseen_artifact_batch()?, + artifacts: self.delete_unseen_artifact_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_HISTORY_RECORD_TAGS => ProviderFileReconciliationCounts { - history_record_tags: self.delete_history_record_tag_batch()?, + history_record_tags: self.delete_history_record_tag_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_RECORD_EDGES => ProviderFileReconciliationCounts { - record_edges: self.delete_unseen_batch("record_edge", "record_edges")?, + record_edges: self.delete_unseen_batch( + &replacement_id, + "record_edge", + "record_edges", + )?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_HISTORY_RECORDS => ProviderFileReconciliationCounts { - history_records: self.delete_unseen_history_record_batch()?, + history_records: self.delete_unseen_history_record_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_VCS_WORKSPACES => ProviderFileReconciliationCounts { - vcs_workspaces: self.delete_unseen_vcs_workspace_batch()?, + vcs_workspaces: self.delete_unseen_vcs_workspace_batch(&replacement_id)?, ..ProviderFileReconciliationCounts::default() }, CLEANUP_PHASE_AUDIT_LOG => ProviderFileReconciliationCounts { - audit_log_entries: self.delete_unseen_batch("audit_log", "audit_log")?, + audit_log_entries: self.delete_unseen_batch( + &replacement_id, + "audit_log", + "audit_log", + )?, ..ProviderFileReconciliationCounts::default() }, _ => unreachable!(), }; - Ok(ReconciliationBatch { + let batch = ReconciliationBatch { visited: scan.visited, phase_complete: scan.phase_complete, source_cursor: scan.source_cursor, entity_cursor: scan.entity_cursor, removed, + }; + self.begin_prior_seen_cleanup_if_needed(phase, &replacement_id, batch) + } + + fn begin_prior_seen_cleanup_if_needed( + &self, + phase: i64, + replacement_id: &str, + mut batch: ReconciliationBatch, + ) -> Result { + if phase != CLEANUP_PHASE_AUDIT_LOG || !batch.phase_complete { + return Ok(batch); + } + let next_cursor = if self + .provider_file_publication_seen_kind_exists(replacement_id, PRIOR_HISTORY_RECORD_KIND)? + { + Some(PRIOR_HISTORY_RECORD_CURSOR) + } else if self + .provider_file_publication_seen_kind_exists(replacement_id, PRIOR_CAPTURE_SOURCE_KIND)? + { + Some(PRIOR_CAPTURE_SOURCE_CURSOR) + } else { + None + }; + if let Some(next_cursor) = next_cursor { + batch.phase_complete = false; + batch.source_cursor = Some(next_cursor.to_owned()); + batch.entity_cursor = None; + } + Ok(batch) + } + + fn reconcile_prior_seen_batch( + &self, + replacement_id: &str, + prior_cursor: &str, + entity_cursor: Option<&str>, + limit: usize, + ) -> Result { + let prior_kind = match prior_cursor { + PRIOR_HISTORY_RECORD_CURSOR => PRIOR_HISTORY_RECORD_KIND, + PRIOR_CAPTURE_SOURCE_CURSOR => PRIOR_CAPTURE_SOURCE_KIND, + _ => return Err(StoreError::InvalidProviderFilePublicationScope), + }; + let sqlite_limit = i64::try_from(limit.checked_add(1).ok_or( + StoreError::ProviderFileReconciliationLimitOutOfRange { + value: limit, + max: PROVIDER_FILE_RECONCILIATION_MAX_ROWS, + }, + )?) + .map_err(|_| StoreError::ProviderFileReconciliationLimitOutOfRange { + value: limit, + max: PROVIDER_FILE_RECONCILIATION_MAX_ROWS, + })?; + let mut stmt = self.conn.prepare_cached(&format!( + "SELECT entity_id FROM {STAGING_SEEN_TABLE} \ + WHERE replacement_id = ?1 AND entity_kind = ?2 \ + AND (?3 IS NULL OR entity_id > ?3) \ + ORDER BY entity_id LIMIT ?4" + ))?; + let rows = stmt.query_map( + params![replacement_id, prior_kind, entity_cursor, sqlite_limit], + |row| row.get::<_, String>(0), + )?; + let mut ids = rows.collect::>>()?; + let complete = ids.len() <= limit; + if !complete { + ids.pop(); + } + if ids.is_empty() { + let next_cursor = if prior_cursor == PRIOR_HISTORY_RECORD_CURSOR + && self.provider_file_publication_seen_kind_exists( + replacement_id, + PRIOR_CAPTURE_SOURCE_KIND, + )? { + Some(PRIOR_CAPTURE_SOURCE_CURSOR.to_owned()) + } else { + None + }; + return Ok(ReconciliationBatch { + visited: 0, + phase_complete: next_cursor.is_none(), + source_cursor: next_cursor, + entity_cursor: None, + removed: ProviderFileReconciliationCounts::default(), + }); + } + + self.conn.execute( + &format!("DELETE FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1"), + params![replacement_id], + )?; + { + let mut insert = self.conn.prepare_cached(&format!( + "INSERT INTO {STAGING_BATCH_TABLE} \ + (replacement_id, source_id, entity_id) VALUES (?1, ?2, ?3)" + ))?; + for id in &ids { + insert.execute(params![replacement_id, prior_cursor, id])?; + } + } + let removed = if prior_kind == PRIOR_HISTORY_RECORD_KIND { + ProviderFileReconciliationCounts { + history_records: self.delete_unseen_history_record_batch(replacement_id)?, + ..ProviderFileReconciliationCounts::default() + } + } else { + self.delete_unseen_capture_source_batch(replacement_id)?; + ProviderFileReconciliationCounts::default() + }; + self.conn.execute( + &format!( + "DELETE FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1 \ + AND entity_kind = ?2 AND entity_id IN ( \ + SELECT entity_id FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1 \ + )" + ), + params![replacement_id, prior_kind], + )?; + + let last_id = ids.last().cloned(); + let next_cursor = if complete + && prior_cursor == PRIOR_HISTORY_RECORD_CURSOR + && self.provider_file_publication_seen_kind_exists( + replacement_id, + PRIOR_CAPTURE_SOURCE_KIND, + )? { + Some(PRIOR_CAPTURE_SOURCE_CURSOR.to_owned()) + } else if complete { + None + } else { + Some(prior_cursor.to_owned()) + }; + Ok(ReconciliationBatch { + visited: ids.len(), + phase_complete: complete && next_cursor.is_none(), + source_cursor: next_cursor, + entity_cursor: (!complete).then_some(last_id).flatten(), + removed, }) } + fn provider_file_publication_seen_kind_exists( + &self, + replacement_id: &str, + entity_kind: &str, + ) -> Result { + self.conn + .query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} \ + WHERE replacement_id = ?1 AND entity_kind = ?2)" + ), + params![replacement_id, entity_kind], + |row| row.get(0), + ) + .map_err(StoreError::from) + } + fn reconciliation_batch_rows( &self, + replacement_id: &str, phase: i64, source_cursor: Option<&str>, entity_cursor: Option<&str>, @@ -114,16 +314,22 @@ impl Store { Some(source_id) => self .conn .query_row( - &format!("SELECT id FROM {STAGING_SCHEMA}.prior_sources WHERE id = ?1"), - params![source_id], + &format!( + "SELECT source_id FROM {STAGING_PRIOR_SOURCES_TABLE} \ + WHERE replacement_id = ?1 AND source_id = ?2" + ), + params![replacement_id, source_id], |row| row.get::<_, String>(0), ) .optional()?, None => self .conn .query_row( - &format!("SELECT id FROM {STAGING_SCHEMA}.prior_sources ORDER BY id LIMIT 1"), - [], + &format!( + "SELECT source_id FROM {STAGING_PRIOR_SOURCES_TABLE} \ + WHERE replacement_id = ?1 ORDER BY source_id LIMIT 1" + ), + params![replacement_id], |row| row.get::<_, String>(0), ) .optional()?, @@ -182,9 +388,11 @@ impl Store { .conn .query_row( &format!( - "SELECT id FROM {STAGING_SCHEMA}.prior_sources WHERE id > ?1 ORDER BY id LIMIT 1" + "SELECT source_id FROM {STAGING_PRIOR_SOURCES_TABLE} \ + WHERE replacement_id = ?1 AND source_id > ?2 \ + ORDER BY source_id LIMIT 1" ), - params![¤t_source], + params![replacement_id, ¤t_source], |row| row.get::<_, String>(0), ) .optional()?; diff --git a/crates/ctx-history-store/src/provider_files/staging_recovery.rs b/crates/ctx-history-store/src/provider_files/staging_recovery.rs index db3be70e..89cf3cae 100644 --- a/crates/ctx-history-store/src/provider_files/staging_recovery.rs +++ b/crates/ctx-history-store/src/provider_files/staging_recovery.rs @@ -1,4 +1,284 @@ impl Store { + fn reset_provider_file_publication_staging_slice( + &self, + scope: &ProviderFilePublicationScope, + marker: &mut ReplacementMarker, + max_rows: usize, + ) -> Result { + let replacement_id = scope.scope_id.to_string(); + let mut processed = 0usize; + while processed < max_rows { + let remaining = max_rows - processed; + let cursor = marker + .preparation_cursor + .as_deref() + .ok_or(StoreError::InvalidProviderFilePublicationScope)?; + let (entity_kind, prior_kind, next_cursor) = match cursor { + RETIREMENT_RESET_BATCH_CURSOR => { + (None, None, RETIREMENT_RESET_HISTORY_RECORD_CURSOR) + } + RETIREMENT_RESET_HISTORY_RECORD_CURSOR => ( + Some("history_record"), + Some(PRIOR_HISTORY_RECORD_KIND), + RETIREMENT_RESET_CAPTURE_SOURCE_CURSOR, + ), + RETIREMENT_RESET_CAPTURE_SOURCE_CURSOR => ( + Some(CURRENT_CAPTURE_SOURCE_KIND), + Some(PRIOR_CAPTURE_SOURCE_KIND), + RETIREMENT_RESET_SEEN_CURSOR, + ), + RETIREMENT_RESET_SEEN_CURSOR => (None, None, ""), + _ => return Err(StoreError::InvalidProviderFilePublicationScope), + }; + + let rows = if cursor == RETIREMENT_RESET_BATCH_CURSOR { + let mut stmt = self.conn.prepare_cached(&format!( + "SELECT source_id, entity_id FROM {STAGING_BATCH_TABLE} \ + WHERE replacement_id = ?1 ORDER BY source_id, entity_id LIMIT ?2" + ))?; + let rows = stmt + .query_map( + params![&replacement_id, capped_i64(remaining as u64)], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + )? + .collect::>>()?; + for (source_id, entity_id) in &rows { + self.conn.execute( + &format!( + "DELETE FROM {STAGING_BATCH_TABLE} WHERE replacement_id = ?1 \ + AND source_id = ?2 AND entity_id = ?3" + ), + params![&replacement_id, source_id, entity_id], + )?; + } + rows.len() + } else if let (Some(entity_kind), Some(prior_kind)) = (entity_kind, prior_kind) { + let mut stmt = self.conn.prepare_cached(&format!( + "SELECT entity_id FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1 \ + AND entity_kind = ?2 ORDER BY entity_id LIMIT ?3" + ))?; + let ids = stmt + .query_map( + params![&replacement_id, entity_kind, capped_i64(remaining as u64)], + |row| row.get::<_, String>(0), + )? + .collect::>>()?; + for id in &ids { + self.conn.execute( + &format!( + "INSERT OR IGNORE INTO {STAGING_SEEN_TABLE} \ + (replacement_id, entity_kind, entity_id) VALUES (?1, ?2, ?3)" + ), + params![&replacement_id, prior_kind, id], + )?; + self.conn.execute( + &format!( + "DELETE FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1 \ + AND entity_kind = ?2 AND entity_id = ?3" + ), + params![&replacement_id, entity_kind, id], + )?; + } + ids.len() + } else { + let mut stmt = self.conn.prepare_cached(&format!( + "SELECT entity_kind, entity_id FROM {STAGING_SEEN_TABLE} \ + WHERE replacement_id = ?1 AND entity_kind NOT IN (?2, ?3) \ + ORDER BY entity_kind, entity_id LIMIT ?4" + ))?; + let rows = stmt + .query_map( + params![ + &replacement_id, + PRIOR_HISTORY_RECORD_KIND, + PRIOR_CAPTURE_SOURCE_KIND, + capped_i64(remaining as u64) + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + )? + .collect::>>()?; + for (entity_kind, entity_id) in &rows { + self.conn.execute( + &format!( + "DELETE FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1 \ + AND entity_kind = ?2 AND entity_id = ?3" + ), + params![&replacement_id, entity_kind, entity_id], + )?; + } + rows.len() + }; + processed = processed.saturating_add(rows); + if rows == remaining { + break; + } + marker.preparation_cursor = (!next_cursor.is_empty()).then(|| next_cursor.to_owned()); + if next_cursor.is_empty() { + break; + } + } + Ok(processed) + } + + fn reset_provider_file_publication_staging( + &self, + scope: &ProviderFilePublicationScope, + ) -> Result<()> { + let replacement_id = scope.scope_id.to_string(); + for table in [STAGING_BATCH_TABLE, STAGING_PRIOR_SOURCES_TABLE] { + self.conn.execute( + &format!("DELETE FROM {table} WHERE replacement_id = ?1"), + params![&replacement_id], + )?; + } + for (current_kind, prior_kind) in [ + ("history_record", PRIOR_HISTORY_RECORD_KIND), + (CURRENT_CAPTURE_SOURCE_KIND, PRIOR_CAPTURE_SOURCE_KIND), + ] { + self.conn.execute( + &format!( + "INSERT OR IGNORE INTO {STAGING_SEEN_TABLE} \ + (replacement_id, entity_kind, entity_id) \ + SELECT replacement_id, ?2, entity_id FROM {STAGING_SEEN_TABLE} \ + WHERE replacement_id = ?1 AND entity_kind = ?3" + ), + params![&replacement_id, prior_kind, current_kind], + )?; + } + self.conn.execute( + &format!( + "DELETE FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1 \ + AND entity_kind NOT IN (?2, ?3)" + ), + params![ + &replacement_id, + PRIOR_HISTORY_RECORD_KIND, + PRIOR_CAPTURE_SOURCE_KIND + ], + )?; + let changed = self.conn.execute( + r#" + UPDATE provider_file_publications + SET staging_initialized = 1, + preparation_complete = CASE WHEN ?2 THEN 0 ELSE 1 END, + preparation_cursor = NULL, + cleanup_phase = 0, + cleanup_source_cursor = NULL, + cleanup_entity_cursor = NULL, + removed_artifacts = 0, + removed_summaries = 0, + removed_history_record_links = 0, + removed_history_records = 0, + removed_history_record_tags = 0, + removed_record_edges = 0, + removed_audit_log_entries = 0, + removed_vcs_workspaces = 0, + removed_vcs_changes = 0, + removed_events = 0, + removed_runs = 0, + removed_files_touched = 0, + removed_session_edges = 0, + tombstoned_sessions = 0, + completion_payload_json = NULL + WHERE replacement_id = ?1 + "#, + params![&replacement_id, scope.tracks_prior_material], + )?; + if changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + Ok(()) + } + + fn initialize_provider_file_publication_retirement( + &self, + scope: &mut ProviderFilePublicationScope, + ) -> Result<()> { + let retirement_started: bool = self.conn.query_row( + "SELECT retirement_started FROM provider_file_publications WHERE replacement_id = ?1", + params![scope.scope_id.to_string()], + |row| row.get(0), + )?; + if retirement_started { + let marker = self.load_replacement_marker(scope)?; + let replacement_id = scope.scope_id.to_string(); + let staged_state_exists: bool = self.conn.query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} WHERE replacement_id = ?1) \ + OR EXISTS (SELECT 1 FROM {STAGING_PRIOR_SOURCES_TABLE} WHERE replacement_id = ?1)" + ), + params![&replacement_id], + |row| row.get(0), + )?; + let progress_without_state = marker.cleanup_phase != CLEANUP_PHASE_COMPLETE + && !staged_state_exists + && (marker.preparation_cursor.is_some() + || marker.preparation_complete + || marker.cleanup_phase != CLEANUP_PHASE_LINKS + || marker.source_cursor.is_some() + || marker.entity_cursor.is_some() + || marker.counts != ProviderFileReconciliationCounts::default()); + if progress_without_state && scope.tracks_prior_material { + let changed = self.conn.execute( + r#" + UPDATE provider_file_publications + SET preparation_complete = 0, preparation_cursor = ?2, + cleanup_phase = 0, cleanup_source_cursor = NULL, + cleanup_entity_cursor = NULL, removed_artifacts = 0, + removed_summaries = 0, removed_history_record_links = 0, + removed_history_records = 0, removed_history_record_tags = 0, + removed_record_edges = 0, removed_audit_log_entries = 0, + removed_vcs_workspaces = 0, removed_vcs_changes = 0, + removed_events = 0, removed_runs = 0, removed_files_touched = 0, + removed_session_edges = 0, tombstoned_sessions = 0 + WHERE replacement_id = ?1 AND retirement_started = 1 + "#, + params![&replacement_id, RETIREMENT_RESET_BATCH_CURSOR], + )?; + if changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + } + scope.reuse_staging_state = true; + return Ok(()); + } + + scope.kind = ProviderFilePublicationKind::Replacement; + // A mutated publication may own material even when its durable staging + // was created by an older schema. Preparation discovers that material + // in bounded slices, so retirement never needs an eager owner scan. + scope.tracks_prior_material = true; + let changed = self.conn.execute( + r#" + UPDATE provider_file_publications + SET publication_kind = 'replacement', tracks_prior_material = ?2, + retirement_started = 1, staging_initialized = 1, + preparation_complete = CASE WHEN ?2 THEN 0 ELSE 1 END, + preparation_cursor = CASE WHEN ?2 THEN ?3 ELSE NULL END, + cleanup_phase = 0, cleanup_source_cursor = NULL, + cleanup_entity_cursor = NULL, removed_artifacts = 0, + removed_summaries = 0, removed_history_record_links = 0, + removed_history_records = 0, removed_history_record_tags = 0, + removed_record_edges = 0, removed_audit_log_entries = 0, + removed_vcs_workspaces = 0, removed_vcs_changes = 0, + removed_events = 0, removed_runs = 0, removed_files_touched = 0, + removed_session_edges = 0, tombstoned_sessions = 0, + completion_payload_json = NULL + WHERE replacement_id = ?1 AND retirement_started = 0 + "#, + params![ + scope.scope_id.to_string(), + scope.tracks_prior_material, + RETIREMENT_RESET_BATCH_CURSOR + ], + )?; + if changed != 1 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + scope.reuse_staging_state = true; + Ok(()) + } + fn reclaim_orphaned_provider_staging( &self, scope: &ProviderFilePublicationScope, @@ -12,7 +292,6 @@ impl Store { &scope.source_path, ); let prefix = format!("{STAGING_DIR_PREFIX}-{lock_owner_id}-"); - let current = format!("{prefix}{}", scope.staging_id); let root = self.store_identity.private_root(); let mut reclaimed = 0usize; for entry in fs::read_dir(&root)? { @@ -24,7 +303,7 @@ impl Store { let Some(name) = name.to_str() else { continue; }; - if !name.starts_with(&prefix) || name == current { + if !name.starts_with(&prefix) { continue; } let metadata = fs::symlink_metadata(entry.path())?; @@ -84,56 +363,6 @@ impl Store { }); } - let attached = self - .provider_file_publication - .borrow() - .as_ref() - .is_some_and(|active| active.attached); - if attached { - if self - .conn - .execute_batch(&format!("DETACH DATABASE {STAGING_SCHEMA}")) - .is_err() - { - return Err(ProviderFileMaintenanceWarning::StagingCleanupDeferred { - publication_id: scope_id.to_string(), - operation: "detach", - }); - } - if let Some(active) = self.provider_file_publication.borrow_mut().as_mut() { - active.attached = false; - } - } - let staging_path = self - .provider_file_publication - .borrow() - .as_ref() - .and_then(|active| active.staging_path.clone()); - if let Some(path) = &staging_path { - if let Err(error) = fs::remove_file(path) { - if error.kind() != std::io::ErrorKind::NotFound { - return Err(ProviderFileMaintenanceWarning::StagingCleanupDeferred { - publication_id: scope_id.to_string(), - operation: "remove-file", - }); - } - } - } - let staging_dir_path = self - .provider_file_publication - .borrow() - .as_ref() - .and_then(|active| active.staging_dir_path.clone()); - if let Some(path) = &staging_dir_path { - if let Err(error) = fs::remove_dir(path) { - if error.kind() != std::io::ErrorKind::NotFound { - return Err(ProviderFileMaintenanceWarning::StagingCleanupDeferred { - publication_id: scope_id.to_string(), - operation: "remove-directory", - }); - } - } - } self.provider_file_publication.replace(None); Ok(()) } @@ -214,4 +443,17 @@ impl Store { source_path, ) } + + fn provider_file_publication_has_staged_material(&self, scope_id: Uuid) -> Result { + self.conn + .query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {STAGING_SEEN_TABLE} \ + WHERE replacement_id = ?1)" + ), + params![scope_id.to_string()], + |row| row.get(0), + ) + .map_err(StoreError::from) + } } diff --git a/crates/ctx-history-store/src/provider_files/tests.rs b/crates/ctx-history-store/src/provider_files/tests.rs index 958818a9..0f821a0f 100644 --- a/crates/ctx-history-store/src/provider_files/tests.rs +++ b/crates/ctx-history-store/src/provider_files/tests.rs @@ -1,8 +1,8 @@ use chrono::{DateTime, Utc}; use ctx_history_core::{ AgentType, CaptureProvider, CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, - EntityTimestamps, Event, EventRole, EventType, Fidelity, Session, SessionStatus, SyncMetadata, - SyncState, Visibility, + Confidence, EntityTimestamps, Event, EventRole, EventType, Fidelity, Session, SessionEdge, + SessionEdgeType, SessionStatus, SyncMetadata, SyncState, Visibility, }; use rusqlite::{params, OptionalExtension}; use serde_json::json; @@ -23,6 +23,45 @@ const FORMAT: &str = "claude_projects_jsonl_tree"; const MATERIAL_FORMAT: &str = "claude_projects_jsonl"; const ROOT: &str = "/history/claude/projects"; const PATH_A: &str = "/history/claude/projects/a.jsonl"; +const PATH_B: &str = "/history/claude/projects/b.jsonl"; +const WRONG_CATALOG_FORMAT: &str = "claude_other_catalog"; +const WRONG_SOURCE_IMPORT_FORMAT: &str = "claude_other_source_import"; + +#[derive(Debug, Clone, Copy)] +enum RetirementInventoryFamily { + Catalog, + SourceImport, +} + +impl RetirementInventoryFamily { + fn inventory_source_format(self) -> &'static str { + match self { + Self::Catalog => MATERIAL_FORMAT, + Self::SourceImport => FORMAT, + } + } + + fn wrong_source_format(self) -> &'static str { + match self { + Self::Catalog => WRONG_CATALOG_FORMAT, + Self::SourceImport => WRONG_SOURCE_IMPORT_FORMAT, + } + } + + fn inventory_table(self) -> &'static str { + match self { + Self::Catalog => "catalog_sessions", + Self::SourceImport => "source_import_files", + } + } + + fn opposite_inventory_table(self) -> &'static str { + match self { + Self::Catalog => "source_import_files", + Self::SourceImport => "catalog_sessions", + } + } +} fn source_file(size: u64, modified_at_ms: i64) -> SourceImportFile { SourceImportFile { @@ -55,7 +94,7 @@ fn catalog_file(size: u64, modified_at_ms: i64) -> CatalogSession { file_modified_at_ms: modified_at_ms, import_revision: 7, cataloged_at_ms: modified_at_ms + 1, - metadata: json!({}), + metadata: json!({"file_observation_token_v1": "test-catalog-token"}), } } @@ -106,6 +145,28 @@ fn source_outcome<'a>( } } +fn catalog_observation( + catalog: &CatalogSession, + inventory_generation: u64, + indexed_at_ms: i64, +) -> ProviderFileInventoryObservation<'_> { + ProviderFileInventoryObservation::ObservedCatalog { + source_format: &catalog.source_format, + update: CatalogSourceIndexUpdate { + source_root: &catalog.source_root, + source_path: &catalog.source_path, + file_size_bytes: catalog.file_size_bytes, + file_modified_at_ms: catalog.file_modified_at_ms, + import_revision: catalog.import_revision, + inventory_generation, + file_sha256: None, + event_count: None, + indexed_at_ms, + }, + metadata: &catalog.metadata, + } +} + fn checkpoint_for_catalog( catalog: &CatalogSession, size: u64, diff --git a/crates/ctx-history-store/src/provider_files/tests/archive_visibility.rs b/crates/ctx-history-store/src/provider_files/tests/archive_visibility.rs index 1ec41c7d..3134f691 100644 --- a/crates/ctx-history-store/src/provider_files/tests/archive_visibility.rs +++ b/crates/ctx-history-store/src/provider_files/tests/archive_visibility.rs @@ -85,6 +85,7 @@ fn archive_export_uses_one_visibility_snapshot_across_publication_commit() { assert!(store.list_events().unwrap().is_empty()); assert!(store.list_files_touched().unwrap().is_empty()); } + #[test] fn ancillary_owner_material_is_hidden_then_removed_without_archive_or_cache_leaks() { let temp = tempdir().unwrap(); @@ -291,6 +292,7 @@ fn ancillary_owner_material_is_hidden_then_removed_without_archive_or_cache_leak } assert!(store.search_records("hidden", 10).unwrap().is_empty()); } + #[test] fn replacement_tracks_source_less_vcs_change_through_its_owned_workspace() { let temp = tempdir().unwrap(); @@ -360,6 +362,7 @@ fn replacement_tracks_source_less_vcs_change_through_its_owned_workspace() { assert!(row_exists(&store, "vcs_workspaces", workspace)); assert_eq!(store.list_vcs_changes().unwrap()[0].id, retained); } + #[test] fn cross_process_vcs_hijack_is_rejected_and_scoped_source_less_upsert_is_valid() { let temp = tempdir().unwrap(); @@ -503,6 +506,7 @@ fn omitted_session_releases_transcript_artifact_before_artifact_reconciliation() .unwrap(); assert_eq!(session_state, (None, Some(100))); } + #[test] fn pending_catalog_owner_is_hidden_from_ctx_sources_view() { let temp = tempdir().unwrap(); @@ -523,7 +527,7 @@ fn pending_catalog_owner_is_hidden_from_ctx_sources_view() { file_modified_at_ms: 100, import_revision: 1, cataloged_at_ms: 101, - metadata: json!({}), + metadata: json!({"file_observation_token_v1": "archive-catalog-token"}), }; let generation = store .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) @@ -544,9 +548,10 @@ fn pending_catalog_owner_is_hidden_from_ctx_sources_view() { }; let outcome = ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::Indexed, error: None, @@ -619,6 +624,7 @@ fn pending_catalog_owner_is_hidden_from_ctx_sources_view() { ) .unwrap()); } + #[test] fn pending_source_import_owner_is_hidden_from_pending_list_and_counts() { let temp = tempdir().unwrap(); diff --git a/crates/ctx-history-store/src/provider_files/tests/batching_and_scaling.rs b/crates/ctx-history-store/src/provider_files/tests/batching_and_scaling.rs index 5753077e..d57d2997 100644 --- a/crates/ctx-history-store/src/provider_files/tests/batching_and_scaling.rs +++ b/crates/ctx-history-store/src/provider_files/tests/batching_and_scaling.rs @@ -1,5 +1,5 @@ #[test] -fn large_batched_seen_scope_spills_outside_work_database() { +fn large_batched_seen_scope_is_stored_in_main_database() { const EVENT_COUNT: u128 = 5_000; const BATCH: u128 = 100; @@ -26,8 +26,6 @@ fn large_batched_seen_scope_spills_outside_work_database() { ); } store.commit_batch().unwrap(); - store.checkpoint_wal_truncate().unwrap(); - let main_before = main_database_footprint(&store, &path); let outcome = source_outcome(&file, generation, 120); let scope = store .begin_provider_file_publication( @@ -39,51 +37,16 @@ fn large_batched_seen_scope_spills_outside_work_database() { ) .unwrap(); assert!(scope.tracks_prior_material); - assert_eq!( - pragma_i64(&store, "PRAGMA provider_replacement_stage.cache_size"), - -8192 - ); - assert!(pragma_i64(&store, "PRAGMA provider_replacement_stage.page_count") <= 16); - let prior_entity_table: bool = store + let attached_staging: bool = store .conn .query_row( - "SELECT EXISTS (SELECT 1 FROM provider_replacement_stage.sqlite_master WHERE type = 'table' AND name = 'prior_entities')", + "SELECT EXISTS (SELECT 1 FROM pragma_database_list WHERE name = 'provider_replacement_stage')", [], |row| row.get(0), ) .unwrap(); - assert!(!prior_entity_table); - let staging_dir = store - .provider_file_publication - .borrow() - .as_ref() - .and_then(|active| active.staging_dir_path.clone()) - .unwrap(); - assert!(staging_dir.is_dir()); - #[cfg(unix)] - assert!(store - .provider_file_publication - .borrow() - .as_ref() - .unwrap() - .staging_path - .is_none()); - assert_eq!( - store - .provider_file_publication - .borrow() - .as_ref() - .and_then(|active| active.staging_file_mode), - Some(0o600) - ); - assert_eq!( - store - .provider_file_publication - .borrow() - .as_ref() - .and_then(|active| active.staging_dir_mode), - Some(0o700) - ); + assert!(!attached_staging); + assert!(main_table_exists(&store, "provider_file_publication_seen")); for start in (0..EVENT_COUNT).step_by(BATCH as usize) { store.begin_immediate_batch().unwrap(); @@ -96,12 +59,6 @@ fn large_batched_seen_scope_spills_outside_work_database() { } assert_eq!(staged_seen_count(&store), EVENT_COUNT as i64); - assert!(pragma_i64(&store, "PRAGMA provider_replacement_stage.page_count") > 1); - let main_after_staging = main_database_footprint(&store, &path); - assert_eq!(main_after_staging.0, main_before.0); - assert_eq!(main_after_staging.1, main_before.1); - assert_eq!(main_after_staging.2, main_before.2); - assert!(main_after_staging.3 <= 64 * 1024); reconcile_all(&store, &scope, 127); let counts = store .finalize_provider_file_publication( @@ -117,8 +74,17 @@ fn large_batched_seen_scope_spills_outside_work_database() { .unwrap(); assert_eq!(counts.reconciliation.events, 0); assert!(store.provider_file_publication.borrow().is_none()); - assert!(!staging_dir.exists()); + let staged_seen: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM provider_file_publication_seen", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(staged_seen, 0); } + #[test] fn replacement_preparation_pages_prior_source_identity_snapshot_without_a_total_cap() { const SOURCE_COUNT: u128 = 4_097; @@ -174,6 +140,275 @@ fn replacement_preparation_pages_prior_source_identity_snapshot_without_a_total_ assert_eq!(staged_prior_source_count(&store), SOURCE_COUNT as i64); store.abandon_provider_file_publication(scope).unwrap(); } + +#[test] +fn publication_phases_resume_after_single_1024_row_slices_without_advancing_outcome() { + const ROW_COUNT: u128 = 1_025; + const SLICE_ROWS: usize = 1_024; + + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let original = source_file(10, 90); + let changed = source_file(20, 100); + let old_checkpoint = checkpoint(10, 3, "unix:2049:phased", 95); + let new_checkpoint = checkpoint(20, 5, "unix:2049:phased", 150); + let completion = ProviderFilePublicationCompletion { + version: 1, + payload: json!({ + "summary": {"imported": 0, "skipped": 1025}, + "checkpoint": {"committed_byte_offset": 20} + }), + }; + let changed_generation; + let pending_indexed_state; + + { + let store = Store::open(&path).unwrap(); + let original_generation = store + .allocate_source_import_inventory_generation(original.provider, &original.source_root) + .unwrap(); + store + .upsert_source_import_files(original_generation, std::slice::from_ref(&original)) + .unwrap(); + store + .upsert_provider_file_checkpoint( + source_outcome(&original, original_generation, 95), + &old_checkpoint, + ) + .unwrap(); + store.begin_immediate_batch().unwrap(); + for index in 0..ROW_COUNT { + let source_id = Uuid::from_u128(150_000 + index); + insert_capture_source(&store, source_id, PATH_A, &format!("phased-source-{index}")); + insert_raw_event( + &store, + Uuid::from_u128(160_000 + index), + 160_000 + index as i64, + source_id, + "stale phased event", + ); + } + store.commit_batch().unwrap(); + changed_generation = store + .allocate_source_import_inventory_generation(changed.provider, &changed.source_root) + .unwrap(); + store + .upsert_source_import_files(changed_generation, std::slice::from_ref(&changed)) + .unwrap(); + pending_indexed_state = store + .conn + .query_row( + "SELECT indexed_status, indexed_file_size_bytes, \ + indexed_file_modified_at_ms \ + FROM source_import_files WHERE provider = 'claude' \ + AND source_root = ?1 AND source_path = ?2", + params![ROOT, PATH_A], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + }, + ) + .unwrap(); + let scope = store + .begin_provider_file_publication( + changed.provider, + source_outcome(&changed, changed_generation, 150).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::Preparing + ); + let preparation = store + .prepare_provider_file_publication_slice(&scope, SLICE_ROWS) + .unwrap(); + assert_eq!(preparation.source_ids_staged, SLICE_ROWS); + assert!(!preparation.complete); + assert_eq!(staged_prior_source_count(&store), SLICE_ROWS as i64); + store.abandon_provider_file_publication(scope).unwrap(); + } + + { + let store = Store::open(&path).unwrap(); + let outcome = source_outcome(&changed, changed_generation, 150); + let scope = store + .begin_provider_file_publication( + changed.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 110, + ) + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::Preparing + ); + let preparation = store + .prepare_provider_file_publication_slice(&scope, SLICE_ROWS) + .unwrap(); + assert_eq!(preparation.source_ids_staged, 1); + assert!(preparation.complete); + assert_eq!(staged_prior_source_count(&store), ROW_COUNT as i64); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::Importing + ); + store + .stage_provider_file_publication_completion(&scope, &completion) + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::Reconciling + ); + store.abandon_provider_file_publication(scope).unwrap(); + } + + { + let store = Store::open(&path).unwrap(); + let outcome = source_outcome(&changed, changed_generation, 150); + let scope = store + .begin_provider_file_publication( + changed.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 120, + ) + .unwrap(); + assert_eq!( + store + .load_provider_file_publication_completion(&scope) + .unwrap(), + Some(completion.clone()) + ); + let reconciliation = store + .reconcile_provider_file_publication_slice(&scope, SLICE_ROWS) + .unwrap(); + assert_eq!(reconciliation.rows_scanned, SLICE_ROWS); + assert_eq!( + reconciliation.counts, + ProviderFileReconciliationCounts::default() + ); + assert!(!reconciliation.complete); + let cleanup_cursor: Option = store + .conn + .query_row( + "SELECT cleanup_source_cursor FROM provider_file_publications", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(cleanup_cursor.is_some()); + assert_eq!( + store + .provider_file_checkpoint(old_checkpoint.key()) + .unwrap(), + Some(old_checkpoint.clone()) + ); + let visible_indexed_state: (String, Option, Option) = store + .conn + .query_row( + "SELECT indexed_status, indexed_file_size_bytes, \ + indexed_file_modified_at_ms \ + FROM source_import_files WHERE provider = 'claude' \ + AND source_root = ?1 AND source_path = ?2", + params![ROOT, PATH_A], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(visible_indexed_state, pending_indexed_state); + store.abandon_provider_file_publication(scope).unwrap(); + } + + let store = Store::open(&path).unwrap(); + let outcome = source_outcome(&changed, changed_generation, 150); + let scope = store + .begin_provider_file_publication( + changed.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 130, + ) + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::Reconciling + ); + assert_eq!( + store + .load_provider_file_publication_completion(&scope) + .unwrap(), + Some(completion) + ); + let reconciliation = loop { + let reconciliation = store + .reconcile_provider_file_publication_slice(&scope, SLICE_ROWS) + .unwrap(); + assert!(reconciliation.rows_scanned <= SLICE_ROWS); + if reconciliation.complete { + break reconciliation; + } + }; + assert_eq!(reconciliation.counts.events, ROW_COUNT as usize); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::ReadyToFinalize + ); + assert_eq!( + store + .provider_file_checkpoint(old_checkpoint.key()) + .unwrap(), + Some(old_checkpoint) + ); + let visible_indexed_state: (String, Option, Option) = store + .conn + .query_row( + "SELECT indexed_status, indexed_file_size_bytes, indexed_file_modified_at_ms \ + FROM source_import_files WHERE provider = 'claude' \ + AND source_root = ?1 AND source_path = ?2", + params![ROOT, PATH_A], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(visible_indexed_state, pending_indexed_state); + + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(Some(&new_checkpoint)), + ) + .unwrap(); + assert_eq!( + store + .provider_file_checkpoint(new_checkpoint.key()) + .unwrap(), + Some(new_checkpoint) + ); + let finalized_indexed_state: (String, Option, Option) = store + .conn + .query_row( + "SELECT indexed_status, indexed_file_size_bytes, indexed_file_modified_at_ms \ + FROM source_import_files WHERE provider = 'claude' \ + AND source_root = ?1 AND source_path = ?2", + params![ROOT, PATH_A], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + finalized_indexed_state, + ("indexed".into(), Some(20), Some(100)) + ); +} + #[test] fn reconciliation_queries_owner_indexes_without_visiting_unrelated_corpus_rows() { const UNRELATED_EVENTS: u128 = 5_000; @@ -222,7 +457,13 @@ fn reconciliation_queries_owner_indexes_without_visiting_unrelated_corpus_rows() .unwrap(); prepare_all(&store, &scope, 1); let scan = store - .reconciliation_batch_rows(CLEANUP_PHASE_EVENTS, None, None, 1) + .reconciliation_batch_rows( + &scope.scope_id.to_string(), + CLEANUP_PHASE_EVENTS, + None, + None, + 1, + ) .unwrap(); assert_eq!(scan.visited, 1); assert_eq!(scan.owned_entity_ids, vec![owner_event.to_string()]); @@ -259,6 +500,7 @@ fn reconciliation_queries_owner_indexes_without_visiting_unrelated_corpus_rows() } store.abandon_provider_file_publication(scope).unwrap(); } + #[test] fn large_owner_tiny_slices_keep_candidate_work_linear_across_interrupted_retry() { const EVENT_COUNT: usize = 600; @@ -302,6 +544,7 @@ fn large_owner_tiny_slices_keep_candidate_work_linear_across_interrupted_retry() ) .unwrap(); prepare_all(&store, &scope, 1); + stage_test_completion(&store, &scope); for _ in 0..FIRST_ATTEMPT_SLICES { let progress = store .reconcile_provider_file_publication_slice(&scope, 1) diff --git a/crates/ctx-history-store/src/provider_files/tests/checkpoints_and_append.rs b/crates/ctx-history-store/src/provider_files/tests/checkpoints_and_append.rs index 1848c597..951dd1de 100644 --- a/crates/ctx-history-store/src/provider_files/tests/checkpoints_and_append.rs +++ b/crates/ctx-history-store/src/provider_files/tests/checkpoints_and_append.rs @@ -54,8 +54,9 @@ fn checkpoint_round_trip_preserves_exact_boundary_data_and_version() { ); } } + #[test] -fn stale_generation_is_typed_and_never_advances_checkpoint() { +fn exact_observation_survives_newer_scan_generation_and_advances_checkpoint() { let temp = tempdir().unwrap(); let store = Store::open(temp.path().join("work.sqlite")).unwrap(); let file = source_file(10, 100); @@ -73,21 +74,12 @@ fn stale_generation_is_typed_and_never_advances_checkpoint() { .unwrap(); let checkpoint = checkpoint(10, 3, "unix:2049:881", 110); - let error = store + store .upsert_provider_file_checkpoint(source_outcome(&file, stale_generation, 110), &checkpoint) - .unwrap_err(); - - assert!(matches!( - error, - StoreError::ImportInventorySuperseded { - inventory_family: SOURCE_IMPORT_INVENTORY_FAMILY, - expected_generation, - .. - } if expected_generation == stale_generation - )); + .unwrap(); assert_eq!( store.provider_file_checkpoint(checkpoint.key()).unwrap(), - None + Some(checkpoint) ); let status: String = store .conn @@ -97,8 +89,9 @@ fn stale_generation_is_typed_and_never_advances_checkpoint() { |row| row.get(0), ) .unwrap(); - assert_eq!(status, "pending"); + assert_eq!(status, "indexed"); } + #[test] fn imported_content_without_finalization_replays_before_checkpoint_advances() { let temp = tempdir().unwrap(); @@ -142,6 +135,8 @@ fn imported_content_without_finalization_replays_before_checkpoint_advances() { ); let second_checkpoint = checkpoint(20, 5, "unix:2049:881", 130); + let mut second_checkpoint = second_checkpoint; + second_checkpoint.head_sha256 = "f".repeat(64); store .upsert_provider_file_checkpoint( source_outcome(&appended, second_generation, 130), @@ -156,6 +151,7 @@ fn imported_content_without_finalization_replays_before_checkpoint_advances() { second_checkpoint ); } + #[test] fn deferred_partial_completion_retains_checkpoint_and_completes_exact_observation() { let temp = tempdir().unwrap(); @@ -207,6 +203,7 @@ fn deferred_partial_completion_retains_checkpoint_and_completes_exact_observatio .unwrap(); assert_eq!(state, ("indexed".to_owned(), 15, 120)); } + #[test] fn changed_checkpoint_version_requires_replacement_without_completing_observation() { let temp = tempdir().unwrap(); @@ -263,6 +260,7 @@ fn changed_checkpoint_version_requires_replacement_without_completing_observatio .unwrap(); assert_eq!(status, "pending"); } + #[test] fn catalog_observation_can_finalize_a_provider_file_checkpoint() { let temp = tempdir().unwrap(); @@ -283,7 +281,7 @@ fn catalog_observation_can_finalize_a_provider_file_checkpoint() { file_modified_at_ms: 20, import_revision: 3, cataloged_at_ms: 21, - metadata: json!({}), + metadata: json!({"file_observation_token_v1": "checkpoint-catalog-token"}), }; let generation = store .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) @@ -308,7 +306,7 @@ fn catalog_observation_can_finalize_a_provider_file_checkpoint() { }; let outcome = ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: CatalogSourceIndexUpdate { source_root: &catalog.source_root, @@ -321,6 +319,7 @@ fn catalog_observation_can_finalize_a_provider_file_checkpoint() { event_count: Some(2), indexed_at_ms: 30, }, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::Indexed, error: None, @@ -338,6 +337,62 @@ fn catalog_observation_can_finalize_a_provider_file_checkpoint() { checkpoint ); } + +#[test] +fn catalog_publication_requires_an_observation_token() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let mut catalog = catalog_file(12, 20); + catalog.metadata = json!({}); + let generation = store + .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) + .unwrap(); + store + .upsert_catalog_sessions(generation, std::slice::from_ref(&catalog)) + .unwrap(); + let update = CatalogSourceIndexUpdate { + source_root: &catalog.source_root, + source_path: &catalog.source_path, + file_size_bytes: catalog.file_size_bytes, + file_modified_at_ms: catalog.file_modified_at_ms, + import_revision: catalog.import_revision, + inventory_generation: generation, + file_sha256: None, + event_count: None, + indexed_at_ms: 30, + }; + + assert!(matches!( + store + .record_observed_catalog_source_import_result( + catalog.provider, + update, + &catalog.metadata, + CatalogIndexedStatus::Indexed, + None, + ) + .unwrap_err(), + StoreError::InvalidProviderFileCheckpoint("catalog observation token is required") + )); + + assert!(matches!( + store + .begin_provider_file_publication( + catalog.provider, + ProviderFileInventoryObservation::ObservedCatalog { + source_format: &catalog.source_format, + update, + metadata: &catalog.metadata, + }, + &catalog.source_format, + ProviderFilePublicationKind::Replacement, + 30, + ) + .unwrap_err(), + StoreError::InvalidProviderFileCheckpoint("catalog observation token is required") + )); +} + #[test] fn catalog_append_completion_preserves_rejections_and_accumulates_event_count() { let temp = tempdir().unwrap(); @@ -358,7 +413,7 @@ fn catalog_append_completion_preserves_rejections_and_accumulates_event_count() file_modified_at_ms: 100, import_revision: 3, cataloged_at_ms: 101, - metadata: json!({}), + metadata: json!({"file_observation_token_v1": "append-catalog-token"}), }; let first_generation = store .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) @@ -379,9 +434,10 @@ fn catalog_append_completion_preserves_rejections_and_accumulates_event_count() }; let first_outcome = ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: first_update, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::CompletedWithRejections, error: Some("one malformed event"), @@ -443,9 +499,10 @@ fn catalog_append_completion_preserves_rejections_and_accumulates_event_count() .upsert_provider_file_checkpoint( ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: second_update, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::Indexed, error: None, @@ -472,6 +529,7 @@ fn catalog_append_completion_preserves_rejections_and_accumulates_event_count() ) ); } + #[test] fn append_and_retained_tail_share_the_owner_lease_and_atomic_publication() { let temp = tempdir().unwrap(); @@ -621,6 +679,7 @@ fn append_and_retained_tail_share_the_owner_lease_and_atomic_publication() { Some(second_checkpoint) ); } + #[test] fn initial_import_uses_zero_seen_staging_rows() { let temp = tempdir().unwrap(); @@ -647,13 +706,14 @@ fn initial_import_uses_zero_seen_staging_rows() { assert!(!scope.tracks_prior_material); assert!(store.provider_file_publication.borrow().is_some()); assert!( - !store + store .provider_file_publication .borrow() .as_ref() .unwrap() .attached ); + assert_eq!(staged_seen_count(&store), 0); assert!(store.has_pending_provider_file_publications().unwrap()); assert!(matches!( store @@ -682,3 +742,590 @@ fn initial_import_uses_zero_seen_staging_rows() { ); assert_eq!(store.semantic_replacement_revision().unwrap(), 0); } + +#[test] +fn provider_publications_are_globally_capped_and_exactly_resumable() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let files = (0..129) + .map(|index| { + let mut file = source_file(10, 100); + file.source_path = format!("{ROOT}/{index:03}.jsonl"); + file + }) + .collect::>(); + let generation; + { + let store = Store::open(&path).unwrap(); + generation = store + .allocate_source_import_inventory_generation(files[0].provider, ROOT) + .unwrap(); + store + .upsert_source_import_files(generation, &files) + .unwrap(); + let outcome = source_outcome(&files[0], generation, 110); + let scope = store + .begin_provider_file_publication( + files[0].provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + drop(scope); + } + + for file in &files[1..] { + let observer = Store::open(&path).unwrap(); + let outcome = source_outcome(file, generation, 110); + assert!(matches!( + observer + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 106, + ) + .unwrap_err(), + StoreError::ProviderFileReplacementBusy { .. } + )); + let marker_count: usize = observer + .conn + .query_row( + "SELECT COUNT(*) FROM provider_file_publications", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1); + } + + let reopened = Store::open(&path).unwrap(); + let outcome = source_outcome(&files[0], generation, 110); + let adopted = reopened + .begin_provider_file_publication( + files[0].provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 107, + ) + .unwrap(); + assert!(matches!( + reopened.abort_provider_file_publication(adopted).unwrap(), + std::ops::ControlFlow::Continue(None) + )); + assert!(!reopened.has_pending_provider_file_publications().unwrap()); +} + +#[test] +fn first_rejected_replacement_finalizes_without_a_checkpoint() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = ProviderFileImportOutcome { + provider: file.provider, + observation: ProviderFileInventoryObservation::SourceImport { + source_format: &file.source_format, + update: SourceImportFileIndexUpdate { + source_root: &file.source_root, + source_path: &file.source_path, + file_size_bytes: file.file_size_bytes, + file_modified_at_ms: file.file_modified_at_ms, + import_revision: file.import_revision, + inventory_generation: generation, + metadata: &file.metadata, + indexed_at_ms: 110, + }, + }, + status: CatalogIndexedStatus::Rejected, + error: Some("all source content was rejected"), + }; + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + + let state: (String, Option) = store + .conn + .query_row( + "SELECT indexed_status, indexed_error FROM source_import_files WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3", + params![file.provider.as_str(), &file.source_root, &file.source_path], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(state.0, "rejected"); + assert_eq!(state.1.as_deref(), outcome.error); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(store + .provider_file_checkpoint(ProviderFileCheckpointKey { + provider: file.provider, + source_format: &file.source_format, + source_root: &file.source_root, + source_path: &file.source_path, + }) + .unwrap() + .is_none()); +} + +#[test] +fn rejected_replacement_cannot_discard_prior_material() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let prior_source = Uuid::from_u128(901); + insert_capture_source(&store, prior_source, PATH_A, "prior-material"); + insert_raw_event( + &store, + Uuid::from_u128(903), + 1, + prior_source, + "prior material", + ); + let mut outcome = source_outcome(&file, generation, 110); + outcome.status = CatalogIndexedStatus::Rejected; + outcome.error = Some("replacement rejected"); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + assert!(scope.tracks_prior_material()); + prepare_all(&store, &scope, 8); + + assert!(matches!( + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap_err(), + StoreError::InvalidProviderFileCheckpoint(_) + )); +} + +#[test] +fn rejected_replacement_cannot_publish_staged_material() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let mut outcome = source_outcome(&file, generation, 110); + outcome.status = CatalogIndexedStatus::Rejected; + outcome.error = Some("replacement rejected"); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let mut record = ctx_history_core::HistoryRecord::new( + "staged record", + "must not escape a rejected publication", + Vec::new(), + "note", + None, + ); + record.id = Uuid::from_u128(902); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_record(&record)) + .unwrap(); + + assert!(matches!( + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap_err(), + StoreError::InvalidProviderFileCheckpoint(_) + )); +} + +#[test] +fn rejected_replacement_cannot_publish_partial_capture_source() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let store = Store::open(&path).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let mut outcome = source_outcome(&file, generation, 110); + outcome.status = CatalogIndexedStatus::Rejected; + outcome.error = Some("source was written before the import was rejected"); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let source_id = Uuid::from_u128(905); + let source = capture_source_fixture(source_id, PATH_A, "rejected-partial-source"); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_capture_source(&source)) + .unwrap(); + let observer = Store::open(&path).unwrap(); + assert!(matches!( + observer.get_capture_source(source_id), + Err(StoreError::NotFound(id)) if id == source_id + )); + + assert!(matches!( + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap_err(), + StoreError::InvalidProviderFileCheckpoint(_) + )); + assert!(store.has_pending_provider_file_publications().unwrap()); + assert!(matches!( + observer.get_capture_source(source_id), + Err(StoreError::NotFound(id)) if id == source_id + )); + + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 120, + generation, + ) + .unwrap(); + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 125, + ) + .unwrap() + .unwrap(); + reconcile_all(&store, &retirement, 1); + store.retire_provider_file_publication(retirement).unwrap(); + assert!(!row_exists(&store, "capture_sources", source_id)); + assert!(!store.has_provider_data(file.provider).unwrap()); +} + +#[test] +fn completed_rejected_replacement_adopts_after_discarding_orphan_source() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(10, 100); + let generation; + { + let store = Store::open(&path).unwrap(); + generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let source_id = Uuid::from_u128(906); + let source = capture_source_fixture(source_id, PATH_A, "rejected-orphan-source"); + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_capture_source(&source) + }) + .unwrap(); + assert_eq!( + store + .discard_provider_file_publication_orphan_capture_sources(&scope) + .unwrap(), + 1 + ); + assert!(!row_exists(&store, "capture_sources", source_id)); + store + .stage_provider_file_publication_completion( + &scope, + &ProviderFilePublicationCompletion { + version: 1, + payload: json!({"rejected": true}), + }, + ) + .unwrap(); + store.abandon_provider_file_publication(scope).unwrap(); + } + + let reopened = Store::open(&path).unwrap(); + let mut outcome = source_outcome(&file, generation, 120); + outcome.status = CatalogIndexedStatus::Rejected; + outcome.error = Some("all source content was rejected"); + let scope = reopened + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 115, + ) + .unwrap(); + assert!(!scope.tracks_prior_material()); + reopened + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + assert!(!reopened.has_pending_provider_file_publications().unwrap()); +} + +#[test] +fn first_import_record_writes_stay_hidden_across_reopen() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let record_ids = [ + Uuid::from_u128(910), + Uuid::from_u128(911), + Uuid::from_u128(912), + ]; + { + let store = Store::open(&path).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = source_outcome(&file, generation, 110); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let records = record_ids + .iter() + .enumerate() + .map(|(index, id)| { + let mut record = ctx_history_core::HistoryRecord::new( + format!("private staged record {index}"), + format!("unpublished-token-{index}"), + Vec::new(), + "note", + None, + ); + record.id = *id; + record + }) + .collect::>(); + store + .with_provider_file_publication_writes(&scope, |store| { + store.insert_record(&records[0])?; + store.upsert_record(&records[1])?; + store.upsert_records(&records[2..])?; + for record in &records { + assert_eq!(store.get_record(record.id)?.id, record.id); + } + Ok(()) + }) + .unwrap(); + + let observer = Store::open(&path).unwrap(); + for id in record_ids { + assert!(matches!( + observer.get_record(id), + Err(StoreError::NotFound(_)) + )); + } + assert!(observer.list_records(10).unwrap().is_empty()); + assert!(observer + .search_records("unpublished-token", 10) + .unwrap() + .is_empty()); + drop(scope); + } + + let reopened = Store::open(&path).unwrap(); + for id in record_ids { + assert!(matches!( + reopened.get_record(id), + Err(StoreError::NotFound(_)) + )); + } + assert!(reopened.list_records(10).unwrap().is_empty()); + assert!(reopened + .search_records("unpublished-token", 10) + .unwrap() + .is_empty()); +} + +#[test] +fn staged_completion_is_bounded_atomic_and_survives_abandon_and_reopen() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(10, 100); + let generation; + let completion = ProviderFilePublicationCompletion { + version: 1, + payload: json!({"summary": {"imported": 3}, "checkpoint": "opaque"}), + }; + { + let store = Store::open(&path).unwrap(); + generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::Importing + ); + assert!(matches!( + store + .stage_provider_file_publication_completion( + &scope, + &ProviderFilePublicationCompletion { + version: 0, + payload: json!({}), + }, + ) + .unwrap_err(), + StoreError::InvalidProviderFilePublicationScope + )); + assert!(matches!( + store + .stage_provider_file_publication_completion( + &scope, + &ProviderFilePublicationCompletion { + version: 1, + payload: json!("x".repeat(PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES)), + }, + ) + .unwrap_err(), + StoreError::InvalidProviderFilePublicationScope + )); + store.inject_provider_file_fault(ProviderFileFaultPoint::CompletionBeforeCommit); + assert!(matches!( + store + .stage_provider_file_publication_completion(&scope, &completion) + .unwrap_err(), + StoreError::ProviderFileStaging + )); + assert_eq!( + store + .load_provider_file_publication_completion(&scope) + .unwrap(), + None + ); + store + .stage_provider_file_publication_completion(&scope, &completion) + .unwrap(); + store + .stage_provider_file_publication_completion(&scope, &completion) + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::ReadyToFinalize + ); + assert!(matches!( + store + .with_provider_file_publication_writes(&scope, |_| Ok(())) + .unwrap_err(), + StoreError::InvalidProviderFilePublicationScope + )); + store.abandon_provider_file_publication(scope).unwrap(); + } + + let reopened = Store::open(&path).unwrap(); + let scope = reopened + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 120).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 115, + ) + .unwrap(); + assert_eq!( + reopened + .load_provider_file_publication_completion(&scope) + .unwrap(), + Some(completion) + ); + assert_eq!( + reopened.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::ReadyToFinalize + ); + assert!(matches!( + reopened.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Continue(None) + )); + assert!(!reopened.has_pending_provider_file_publications().unwrap()); +} diff --git a/crates/ctx-history-store/src/provider_files/tests/lifecycle_and_adoption.rs b/crates/ctx-history-store/src/provider_files/tests/lifecycle_and_adoption.rs index 5a8b051d..50e729f8 100644 --- a/crates/ctx-history-store/src/provider_files/tests/lifecycle_and_adoption.rs +++ b/crates/ctx-history-store/src/provider_files/tests/lifecycle_and_adoption.rs @@ -133,6 +133,7 @@ fn first_owner_lock_blocks_other_store_and_stale_marker_is_adopted() { assert!(!observer.has_pending_provider_file_publications().unwrap()); assert_eq!(observer.get_capture_source(source).unwrap().id, source); } + #[test] fn dropping_publication_scope_releases_owner_lock_for_durable_marker_adoption() { let temp = tempdir().unwrap(); @@ -194,6 +195,184 @@ fn dropping_publication_scope_releases_owner_lock_for_durable_marker_adoption() first.abandon_provider_file_publication(readopted).unwrap(); } +#[test] +fn abort_publication_discards_only_unmutated_markers_and_releases_the_scope() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = source_outcome(&file, generation, 110); + let unmutated = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 105, + ) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(unmutated).unwrap(), + std::ops::ControlFlow::Continue(None) + )); + assert!(store.provider_file_publication.borrow().is_none()); + assert!(!store.has_pending_provider_file_publications().unwrap()); + + let source = Uuid::from_u128(64_050); + insert_capture_source(&store, source, PATH_A, "abort-after-mutation"); + let mutated = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 106, + ) + .unwrap(); + let event = event_fixture( + Uuid::from_u128(64_051), + 1, + source, + "abort-after-mutation".to_owned(), + "durably fenced", + ); + store + .with_provider_file_publication_writes(&mutated, |store| store.upsert_event(&event)) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(mutated).unwrap(), + std::ops::ControlFlow::Break(None) + )); + assert!(store.provider_file_publication.borrow().is_none()); + assert!(store.has_pending_provider_file_publications().unwrap()); +} + +#[test] +fn abort_outcome_surfaces_cleanup_warning_without_losing_recovery_state() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 105, + ) + .unwrap(); + let source = Uuid::from_u128(64_060); + insert_capture_source(&store, source, PATH_A, "abort-warning"); + let mut event = event_fixture( + Uuid::from_u128(64_061), + 1, + source, + "abort-warning".to_owned(), + "durably fenced", + ); + event.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&event)) + .unwrap(); + + store.inject_provider_file_fault(ProviderFileFaultPoint::Cleanup); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(Some( + ProviderFileMaintenanceWarning::StagingCleanupDeferred { .. } + )) + )); + assert!(store.has_pending_provider_file_publications().unwrap()); + store.cleanup_abandoned_provider_file_publication().unwrap(); + assert!(store.provider_file_publication.borrow().is_none()); +} + +#[test] +fn invalid_finalize_discards_an_unmutated_marker_before_consuming_the_scope() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = source_outcome(&file, generation, 110); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 105, + ) + .unwrap(); + assert_eq!(scope.kind(), ProviderFilePublicationKind::Incremental); + + assert!(matches!( + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap_err(), + StoreError::InvalidProviderFilePublicationScope + )); + assert!(store.provider_file_publication.borrow().is_none()); + assert!(!store.has_pending_provider_file_publications().unwrap()); +} + +#[test] +fn mutable_publication_write_scope_resets_after_panic() { + let temp = tempdir().unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = source_outcome(&file, generation, 110); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 105, + ) + .unwrap(); + + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = store.with_provider_file_publication_writes_mut::<(), StoreError>(&scope, |_| { + panic!("publication write panic") + }); + })); + assert!(panicked.is_err()); + assert!(store.provider_file_write_scope.get().is_none()); + store + .with_provider_file_publication_writes_mut::<(), StoreError>(&scope, |_| Ok(())) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Continue(None) + )); +} + #[test] fn crashed_mutated_replacement_cannot_be_adopted_or_finalized_as_incremental() { let temp = tempdir().unwrap(); @@ -310,3 +489,293 @@ fn crashed_mutated_replacement_cannot_be_adopted_or_finalized_as_incremental() { assert_eq!(second.list_events().unwrap()[0].id, new_event); assert!(!row_exists(&second, "events", old_event)); } + +#[test] +fn changed_observation_adoption_resets_prior_staging_progress() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let original = source_file(20, 100); + let source = Uuid::from_u128(64_200); + let replacement_event = Uuid::from_u128(64_201); + let prior_event = Uuid::from_u128(64_202); + + { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(original.provider, &original.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&original)) + .unwrap(); + insert_capture_source(&store, source, PATH_A, "changed-staging-adoption"); + insert_raw_event(&store, prior_event, 1, source, "prior generation"); + let scope = store + .begin_provider_file_publication( + original.provider, + source_outcome(&original, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + prepare_all(&store, &scope, 1); + let mut event = event_fixture( + replacement_event, + 2, + source, + "changed-staging-adoption".to_owned(), + "old attempt", + ); + event.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&event)) + .unwrap(); + let completion = ProviderFilePublicationCompletion { + version: 1, + payload: json!({"attempt": "old-observation"}), + }; + store + .stage_provider_file_publication_completion(&scope, &completion) + .unwrap(); + store + .reconcile_provider_file_publication_slice(&scope, 1) + .unwrap(); + assert_eq!(staged_seen_count(&store), 1); + drop(scope); + } + + let store = Store::open(&path).unwrap(); + let changed = source_file(30, 120); + let generation = store + .allocate_source_import_inventory_generation(changed.provider, &changed.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&changed)) + .unwrap(); + let adopted = store + .begin_provider_file_publication( + changed.provider, + source_outcome(&changed, generation, 130).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 125, + ) + .unwrap(); + assert_eq!(staged_seen_count(&store), 0); + assert_eq!( + store + .load_provider_file_publication_completion(&adopted) + .unwrap(), + None + ); + let reset_progress: (i64, Option, Option) = store + .conn + .query_row( + "SELECT cleanup_phase, cleanup_source_cursor, cleanup_entity_cursor \ + FROM provider_file_publications", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(reset_progress, (0, None, None)); + assert_eq!( + store.provider_file_publication_phase(&adopted).unwrap(), + ProviderFilePublicationPhase::Preparing + ); + assert!(matches!( + store.abort_provider_file_publication(adopted).unwrap(), + std::ops::ControlFlow::Break(None) + )); +} + +#[test] +fn changed_observation_adoption_cleans_first_attempt_material() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let original = source_file(20, 100); + let source_id = Uuid::from_u128(64_210); + let event_id = Uuid::from_u128(64_211); + let record_id = Uuid::from_u128(64_212); + + { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(original.provider, &original.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&original)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + original.provider, + source_outcome(&original, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + assert!(!scope.tracks_prior_material()); + let source = capture_source_fixture(source_id, PATH_A, "first-attempt-source"); + let mut event = event_fixture( + event_id, + 1, + source_id, + "first-attempt-event".to_owned(), + "must be removed after changed adoption", + ); + event.dedupe_key = None; + let mut record = ctx_history_core::HistoryRecord::new( + "first-attempt-record", + "must be removed after changed adoption", + Vec::new(), + "note", + None, + ); + record.id = record_id; + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_capture_source(&source)?; + store.upsert_event(&event)?; + store.upsert_record(&record) + }) + .unwrap(); + drop(scope); + } + + let store = Store::open(&path).unwrap(); + let changed = source_file(30, 120); + let generation = store + .allocate_source_import_inventory_generation(changed.provider, &changed.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&changed)) + .unwrap(); + let outcome = source_outcome(&changed, generation, 130); + let adopted = store + .begin_provider_file_publication( + changed.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 125, + ) + .unwrap(); + assert!(adopted.tracks_prior_material()); + assert!(store.list_events().unwrap().is_empty()); + assert!(store.list_records(10).unwrap().is_empty()); + reconcile_all(&store, &adopted, 1); + store + .finalize_provider_file_publication( + adopted, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + + assert!(!row_exists(&store, "events", event_id)); + assert!(!row_exists(&store, "history_records", record_id)); + assert!(!row_exists(&store, "capture_sources", source_id)); +} + +#[test] +fn retirement_cleans_source_less_record_from_crashed_first_attempt() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(20, 100); + let record_ids = [ + Uuid::from_u128(64_220), + Uuid::from_u128(64_221), + Uuid::from_u128(64_222), + ]; + let generation = { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let records = record_ids + .iter() + .enumerate() + .map(|(index, id)| { + let mut record = ctx_history_core::HistoryRecord::new( + format!("retired-first-attempt-record-{index}"), + "must remain hidden until retirement removes it", + Vec::new(), + "note", + None, + ); + record.id = *id; + record + }) + .collect::>(); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_records(&records)) + .unwrap(); + drop(scope); + generation + }; + + let store = Store::open(&path).unwrap(); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 120, + generation, + ) + .unwrap(); + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 125, + ) + .unwrap() + .unwrap(); + assert!(retirement.tracks_prior_material()); + assert!(store.list_records(10).unwrap().is_empty()); + prepare_all(&store, &retirement, 1); + let first_slice = store + .reconcile_provider_file_publication_slice(&retirement, 1) + .unwrap(); + assert_eq!(first_slice.rows_scanned, 1); + assert!(!first_slice.complete); + store.abandon_provider_file_publication(retirement).unwrap(); + drop(store); + + let reopened = Store::open(&path).unwrap(); + let retirement = reopened + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 130, + ) + .unwrap() + .unwrap(); + assert!(reopened.list_records(10).unwrap().is_empty()); + reconcile_all(&reopened, &retirement, 1); + reopened + .retire_provider_file_publication(retirement) + .unwrap(); + + for record_id in record_ids { + assert!(!row_exists(&reopened, "history_records", record_id)); + } + assert!(reopened.list_records(10).unwrap().is_empty()); +} diff --git a/crates/ctx-history-store/src/provider_files/tests/locks_and_security.rs b/crates/ctx-history-store/src/provider_files/tests/locks_and_security.rs index 643e7488..75e4e8fe 100644 --- a/crates/ctx-history-store/src/provider_files/tests/locks_and_security.rs +++ b/crates/ctx-history-store/src/provider_files/tests/locks_and_security.rs @@ -11,6 +11,7 @@ fn existing_owner_lock_directory_must_already_be_private() { let error = create_or_validate_private_lock_dir(&lock_dir).unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); } + #[cfg(windows)] #[test] fn windows_private_paths_validate_exact_acl_and_reject_wrong_object_type() { @@ -25,6 +26,7 @@ fn windows_private_paths_validate_exact_acl_and_reject_wrong_object_type() { validate_existing_private_windows_path(&file_path, false).unwrap(); assert!(validate_existing_private_windows_path(&file_path, true).is_err()); } + #[cfg(unix)] #[test] fn unix_owner_lock_rejects_symlink_hardlink_permissive_owner_and_inode_swap() { @@ -78,6 +80,63 @@ fn unix_owner_lock_rejects_symlink_hardlink_permissive_owner_and_inode_swap() { std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o600)).unwrap(); assert!(validate_open_private_owner_lock_file(&opened, &target).is_err()); } + +#[cfg(unix)] +#[test] +fn non_utf8_private_root_does_not_block_main_database_staging() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(20, 100); + let retained_event = Uuid::from_u128(69_900); + let generation = { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let source = Uuid::from_u128(69_901); + insert_capture_source(&store, source, PATH_A, "non-utf8-staging-root"); + insert_raw_event(&store, retained_event, 1, source, "retained visible event"); + generation + }; + let invalid_temp_root = temp + .path() + .join(OsString::from_vec(b"non-utf8-\xff".to_vec())); + std::fs::create_dir(&invalid_temp_root).unwrap(); + + let output = Command::new(std::env::current_exe().unwrap()) + .arg("--ignored") + .arg("--exact") + .arg("provider_files::tests::provider_file_subprocess_helper") + .arg("--test-threads=1") + .env("CTX_PROVIDER_FILE_HELPER_ACTION", "non-utf8-private-root") + .env("CTX_PROVIDER_FILE_HELPER_STORE", &path) + .env( + "CTX_PROVIDER_FILE_HELPER_GENERATION", + generation.to_string(), + ) + .env("TMPDIR", &invalid_temp_root) + .output() + .unwrap(); + assert!( + output.status.success(), + "subprocess failed with {}:\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let reopened = Store::open(&path).unwrap(); + assert!(!reopened.has_pending_provider_file_publications().unwrap()); + assert!(row_exists(&reopened, "events", retained_event)); + assert_eq!(reopened.list_events().unwrap().len(), 1); +} + #[cfg(unix)] #[test] fn canonical_store_lock_identity_survives_rename_hardlink_and_symlink_aliases() { @@ -105,6 +164,7 @@ fn canonical_store_lock_identity_survives_rename_hardlink_and_symlink_aliases() crate::store_identity::CanonicalStoreIdentity::open_target(&renamed, false).unwrap(); assert_eq!(first.digest(), moved.digest()); } + #[cfg(unix)] #[test] fn subprocess_owner_lock_excludes_aliases_and_releases_on_process_exit() { @@ -168,6 +228,7 @@ fn subprocess_owner_lock_excludes_aliases_and_releases_on_process_exit() { std::fs::read_to_string(second_digest).unwrap() ); } + #[test] fn subprocess_partial_cleanup_supersession_stays_hidden_and_is_adopted() { let temp = tempdir().unwrap(); diff --git a/crates/ctx-history-store/src/provider_files/tests/recovery_and_faults.rs b/crates/ctx-history-store/src/provider_files/tests/recovery_and_faults.rs index de94db3e..8a55d97a 100644 --- a/crates/ctx-history-store/src/provider_files/tests/recovery_and_faults.rs +++ b/crates/ctx-history-store/src/provider_files/tests/recovery_and_faults.rs @@ -18,7 +18,7 @@ fn catalog_retain_completion_preserves_every_legacy_import_cursor_field() { file_modified_at_ms: 100, import_revision: 1, cataloged_at_ms: 101, - metadata: json!({}), + metadata: json!({"file_observation_token_v1": "recovery-catalog-token"}), }; let first_generation = store .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) @@ -41,9 +41,10 @@ fn catalog_retain_completion_preserves_every_legacy_import_cursor_field() { .upsert_provider_file_checkpoint( ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: first_update, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::Indexed, error: None, @@ -76,9 +77,10 @@ fn catalog_retain_completion_preserves_every_legacy_import_cursor_field() { store .complete_provider_file_observation_retaining_checkpoint(ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: second_update, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::Indexed, error: None, @@ -98,8 +100,9 @@ fn catalog_retain_completion_preserves_every_legacy_import_cursor_field() { .unwrap(); assert_eq!(indexed, (130, 15, 120)); } + #[test] -fn crashed_replacement_keeps_old_rows_and_checkpoint_without_main_staging() { +fn crashed_replacement_keeps_rows_checkpoint_and_main_staging_transactionally() { let temp = tempdir().unwrap(); let path = temp.path().join("work.sqlite"); let old_event = Uuid::from_u128(80); @@ -207,7 +210,16 @@ fn crashed_replacement_keeps_old_rows_and_checkpoint_without_main_staging() { .unwrap() .is_empty()); assert!(store.provider_file_publication.borrow().is_none()); - assert!(!main_table_exists(&store, "provider_file_publication_seen")); + assert!(main_table_exists(&store, "provider_file_publication_seen")); + let staged_seen: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM provider_file_publication_seen", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(staged_seen, 1); let rewritten = source_file(20, 120); let generation = store @@ -244,6 +256,199 @@ fn crashed_replacement_keeps_old_rows_and_checkpoint_without_main_staging() { assert_eq!(store.list_events().unwrap().len(), 1); assert!(!store.search_event_hits("new", 10).unwrap().is_empty()); } + +#[test] +fn replacement_reopen_reconstructs_when_cursor_outlives_staging_rows() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(20, 100); + let source = Uuid::from_u128(329_000); + let stale_event = Uuid::from_u128(329_001); + let (generation, replacement_id) = { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + insert_capture_source(&store, source, PATH_A, "missing-replacement-staging"); + insert_raw_event(&store, stale_event, 1, source, "stale replacement event"); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + prepare_all(&store, &scope, 1); + assert_eq!(staged_prior_source_count(&store), 1); + let replacement_id = scope.scope_id.to_string(); + drop(scope); + (generation, replacement_id) + }; + + { + let store = Store::open(&path).unwrap(); + store + .conn + .execute( + "DELETE FROM provider_file_publication_prior_sources \ + WHERE replacement_id = ?1", + params![&replacement_id], + ) + .unwrap(); + } + let store = Store::open(&path).unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 120).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 115, + ) + .unwrap(); + + assert_eq!(scope.scope_id.to_string(), replacement_id); + assert_eq!(staged_prior_source_count(&store), 0); + assert!(store.has_pending_provider_file_publications().unwrap()); + assert!(row_exists(&store, "events", stale_event)); + assert!(store.list_events().unwrap().is_empty()); + prepare_all(&store, &scope, 1); + assert_eq!(staged_prior_source_count(&store), 1); + reconcile_all(&store, &scope, 1); + store + .finalize_provider_file_publication( + scope, + source_outcome(&file, generation, 120), + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + assert!(!row_exists(&store, "events", stale_event)); +} + +#[test] +fn retirement_reopen_reconstructs_when_cursor_outlives_staging_rows() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(20, 100); + let source = Uuid::from_u128(329_010); + let retained_event = Uuid::from_u128(329_011); + let replacement_id = { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + insert_capture_source(&store, source, PATH_A, "missing-retirement-staging"); + insert_raw_event( + &store, + retained_event, + 1, + source, + "retained retirement event", + ); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + prepare_all(&store, &scope, 1); + store + .track_provider_file_publication_event(retained_event) + .unwrap(); + let mut mutation = event_fixture( + Uuid::from_u128(329_012), + 2, + source, + "missing-retirement-staging".to_owned(), + "mutation before retirement", + ); + mutation.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&mutation)) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 120, + generation, + ) + .unwrap(); + + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 125, + ) + .unwrap() + .unwrap(); + prepare_all(&store, &retirement, 1); + let progress = store + .reconcile_provider_file_publication_slice(&retirement, 1) + .unwrap(); + assert!(!progress.complete); + let replacement_id = retirement.scope_id.to_string(); + store.abandon_provider_file_publication(retirement).unwrap(); + replacement_id + }; + + { + let store = Store::open(&path).unwrap(); + for table in [ + "provider_file_publication_seen", + "provider_file_publication_prior_sources", + ] { + store + .conn + .execute( + &format!("DELETE FROM {table} WHERE replacement_id = ?1"), + params![&replacement_id], + ) + .unwrap(); + } + } + let store = Store::open(&path).unwrap(); + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 130, + ) + .unwrap() + .unwrap(); + + assert_eq!(retirement.scope_id.to_string(), replacement_id); + assert!(store.has_pending_provider_file_publications().unwrap()); + assert!(row_exists(&store, "events", retained_event)); + assert!(store.list_events().unwrap().is_empty()); + prepare_all(&store, &retirement, 1); + reconcile_all(&store, &retirement, 1); + store.retire_provider_file_publication(retirement).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(!row_exists(&store, "events", retained_event)); +} + #[test] fn replacement_scope_fails_if_same_generation_observation_changes() { let temp = tempdir().unwrap(); @@ -289,6 +494,7 @@ fn replacement_scope_fails_if_same_generation_observation_changes() { assert!(row_exists(&store, "events", old_event)); assert!(store.provider_file_publication.borrow().is_none()); } + #[test] fn transaction_boundary_faults_roll_back_and_restart_converges() { let temp = tempdir().unwrap(); @@ -435,19 +641,14 @@ fn transaction_boundary_faults_roll_back_and_restart_converges() { ) .unwrap(); assert_eq!(scope.kind(), ProviderFilePublicationKind::Replacement); - prepare_all(&store, &scope, 1); - let mut replacement = event_fixture( - new_event, - 2, - source, - "transaction-boundary-faults".to_owned(), - "new generation", + assert_eq!( + store.provider_file_publication_phase(&scope).unwrap(), + ProviderFilePublicationPhase::ReadyToFinalize ); - replacement.dedupe_key = None; - store - .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&replacement)) - .unwrap(); - reconcile_all(&store, &scope, 1); + assert!(store + .load_provider_file_publication_completion(&scope) + .unwrap() + .is_some()); let mut new_checkpoint = old_checkpoint.clone(); new_checkpoint.updated_at_ms = 130; store diff --git a/crates/ctx-history-store/src/provider_files/tests/retirement_observations.rs b/crates/ctx-history-store/src/provider_files/tests/retirement_observations.rs index eb0aa1ac..ce558ad1 100644 --- a/crates/ctx-history-store/src/provider_files/tests/retirement_observations.rs +++ b/crates/ctx-history-store/src/provider_files/tests/retirement_observations.rs @@ -1,3 +1,501 @@ +fn create_mutated_retirement_marker(store: &Store, family: RetirementInventoryFamily) { + let (source_id, old_event_id, replacement_event_id) = match family { + RetirementInventoryFamily::Catalog => ( + Uuid::from_u128(65_200), + Uuid::from_u128(65_201), + Uuid::from_u128(65_202), + ), + RetirementInventoryFamily::SourceImport => ( + Uuid::from_u128(65_210), + Uuid::from_u128(65_211), + Uuid::from_u128(65_212), + ), + }; + insert_capture_source(store, source_id, PATH_A, "retirement-family-format"); + insert_raw_event(store, old_event_id, 1, source_id, "old generation"); + + let scope = match family { + RetirementInventoryFamily::Catalog => { + let catalog = catalog_file(20, 100); + let generation = store + .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) + .unwrap(); + store + .upsert_catalog_sessions(generation, std::slice::from_ref(&catalog)) + .unwrap(); + store + .begin_provider_file_publication( + catalog.provider, + catalog_observation(&catalog, generation, 110), + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap() + } + RetirementInventoryFamily::SourceImport => { + let file = source_file(20, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap() + } + }; + prepare_all(store, &scope, 1); + let mut replacement = event_fixture( + replacement_event_id, + 2, + source_id, + "retirement-family-format".to_owned(), + "replacement generation", + ); + replacement.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&replacement)) + .unwrap(); + drop(scope); +} + +fn add_nonmatching_current_retirement_observations( + store: &Store, + family: RetirementInventoryFamily, +) -> u64 { + match family { + RetirementInventoryFamily::Catalog => { + let generation = store + .allocate_catalog_inventory_generation(CaptureProvider::Claude, ROOT) + .unwrap(); + let mut wrong_format = catalog_file(30, 130); + wrong_format.source_format = WRONG_CATALOG_FORMAT.to_owned(); + wrong_format.external_session_id = Some("wrong-catalog-format".to_owned()); + store + .upsert_catalog_sessions(generation, &[wrong_format]) + .unwrap(); + + let mut opposite_family = source_file(40, 140); + opposite_family.source_format = MATERIAL_FORMAT.to_owned(); + let opposite_generation = store + .allocate_source_import_inventory_generation( + opposite_family.provider, + &opposite_family.source_root, + ) + .unwrap(); + store + .upsert_source_import_files(opposite_generation, &[opposite_family]) + .unwrap(); + generation + } + RetirementInventoryFamily::SourceImport => { + let generation = store + .allocate_source_import_inventory_generation(CaptureProvider::Claude, ROOT) + .unwrap(); + let mut wrong_format = source_file(30, 130); + wrong_format.source_format = WRONG_SOURCE_IMPORT_FORMAT.to_owned(); + store + .upsert_source_import_files(generation, &[wrong_format]) + .unwrap(); + + let mut opposite_family = catalog_file(40, 140); + opposite_family.source_format = FORMAT.to_owned(); + opposite_family.external_session_id = Some("opposite-catalog-family".to_owned()); + let opposite_generation = store + .allocate_catalog_inventory_generation( + opposite_family.provider, + &opposite_family.source_root, + ) + .unwrap(); + store + .upsert_catalog_sessions(opposite_generation, &[opposite_family]) + .unwrap(); + generation + } + } +} + +fn revive_matching_retirement_observation( + store: &Store, + family: RetirementInventoryFamily, + generation: u64, +) { + match family { + RetirementInventoryFamily::Catalog => store + .upsert_catalog_sessions(generation, &[catalog_file(20, 100)]) + .unwrap(), + RetirementInventoryFamily::SourceImport => store + .upsert_source_import_files(generation, &[source_file(20, 100)]) + .unwrap(), + }; +} + +fn current_retirement_observation_exists(store: &Store, table: &str, source_format: &str) -> bool { + store + .conn + .query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM {table} WHERE provider = ?1 \ + AND source_format = ?2 AND source_root = ?3 AND source_path = ?4 \ + AND is_stale = 0)" + ), + params![ + CaptureProvider::Claude.as_str(), + source_format, + ROOT, + PATH_A + ], + |row| row.get(0), + ) + .unwrap() +} + +fn provider_file_publication_observation_invalidated(store: &Store) -> bool { + store + .conn + .query_row( + "SELECT inventory_observation_invalidated FROM provider_file_publications", + [], + |row| row.get(0), + ) + .unwrap() +} + +#[test] +fn retirement_ignores_opposite_family_and_wrong_format_current_observations() { + for family in [ + RetirementInventoryFamily::Catalog, + RetirementInventoryFamily::SourceImport, + ] { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + { + let store = Store::open(&path).unwrap(); + create_mutated_retirement_marker(&store, family); + } + + let store = Store::open(&path).unwrap(); + add_nonmatching_current_retirement_observations(&store, family); + assert!(current_retirement_observation_exists( + &store, + family.inventory_table(), + family.wrong_source_format(), + )); + assert!(current_retirement_observation_exists( + &store, + family.opposite_inventory_table(), + family.inventory_source_format(), + )); + + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 1, + "{family:?}", + ); + assert!(store + .list_provider_file_publication_retirement_work(0) + .unwrap() + .is_empty()); + assert_eq!( + store + .list_provider_file_publication_retirement_work(10) + .unwrap(), + vec![ProviderFilePublicationRetirementWork { + provider: CaptureProvider::Claude, + material_source_format: MATERIAL_FORMAT.to_owned(), + material_source_root: ROOT.to_owned(), + source_path: PATH_A.to_owned(), + estimated_bytes: 20, + last_attempt_at_ms: 105, + }], + "{family:?}", + ); + + let retirement = store + .begin_provider_file_publication_retirement( + CaptureProvider::Claude, + MATERIAL_FORMAT, + ROOT, + PATH_A, + 150, + ) + .unwrap() + .unwrap_or_else(|| panic!("{family:?} retirement was not adoptable")); + prepare_all(&store, &retirement, 1); + reconcile_all(&store, &retirement, 1); + store.retire_provider_file_publication(retirement).unwrap(); + + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 0, + "{family:?}", + ); + assert!(store + .list_provider_file_publication_retirement_work(10) + .unwrap() + .is_empty()); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert!(current_retirement_observation_exists( + &store, + family.inventory_table(), + family.wrong_source_format(), + )); + assert!(current_retirement_observation_exists( + &store, + family.opposite_inventory_table(), + family.inventory_source_format(), + )); + } +} + +fn create_source_import_retirement_work( + store: &Store, + source_path: &str, + source_id: Uuid, + event_id: Uuid, + created_at_ms: i64, +) { + let mut file = source_file(20, created_at_ms - 5); + file.source_path = source_path.to_owned(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + insert_capture_source(store, source_id, source_path, "retirement-attempt-order"); + insert_raw_event( + store, + event_id, + created_at_ms, + source_id, + "retirement attempt ordering", + ); + + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, created_at_ms + 5).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + created_at_ms, + ) + .unwrap(); + prepare_all(store, &scope, 1); + let mut mutation = event_fixture( + Uuid::from_u128(event_id.as_u128() + 1), + (created_at_ms + 1) as u64, + source_id, + "retirement-attempt-order".to_owned(), + "mutation before retirement", + ); + mutation.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&mutation)) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + created_at_ms + 10, + generation, + ) + .unwrap(); +} + +#[test] +fn retirement_attempt_timestamps_rotate_fairly_and_survive_reopen() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + { + let store = Store::open(&path).unwrap(); + create_source_import_retirement_work( + &store, + PATH_A, + Uuid::from_u128(65_300), + Uuid::from_u128(65_301), + 100, + ); + // Simulate a legacy database created before publication work was + // globally serialized. An ineffective marker would not have blocked + // the second owner, then both could later become durable recovery work. + store + .conn + .execute( + "UPDATE provider_file_publications SET mutation_started = 0 WHERE source_path = ?1", + [PATH_A], + ) + .unwrap(); + create_source_import_retirement_work( + &store, + PATH_B, + Uuid::from_u128(65_310), + Uuid::from_u128(65_311), + 200, + ); + store + .conn + .execute( + "UPDATE provider_file_publications SET mutation_started = 1 WHERE source_path = ?1", + [PATH_A], + ) + .unwrap(); + + let initial = store + .list_provider_file_publication_retirement_work(10) + .unwrap(); + assert_eq!( + initial + .iter() + .map(|work| (work.source_path.as_str(), work.last_attempt_at_ms)) + .collect::>(), + vec![(PATH_A, 100), (PATH_B, 200)] + ); + + let retirement = store + .begin_provider_file_publication_retirement( + CaptureProvider::Claude, + MATERIAL_FORMAT, + ROOT, + PATH_A, + 300, + ) + .unwrap() + .unwrap(); + let after_begin = store + .list_provider_file_publication_retirement_work(10) + .unwrap(); + assert_eq!( + after_begin + .iter() + .map(|work| (work.source_path.as_str(), work.last_attempt_at_ms)) + .collect::>(), + vec![(PATH_B, 200), (PATH_A, 300)] + ); + store.abandon_provider_file_publication(retirement).unwrap(); + } + + let reopened = Store::open(&path).unwrap(); + let after_abandon = reopened + .list_provider_file_publication_retirement_work(10) + .unwrap(); + assert_eq!(after_abandon[0].source_path, PATH_B); + assert_eq!(after_abandon[0].last_attempt_at_ms, 200); + assert_eq!(after_abandon[1].source_path, PATH_A); + assert!(after_abandon[1].last_attempt_at_ms > 300); +} + +#[test] +fn newer_generation_of_same_observation_adopts_existing_publication() { + for family in [ + RetirementInventoryFamily::Catalog, + RetirementInventoryFamily::SourceImport, + ] { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + { + let store = Store::open(&path).unwrap(); + create_mutated_retirement_marker(&store, family); + } + + let store = Store::open(&path).unwrap(); + let generation = add_nonmatching_current_retirement_observations(&store, family); + revive_matching_retirement_observation(&store, family, generation); + assert!(current_retirement_observation_exists( + &store, + family.inventory_table(), + family.inventory_source_format(), + )); + + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 0, + "{family:?}", + ); + assert!(store + .begin_provider_file_publication_retirement( + CaptureProvider::Claude, + MATERIAL_FORMAT, + ROOT, + PATH_A, + 150, + ) + .unwrap() + .is_none()); + match family { + RetirementInventoryFamily::Catalog => { + let catalog = catalog_file(20, 100); + let observation = catalog_observation(&catalog, generation, 160); + let scope = store + .begin_provider_file_publication( + catalog.provider, + observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 155, + ) + .unwrap(); + let outcome = ProviderFileImportOutcome { + provider: catalog.provider, + observation, + status: CatalogIndexedStatus::Indexed, + error: None, + }; + reconcile_all(&store, &scope, 1); + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + } + RetirementInventoryFamily::SourceImport => { + let file = source_file(20, 100); + let outcome = source_outcome(&file, generation, 160); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 155, + ) + .unwrap(); + reconcile_all(&store, &scope, 1); + store + .finalize_provider_file_publication( + scope, + outcome, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + } + } + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert_eq!(table_row_count(&store, "provider_file_publications"), 0); + } +} + #[test] fn tombstoned_observation_retires_visibility_fence_and_can_be_adopted() { let temp = tempdir().unwrap(); @@ -71,3 +569,211 @@ fn tombstoned_observation_retires_visibility_fence_and_can_be_adopted() { .unwrap(); assert_eq!(store.get_capture_source(source).unwrap().id, source); } + +#[test] +fn changed_observation_rebind_clears_explicit_invalidation() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let original = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(original.provider, &original.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&original)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + original.provider, + source_outcome(&original, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let source = capture_source_fixture(Uuid::from_u128(66), PATH_A, "rebound-owner"); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_capture_source(&source)) + .unwrap(); + let owner = store + .effective_provider_file_publication_inventory_owner() + .unwrap() + .unwrap(); + assert!(store + .invalidate_effective_provider_file_publication_observation(&owner, 120) + .unwrap()); + assert!(provider_file_publication_observation_invalidated(&store)); + store.abandon_provider_file_publication(scope).unwrap(); + + let changed = source_file(20, 130); + let changed_generation = store + .allocate_source_import_inventory_generation(changed.provider, &changed.source_root) + .unwrap(); + store + .upsert_source_import_files(changed_generation, std::slice::from_ref(&changed)) + .unwrap(); + let rebound = store + .begin_provider_file_publication( + changed.provider, + source_outcome(&changed, changed_generation, 140).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 135, + ) + .unwrap(); + + assert!(!provider_file_publication_observation_invalidated(&store)); + assert_eq!( + store.provider_file_publication_phase(&rebound).unwrap(), + ProviderFilePublicationPhase::Preparing + ); + let rebound_source = + capture_source_fixture(Uuid::from_u128(67), PATH_A, "rebound-new-observation"); + store + .with_provider_file_publication_writes(&rebound, |store| { + store.upsert_capture_source(&rebound_source) + }) + .unwrap(); + reconcile_all(&store, &rebound, 1); + store + .finalize_provider_file_publication( + rebound, + source_outcome(&changed, changed_generation, 145), + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + assert_eq!( + store.get_capture_source(rebound_source.id).unwrap().id, + rebound_source.id + ); + assert!(!store.has_pending_provider_file_publications().unwrap()); +} + +#[test] +fn retirement_adoption_preserves_invalidation_after_identical_inventory_revival() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = source_outcome(&file, generation, 110); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + let source = capture_source_fixture(Uuid::from_u128(66), PATH_A, "invalidated-owner"); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_capture_source(&source)) + .unwrap(); + + let owner = store + .effective_provider_file_publication_inventory_owner() + .unwrap() + .unwrap(); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 115, + generation, + ) + .unwrap(); + assert!(store + .invalidate_effective_provider_file_publication_observation(&owner, 120) + .unwrap()); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + + assert_eq!( + store + .provider_file_publication_retirement_work_count() + .unwrap(), + 1 + ); + assert!(!store + .provider_file_publication_matches_candidate( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + &file.source_root, + ) + .unwrap()); + store.abandon_provider_file_publication(scope).unwrap(); + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 125, + ) + .unwrap() + .unwrap(); + assert!(provider_file_publication_observation_invalidated(&store)); + assert_eq!( + store.provider_file_publication_phase(&retirement).unwrap(), + ProviderFilePublicationPhase::Preparing + ); + store.abandon_provider_file_publication(retirement).unwrap(); +} + +#[test] +fn explicit_observation_invalidation_discards_an_unmutated_publication() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let file = source_file(10, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let outcome = source_outcome(&file, generation, 110); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + store.abandon_provider_file_publication(scope).unwrap(); + + let owner = store + .effective_provider_file_publication_inventory_owner() + .unwrap() + .unwrap(); + assert!(store + .invalidate_effective_provider_file_publication_observation(&owner, 120) + .unwrap()); + assert_eq!(table_row_count(&store, "provider_file_publications"), 0); + + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let restarted = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 125, + ) + .unwrap(); + assert!(!restarted.tracks_prior_material()); + assert!(matches!( + store.abort_provider_file_publication(restarted).unwrap(), + std::ops::ControlFlow::Continue(None) + )); +} diff --git a/crates/ctx-history-store/src/provider_files/tests/retirement_reconciliation.rs b/crates/ctx-history-store/src/provider_files/tests/retirement_reconciliation.rs index b9aa1be5..de0f0ee7 100644 --- a/crates/ctx-history-store/src/provider_files/tests/retirement_reconciliation.rs +++ b/crates/ctx-history-store/src/provider_files/tests/retirement_reconciliation.rs @@ -1,3 +1,318 @@ +#[test] +fn retirement_without_child_material_prepares_in_bounded_slices() { + const SOURCE_COUNT: u128 = 130; + + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(10, 100); + { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + for index in 0..SOURCE_COUNT { + insert_capture_source( + &store, + Uuid::from_u128(66_000 + index), + PATH_A, + &format!("retirement-preparation-{index}"), + ); + } + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 105, + ) + .unwrap(); + store + .conn + .execute( + "UPDATE provider_file_publications SET mutation_started = 1 WHERE replacement_id = ?1", + params![scope.scope_id.to_string()], + ) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 120, + generation, + ) + .unwrap(); + } + + let mut staged = 0; + let mut completed = false; + for cycle in 0..64 { + let store = Store::open(&path).unwrap(); + let scope = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 130 + cycle, + ) + .unwrap() + .unwrap(); + match store.provider_file_publication_phase(&scope).unwrap() { + ProviderFilePublicationPhase::Preparing => { + let progress = store + .prepare_provider_file_publication_slice(&scope, 64) + .unwrap(); + assert!(progress.rows_processed <= 64, "{progress:?}"); + staged += progress.source_ids_staged; + } + ProviderFilePublicationPhase::Reconciling => { + let progress = store + .reconcile_provider_file_publication_slice(&scope, 64) + .unwrap(); + assert!(progress.rows_scanned <= 64, "{progress:?}"); + } + ProviderFilePublicationPhase::ReadyToFinalize => { + store.retire_provider_file_publication(scope).unwrap(); + completed = true; + break; + } + ProviderFilePublicationPhase::Importing => { + panic!("retirement entered importer phase") + } + } + assert!(store + .abandon_provider_file_publication(scope) + .unwrap() + .is_none()); + } + + assert_eq!(staged, SOURCE_COUNT as usize); + assert!(completed); + let store = Store::open(&path).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); +} + +#[test] +fn completed_empty_retirement_preparation_survives_reopen() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(10, 100); + { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 105, + ) + .unwrap(); + store + .conn + .execute( + "UPDATE provider_file_publications SET mutation_started = 1 WHERE replacement_id = ?1", + params![scope.scope_id.to_string()], + ) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 120, + generation, + ) + .unwrap(); + } + + { + let store = Store::open(&path).unwrap(); + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 125, + ) + .unwrap() + .unwrap(); + let progress = store + .prepare_provider_file_publication_slice(&retirement, 1) + .unwrap(); + assert_eq!(progress.source_ids_staged, 0); + assert!(progress.complete); + assert_eq!( + store.provider_file_publication_phase(&retirement).unwrap(), + ProviderFilePublicationPhase::ReadyToFinalize + ); + store.abandon_provider_file_publication(retirement).unwrap(); + } + + let store = Store::open(&path).unwrap(); + let retirement = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 130, + ) + .unwrap() + .unwrap(); + assert_eq!( + store.provider_file_publication_phase(&retirement).unwrap(), + ProviderFilePublicationPhase::ReadyToFinalize + ); + store.retire_provider_file_publication(retirement).unwrap(); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert_eq!(table_row_count(&store, "source_import_files"), 0); +} + +#[test] +fn retirement_reconciliation_discards_seen_candidates_across_reopen_cycles() { + const EVENT_COUNT: u128 = 130; + + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let file = source_file(10, 100); + let source = Uuid::from_u128(67_000); + { + let store = Store::open(&path).unwrap(); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + insert_capture_source(&store, source, PATH_A, "retirement-retained"); + for index in 0..EVENT_COUNT { + insert_raw_event( + &store, + Uuid::from_u128(67_100 + index), + index as i64, + source, + &format!("retained event {index}"), + ); + } + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 110).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + prepare_all(&store, &scope, 64); + for index in 0..EVENT_COUNT { + store + .track_provider_file_publication_event(Uuid::from_u128(67_100 + index)) + .unwrap(); + } + let mut mutation = event_fixture( + Uuid::from_u128(67_500), + 500, + source, + "retirement-retained".to_owned(), + "retained mutation", + ); + mutation.dedupe_key = None; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_event(&mutation)) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Break(None) + )); + store + .mark_source_import_missing_paths_stale( + file.provider, + &file.source_root, + &[], + 120, + generation, + ) + .unwrap(); + } + + let mut completed = false; + let mut cycles = 0; + for cycle in 0..32 { + let store = Store::open(&path).unwrap(); + let scope = store + .begin_provider_file_publication_retirement( + file.provider, + MATERIAL_FORMAT, + &file.source_root, + &file.source_path, + 130 + cycle, + ) + .unwrap() + .unwrap(); + if cycle == 0 { + let staged_events: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM provider_file_publication_seen \ + WHERE replacement_id = ?1 AND entity_kind = 'event'", + params![scope.scope_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(staged_events, EVENT_COUNT as i64 + 1); + } + let preparation = store + .prepare_provider_file_publication_slice(&scope, 64) + .unwrap(); + assert!(preparation.rows_processed <= 64); + if !preparation.complete { + cycles += 1; + store.abandon_provider_file_publication(scope).unwrap(); + continue; + } + let progress = store + .reconcile_provider_file_publication_slice(&scope, 64) + .unwrap(); + cycles += 1; + if progress.complete { + store.retire_provider_file_publication(scope).unwrap(); + completed = true; + break; + } + assert!(store + .abandon_provider_file_publication(scope) + .unwrap() + .is_none()); + } + + assert!(completed, "bounded retirement did not converge"); + assert!(cycles >= 3); + let store = Store::open(&path).unwrap(); + assert_eq!(table_row_count(&store, "events"), 0); + assert!(!store.has_pending_provider_file_publications().unwrap()); +} + #[test] fn tombstoned_unmutated_marker_does_not_block_ordinary_owner_entity_writes() { let temp = tempdir().unwrap(); @@ -46,6 +361,7 @@ fn tombstoned_unmutated_marker_does_not_block_ordinary_owner_entity_writes() { .unwrap(); assert_eq!(observer.get_session(session).unwrap().id, session); } + #[test] fn process_crash_tombstone_retirement_restarts_and_is_idempotent() { let temp = tempdir().unwrap(); @@ -107,7 +423,7 @@ fn process_crash_tombstone_retirement_restarts_and_is_idempotent() { } { - let adopter = Store::open(&path).unwrap(); + let mut adopter = Store::open(&path).unwrap(); let retirement = adopter .begin_provider_file_publication_retirement( file.provider, @@ -125,6 +441,16 @@ fn process_crash_tombstone_retirement_restarts_and_is_idempotent() { .unwrap_err(), StoreError::InvalidProviderFilePublicationScope )); + assert!(matches!( + adopter + .with_provider_file_publication_writes_mut::<(), StoreError>( + &retirement, + |_| Ok(()), + ) + .unwrap_err(), + StoreError::InvalidProviderFilePublicationScope + )); + assert!(adopter.provider_file_write_scope.get().is_none()); prepare_all(&adopter, &retirement, 1); adopter .reconcile_provider_file_publication_slice(&retirement, 1) @@ -247,7 +573,7 @@ fn mutated_publication_with_permanently_missing_observation_can_retire() { } #[test] -fn observation_revival_before_retirement_finalization_preserves_the_marker() { +fn newer_generation_reclaims_retirement_marker_through_fresh_import() { let temp = tempdir().unwrap(); let path = temp.path().join("work.sqlite"); let file = source_file(20, 100); @@ -324,10 +650,8 @@ fn observation_revival_before_retirement_finalization_preserves_the_marker() { .upsert_source_import_files(revived_generation, std::slice::from_ref(&file)) .unwrap(); assert!(matches!( - retiring - .retire_provider_file_publication(retirement) - .unwrap_err(), - StoreError::ProviderFileObservationChanged { .. } + retiring.retire_provider_file_publication(retirement), + Err(StoreError::ProviderFileObservationChanged { .. }) )); assert!(reviver.has_pending_provider_file_publications().unwrap()); assert!(reviver.list_events().unwrap().is_empty()); @@ -338,11 +662,12 @@ fn observation_revival_before_retirement_finalization_preserves_the_marker() { file.provider, outcome.observation, MATERIAL_FORMAT, - ProviderFilePublicationKind::Incremental, + ProviderFilePublicationKind::Replacement, 135, ) .unwrap(); assert_eq!(adopted.kind(), ProviderFilePublicationKind::Replacement); + prepare_all(&reviver, &adopted, 1); reconcile_all(&reviver, &adopted, 1); reviver .finalize_provider_file_publication( diff --git a/crates/ctx-history-store/src/provider_files/tests/semantics_and_ownership.rs b/crates/ctx-history-store/src/provider_files/tests/semantics_and_ownership.rs index 1ac2cce2..92c9ca57 100644 --- a/crates/ctx-history-store/src/provider_files/tests/semantics_and_ownership.rs +++ b/crates/ctx-history-store/src/provider_files/tests/semantics_and_ownership.rs @@ -79,6 +79,7 @@ fn equal_count_replacement_advances_semantic_replacement_revision() { assert_eq!(store.cached_event_embedding_document_count().unwrap(), None); assert_eq!(store.semantic_replacement_revision().unwrap(), 1); } + #[test] fn semantic_cache_refresh_and_publication_marker_are_one_serialized_snapshot() { let temp = tempdir().unwrap(); @@ -145,6 +146,7 @@ fn semantic_cache_refresh_and_publication_marker_are_one_serialized_snapshot() { None ); } + #[test] fn replacement_owned_event_conflict_overwrites_and_cross_source_conflict_rejects() { let temp = tempdir().unwrap(); @@ -285,8 +287,9 @@ fn replacement_owned_event_conflict_overwrites_and_cross_source_conflict_rejects assert!(row_exists(&store, "events", owner_id)); assert!(row_exists(&store, "events", sibling_id)); } + #[test] -fn scoped_natural_key_and_unowned_record_writes_cannot_contaminate_publication() { +fn scoped_natural_key_and_existing_record_writes_cannot_contaminate_publication() { let temp = tempdir().unwrap(); let store = Store::open(temp.path().join("work.sqlite")).unwrap(); let file = source_file(20, 100); @@ -377,20 +380,39 @@ fn scoped_natural_key_and_unowned_record_writes_cannot_contaminate_publication() ("sibling".to_owned(), sibling_source.to_string()) ); - let record = ctx_history_core::HistoryRecord::new( - "unowned write", - "must fail closed", + let mut record = ctx_history_core::HistoryRecord::new( + "new record shell", + "created by the exact active publication scope", Vec::new(), "note", None, ); + record.id = Uuid::from_u128(45_005); + assert!(!store + .provider_file_publication_history_record_exists(&scope, record.id) + .unwrap()); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_record(&record)) + .unwrap(); + assert!(store + .provider_file_publication_history_record_exists(&scope, record.id) + .unwrap()); + record.title = "existing record mutation must fail".to_owned(); assert!(matches!( store .with_provider_file_publication_writes(&scope, |store| store.upsert_record(&record)) .unwrap_err(), StoreError::ProviderFilePublicationOwnerMismatch { .. } )); - let busy = store.upsert_record(&record).unwrap_err(); + let mut busy_record = ctx_history_core::HistoryRecord::new( + "foreground write", + "must fail while publication is active", + Vec::new(), + "note", + None, + ); + busy_record.id = Uuid::from_u128(45_006); + let busy = store.upsert_record(&busy_record).unwrap_err(); assert!(matches!( busy, StoreError::ProviderFileReplacementBusy { .. } @@ -398,9 +420,164 @@ fn scoped_natural_key_and_unowned_record_writes_cannot_contaminate_publication() let rendered = busy.to_string(); assert!(!rendered.contains(ROOT)); assert!(!rendered.contains(PATH_A)); - assert!(!row_exists(&store, "history_records", record.id)); + assert!(!row_exists(&store, "history_records", busy_record.id)); + store.abandon_provider_file_publication(scope).unwrap(); +} + +#[test] +fn child_owned_session_and_explicit_edge_allow_sibling_parent_references() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (scope, child_source, _, parent_session, _, _) = + begin_parent_child_ownership_publication(&store); + let child_session = Uuid::from_u128(80_010); + let mut child = session_fixture(child_session, child_source, "publication-child"); + child.parent_session_id = Some(parent_session); + child.root_session_id = Some(parent_session); + child.agent_type = AgentType::Subagent; + child.is_primary = false; + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_session(&child)) + .unwrap(); + + let edge_id = Uuid::from_u128(80_011); + let edge = session_edge_fixture(edge_id, child_session, parent_session, Some(child_source)); + store + .with_provider_file_publication_writes(&scope, |store| store.upsert_session_edge(&edge)) + .unwrap(); + let inherited_edge_id = Uuid::from_u128(80_012); + let inherited_edge = + session_edge_fixture(inherited_edge_id, child_session, parent_session, None); + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_session_edge(&inherited_edge) + }) + .unwrap(); + + let stored_session: (String, String, String) = store + .conn + .query_row( + "SELECT capture_source_id, parent_session_id, root_session_id FROM sessions WHERE id = ?1", + params![child_session.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + stored_session, + ( + child_source.to_string(), + parent_session.to_string(), + parent_session.to_string(), + ) + ); + let stored_edge_source: String = store + .conn + .query_row( + "SELECT source_id FROM session_edges WHERE id = ?1", + params![edge_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored_edge_source, child_source.to_string()); + let inherited_edge_source: Option = store + .conn + .query_row( + "SELECT source_id FROM session_edges WHERE id = ?1", + params![inherited_edge_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(inherited_edge_source, None); + store.abandon_provider_file_publication(scope).unwrap(); +} + +#[test] +fn session_and_edge_owners_reject_sibling_overwrites_and_missing_sources() { + let temp = tempdir().unwrap(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let (scope, child_source, sibling_source, parent_session, sibling_peer, sibling_edge) = + begin_parent_child_ownership_publication(&store); + + let mut hijacked_parent = store.get_session(parent_session).unwrap(); + hijacked_parent.capture_source_id = Some(child_source); + hijacked_parent.role_hint = Some("must not overwrite sibling owner".to_owned()); + assert!(matches!( + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_session(&hijacked_parent) + }) + .unwrap_err(), + StoreError::ProviderFilePublicationOwnerMismatch { .. } + )); + let stored_parent_source: String = store + .conn + .query_row( + "SELECT capture_source_id FROM sessions WHERE id = ?1", + params![parent_session.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored_parent_source, sibling_source.to_string()); + + let hijacked_edge = session_edge_fixture( + sibling_edge, + parent_session, + sibling_peer, + Some(child_source), + ); + assert!(matches!( + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_session_edge(&hijacked_edge) + }) + .unwrap_err(), + StoreError::ProviderFilePublicationOwnerMismatch { .. } + )); + let stored_edge_source: String = store + .conn + .query_row( + "SELECT source_id FROM session_edges WHERE id = ?1", + params![sibling_edge.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored_edge_source, sibling_source.to_string()); + + let missing_session_id = Uuid::from_u128(80_020); + let mut missing_session = + session_fixture(missing_session_id, child_source, "missing-session-owner"); + missing_session.capture_source_id = None; + missing_session.parent_session_id = Some(parent_session); + missing_session.root_session_id = Some(parent_session); + assert!(matches!( + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_session(&missing_session) + }) + .unwrap_err(), + StoreError::ProviderFilePublicationOwnerMismatch { .. } + )); + assert!(!row_exists(&store, "sessions", missing_session_id)); + + let missing_edge_id = Uuid::from_u128(80_021); + let missing_edge = session_edge_fixture( + missing_edge_id, + Uuid::from_u128(80_022), + Uuid::from_u128(80_023), + None, + ); + assert!(matches!( + store + .with_provider_file_publication_writes(&scope, |store| { + store.upsert_session_edge(&missing_edge) + }) + .unwrap_err(), + StoreError::ProviderFilePublicationOwnerMismatch { .. } + )); + assert!(!row_exists(&store, "session_edges", missing_edge_id)); store.abandon_provider_file_publication(scope).unwrap(); } + #[test] fn legacy_root_effective_session_owner_is_hidden_and_seen_empty_session_survives() { let temp = tempdir().unwrap(); @@ -498,6 +675,158 @@ fn legacy_root_effective_session_owner_is_hidden_and_seen_empty_session_survives assert_eq!(session_deleted_at(&store, session), None); assert!(store.get_session(session).is_ok()); } + +#[test] +fn replacement_identity_reads_are_idempotent_and_scoped_to_the_active_file() { + let temp = tempdir().unwrap(); + let path = temp.path().join("work.sqlite"); + let store = Store::open(&path).unwrap(); + let file_a = source_file(20, 100); + let mut file_b = source_file(20, 100); + file_b.source_path = PATH_B.to_owned(); + let generation = store + .allocate_source_import_inventory_generation(file_a.provider, &file_a.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, &[file_a.clone(), file_b.clone()]) + .unwrap(); + + let source_a = Uuid::from_u128(329_100); + let source_b = Uuid::from_u128(329_101); + let session_a = Uuid::from_u128(329_102); + let session_b = Uuid::from_u128(329_103); + let event_a = Uuid::from_u128(329_104); + let event_b = Uuid::from_u128(329_105); + insert_capture_source(&store, source_a, PATH_A, "shared-external-session"); + insert_capture_source(&store, source_b, PATH_B, "shared-external-session"); + let session_a_row = session_fixture(session_a, source_a, "shared-external-session"); + let session_b_row = session_fixture(session_b, source_b, "shared-external-session"); + store.upsert_session(&session_a_row).unwrap(); + store.upsert_session(&session_b_row).unwrap(); + let mut event_a_row = event_fixture( + event_a, + 1, + source_a, + "owner-event-dedupe".to_owned(), + "owner event", + ); + event_a_row.session_id = Some(session_a); + let mut event_b_row = event_fixture( + event_b, + 2, + source_b, + "sibling-event-dedupe".to_owned(), + "sibling event", + ); + event_b_row.session_id = Some(session_b); + store.upsert_event(&event_a_row).unwrap(); + store.upsert_event(&event_b_row).unwrap(); + + let outcome_a = source_outcome(&file_a, generation, 110); + let scope = store + .begin_provider_file_publication( + file_a.provider, + outcome_a.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 105, + ) + .unwrap(); + prepare_all(&store, &scope, 1); + + assert!(store.get_session(session_a).is_err()); + assert!(store.get_session(session_b).is_ok()); + assert!(store.get_event(event_a).is_err()); + assert!(store.get_event(event_b).is_ok()); + let active_owner = + material_source_matches_replacement_owner_predicate("active_source", "active_publication"); + let sibling_matches_active_owner: bool = store + .conn + .query_row( + &format!( + "SELECT EXISTS (SELECT 1 FROM capture_sources AS active_source \ + JOIN provider_file_publications AS active_publication ON ({active_owner}) \ + WHERE active_publication.replacement_id = ?1 AND active_source.id = ?2)" + ), + params![scope.scope_id.to_string(), source_b.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert!(!sibling_matches_active_owner); + store + .with_provider_file_publication_writes(&scope, |store| { + assert_eq!(store.get_capture_source(source_a)?.id, source_a); + assert_eq!(store.get_session(session_a)?.id, session_a); + assert_eq!(store.get_session(session_b)?.id, session_b); + assert_eq!(store.get_event(event_a)?.id, event_a); + assert_eq!(store.get_event(event_b)?.id, event_b); + assert_eq!( + store + .session_by_capture_source_and_external_session( + source_a, + CaptureProvider::Claude, + "shared-external-session", + )? + .unwrap() + .id, + session_a + ); + assert_eq!( + store + .session_by_capture_source_and_external_session( + source_b, + CaptureProvider::Claude, + "shared-external-session", + )? + .unwrap() + .id, + session_b + ); + assert_eq!(store.event_id_by_dedupe_key("owner-event-dedupe")?, event_a); + assert_eq!( + store.event_id_by_dedupe_key("sibling-event-dedupe")?, + event_b + ); + assert_eq!(store.events_for_session_limited(session_a, 10)?.len(), 1); + assert_eq!(store.events_for_session_limited(session_b, 10)?.len(), 1); + + store.upsert_session(&session_a_row)?; + assert_eq!(store.upsert_event(&event_a_row)?, event_a); + Ok(()) + }) + .unwrap(); + + let owner_sessions: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM sessions WHERE capture_source_id = ?1", + params![source_a.to_string()], + |row| row.get(0), + ) + .unwrap(); + let owner_events: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM events WHERE capture_source_id = ?1", + params![source_a.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!((owner_sessions, owner_events), (1, 1)); + assert_eq!(staged_seen_count(&store), 2); + + reconcile_all(&store, &scope, 1); + store + .finalize_provider_file_publication( + scope, + outcome_a, + ProviderFilePublicationCommit::Replacement(None), + ) + .unwrap(); + assert!(store.get_session(session_a).is_ok()); + assert!(store.get_session(session_b).is_ok()); +} + #[test] fn replacement_removes_prior_file_material_but_preserves_shared_session() { let temp = tempdir().unwrap(); @@ -633,6 +962,7 @@ fn replacement_removes_prior_file_material_but_preserves_shared_session() { ); assert_eq!(store.semantic_replacement_revision().unwrap(), 1); } + #[test] fn replacement_preserves_sessions_with_each_kind_of_sibling_owned_material() { let temp = tempdir().unwrap(); diff --git a/crates/ctx-history-store/src/provider_files/tests/support_and_subprocess.rs b/crates/ctx-history-store/src/provider_files/tests/support_and_subprocess.rs index a31490cb..c7109386 100644 --- a/crates/ctx-history-store/src/provider_files/tests/support_and_subprocess.rs +++ b/crates/ctx-history-store/src/provider_files/tests/support_and_subprocess.rs @@ -319,6 +319,96 @@ fn session_fixture(id: Uuid, source_id: Uuid, external_session_id: &str) -> Sess } } +fn session_edge_fixture( + id: Uuid, + from_session_id: Uuid, + to_session_id: Uuid, + source_id: Option, +) -> SessionEdge { + let now = DateTime::parse_from_rfc3339("2026-07-14T12:00:00Z") + .unwrap() + .with_timezone(&Utc); + SessionEdge { + id, + from_session_id, + to_session_id, + edge_type: SessionEdgeType::ParentChild, + confidence: Confidence::Explicit, + source_id, + timestamps: EntityTimestamps { + created_at: now, + updated_at: now, + }, + sync: SyncMetadata { + visibility: Visibility::LocalOnly, + fidelity: Fidelity::Imported, + sync_state: SyncState::LocalOnly, + sync_version: 0, + deleted_at: None, + metadata: json!({}), + }, + } +} + +fn begin_parent_child_ownership_publication( + store: &Store, +) -> (ProviderFilePublicationScope, Uuid, Uuid, Uuid, Uuid, Uuid) { + let file = source_file(20, 100); + let generation = store + .allocate_source_import_inventory_generation(file.provider, &file.source_root) + .unwrap(); + store + .upsert_source_import_files(generation, std::slice::from_ref(&file)) + .unwrap(); + + let child_source = Uuid::from_u128(80_000); + let sibling_source = Uuid::from_u128(80_001); + let parent_session = Uuid::from_u128(80_002); + let sibling_peer = Uuid::from_u128(80_003); + let sibling_edge = Uuid::from_u128(80_004); + insert_capture_source(store, child_source, PATH_A, "publication-child"); + insert_capture_source( + store, + sibling_source, + "/history/claude/projects/parent.jsonl", + "publication-parent", + ); + insert_raw_session(store, parent_session, sibling_source, "publication-parent"); + insert_raw_session( + store, + sibling_peer, + sibling_source, + "publication-sibling-peer", + ); + store + .upsert_session_edge(&session_edge_fixture( + sibling_edge, + parent_session, + sibling_peer, + Some(sibling_source), + )) + .unwrap(); + + let outcome = source_outcome(&file, generation, 120); + let scope = store + .begin_provider_file_publication( + file.provider, + outcome.observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Incremental, + 110, + ) + .unwrap(); + ( + scope, + child_source, + sibling_source, + parent_session, + sibling_peer, + sibling_edge, + ) +} + fn event_fixture(id: Uuid, seq: u64, source_id: Uuid, dedupe_key: String, text: &str) -> Event { Event { id, @@ -366,22 +456,37 @@ fn insert_raw_session(store: &Store, session_id: Uuid, source_id: Uuid, external } fn staged_seen_count(store: &Store) -> i64 { + let replacement_id = store + .provider_file_publication + .borrow() + .as_ref() + .unwrap() + .scope_id + .to_string(); store .conn .query_row( - "SELECT COUNT(*) FROM provider_replacement_stage.seen", - [], + "SELECT COUNT(*) FROM provider_file_publication_seen WHERE replacement_id = ?1", + params![replacement_id], |row| row.get(0), ) .unwrap() } fn staged_prior_source_count(store: &Store) -> i64 { + let replacement_id = store + .provider_file_publication + .borrow() + .as_ref() + .unwrap() + .scope_id + .to_string(); store .conn .query_row( - "SELECT COUNT(*) FROM provider_replacement_stage.prior_sources", - [], + "SELECT COUNT(*) FROM provider_file_publication_prior_sources \ + WHERE replacement_id = ?1", + params![replacement_id], |row| row.get(0), ) .unwrap() @@ -398,22 +503,6 @@ fn main_table_exists(store: &Store, table: &str) -> bool { .unwrap() } -fn pragma_i64(store: &Store, pragma: &str) -> i64 { - store.conn.query_row(pragma, [], |row| row.get(0)).unwrap() -} - -fn main_database_footprint(store: &Store, path: &std::path::Path) -> (i64, i64, u64, u64) { - let page_count = pragma_i64(store, "PRAGMA main.page_count"); - let freelist_count = pragma_i64(store, "PRAGMA main.freelist_count"); - let main_bytes = std::fs::metadata(path).unwrap().len(); - let mut wal_path = path.as_os_str().to_os_string(); - wal_path.push("-wal"); - let wal_bytes = std::fs::metadata(std::path::PathBuf::from(wal_path)) - .map(|metadata| metadata.len()) - .unwrap_or(0); - (page_count, freelist_count, main_bytes, wal_bytes) -} - fn table_row_count(store: &Store, table: &str) -> i64 { store .conn @@ -425,6 +514,7 @@ fn table_row_count(store: &Store, table: &str) -> i64 { fn reconcile_all(store: &Store, scope: &ProviderFilePublicationScope, max_rows: usize) { prepare_all(store, scope, max_rows); + stage_test_completion(store, scope); loop { let progress = store .reconcile_provider_file_publication_slice(scope, max_rows) @@ -436,6 +526,26 @@ fn reconcile_all(store: &Store, scope: &ProviderFilePublicationScope, max_rows: } } +fn stage_test_completion(store: &Store, scope: &ProviderFilePublicationScope) { + if scope.retires_observation + || store + .load_provider_file_publication_completion(scope) + .unwrap() + .is_some() + { + return; + } + store + .stage_provider_file_publication_completion( + scope, + &ProviderFilePublicationCompletion { + version: 1, + payload: json!({"test": "complete"}), + }, + ) + .unwrap(); +} + fn prepare_all(store: &Store, scope: &ProviderFilePublicationScope, max_rows: usize) { loop { let progress = store @@ -579,6 +689,29 @@ fn provider_file_subprocess_helper() { StoreError::ProviderFileReplacementBusy { .. } )); } + "non-utf8-private-root" => { + let store = Store::open(&store_path).unwrap(); + let generation = std::env::var("CTX_PROVIDER_FILE_HELPER_GENERATION") + .unwrap() + .parse::() + .unwrap(); + let file = source_file(20, 100); + let scope = store + .begin_provider_file_publication( + file.provider, + source_outcome(&file, generation, 120).observation, + MATERIAL_FORMAT, + ProviderFilePublicationKind::Replacement, + 110, + ) + .unwrap(); + assert!(matches!( + store.abort_provider_file_publication(scope).unwrap(), + std::ops::ControlFlow::Continue(None) + )); + assert!(!store.has_pending_provider_file_publications().unwrap()); + assert_eq!(store.list_events().unwrap().len(), 1); + } "partial-crash" => { let store = Store::open(&store_path).unwrap(); let generation = std::env::var("CTX_PROVIDER_FILE_HELPER_GENERATION") @@ -599,6 +732,7 @@ fn provider_file_subprocess_helper() { ) .unwrap(); prepare_all(&store, &scope, 1); + stage_test_completion(&store, &scope); for _ in 0..100 { store .reconcile_provider_file_publication_slice(&scope, 1) diff --git a/crates/ctx-history-store/src/provider_files/tests/visibility_fencing.rs b/crates/ctx-history-store/src/provider_files/tests/visibility_fencing.rs index 8f12c291..42a7be18 100644 --- a/crates/ctx-history-store/src/provider_files/tests/visibility_fencing.rs +++ b/crates/ctx-history-store/src/provider_files/tests/visibility_fencing.rs @@ -99,6 +99,7 @@ fn pending_owner_is_hidden_from_views_external_hydration_and_export() { assert_eq!(observer.list_events().unwrap().len(), 1); assert!(observer.file_touched_exists(touched).unwrap()); } + #[test] fn durable_marker_blocks_cross_connection_entity_and_archive_contamination() { let temp = tempdir().unwrap(); @@ -235,6 +236,7 @@ fn durable_marker_blocks_cross_connection_entity_and_archive_contamination() { ); store.abandon_provider_file_publication(scope).unwrap(); } + #[test] fn raw_sql_rejects_base_table_reads_while_crash_marker_fences_owner() { let temp = tempdir().unwrap(); @@ -287,6 +289,7 @@ fn raw_sql_rejects_base_table_reads_while_crash_marker_fences_owner() { )); } } + #[test] fn first_owner_write_in_pre_marker_window_is_serialized_into_reconciliation() { let temp = tempdir().unwrap(); @@ -354,6 +357,7 @@ fn first_owner_write_in_pre_marker_window_is_serialized_into_reconciliation() { assert_eq!(counts.sessions_tombstoned, 1); assert!(session_deleted_at(&writer, session_id).is_some()); } + #[test] fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_releases_them() { let temp = tempdir().unwrap(); @@ -385,7 +389,7 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele file.provider, match pending_source_outcome.observation { ProviderFileInventoryObservation::SourceImport { update, .. } => update, - ProviderFileInventoryObservation::Catalog { .. } => unreachable!(), + ProviderFileInventoryObservation::ObservedCatalog { .. } => unreachable!(), }, CatalogIndexedStatus::Failed, Some("must remain pending"), @@ -411,7 +415,7 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele .unwrap(); let next_source_update = match source_outcome(&file, next_source_generation, 130).observation { ProviderFileInventoryObservation::SourceImport { update, .. } => update, - ProviderFileInventoryObservation::Catalog { .. } => unreachable!(), + ProviderFileInventoryObservation::ObservedCatalog { .. } => unreachable!(), }; assert_eq!( observer @@ -441,7 +445,7 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele file_modified_at_ms: 100, import_revision: 1, cataloged_at_ms: 101, - metadata: json!({}), + metadata: json!({"file_observation_token_v1": "visibility-catalog-token"}), }; let catalog_generation = observer .allocate_catalog_inventory_generation(catalog.provider, &catalog.source_root) @@ -469,9 +473,10 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele }; let catalog_outcome = ProviderFileImportOutcome { provider: catalog.provider, - observation: ProviderFileInventoryObservation::Catalog { + observation: ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: catalog_update, + metadata: &catalog.metadata, }, status: CatalogIndexedStatus::Indexed, error: None, @@ -497,9 +502,10 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele .unwrap()); assert!(matches!( catalog_writer - .record_catalog_source_import_result( + .record_observed_catalog_source_import_result( catalog.provider, catalog_update, + &catalog.metadata, CatalogIndexedStatus::Failed, Some("must remain pending"), ) @@ -537,9 +543,10 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele }; assert_eq!( catalog_writer - .record_catalog_source_import_result( + .record_observed_catalog_source_import_result( catalog.provider, next_catalog_update, + &catalog.metadata, CatalogIndexedStatus::Failed, Some("new generation result"), ) @@ -547,6 +554,7 @@ fn catalog_results_are_fenced_across_connections_and_unmutated_supersession_rele 1 ); } + #[test] fn source_import_publication_blocks_cross_family_catalog_status_and_legacy_cursor() { let temp = tempdir().unwrap(); @@ -591,9 +599,10 @@ fn source_import_publication_blocks_cross_family_catalog_status_and_legacy_curso }; assert!(matches!( observer - .record_catalog_source_import_result( + .record_observed_catalog_source_import_result( catalog.provider, update, + &catalog.metadata, CatalogIndexedStatus::Indexed, None, ) @@ -616,6 +625,7 @@ fn source_import_publication_blocks_cross_family_catalog_status_and_legacy_curso assert_eq!(status, "pending"); assert_eq!(cursor, (None, None, None, None, None)); } + #[test] fn catalog_publication_blocks_cross_family_source_import_status() { let temp = tempdir().unwrap(); @@ -649,9 +659,10 @@ fn catalog_publication_blocks_cross_family_source_import_status() { let catalog_scope = store .begin_provider_file_publication( catalog.provider, - ProviderFileInventoryObservation::Catalog { + ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: catalog_update, + metadata: &catalog.metadata, }, MATERIAL_FORMAT, ProviderFilePublicationKind::Replacement, @@ -663,7 +674,7 @@ fn catalog_publication_blocks_cross_family_source_import_status() { let observer = Store::open(&path).unwrap(); let source_update = match source_outcome(&file, source_generation, 130).observation { ProviderFileInventoryObservation::SourceImport { update, .. } => update, - ProviderFileInventoryObservation::Catalog { .. } => unreachable!(), + ProviderFileInventoryObservation::ObservedCatalog { .. } => unreachable!(), }; assert!(matches!( observer @@ -686,6 +697,7 @@ fn catalog_publication_blocks_cross_family_source_import_status() { .unwrap(); assert_eq!(status, "pending"); } + #[test] fn superseded_mutated_catalog_publication_keeps_new_generation_noncurrent() { let temp = tempdir().unwrap(); @@ -711,9 +723,10 @@ fn superseded_mutated_catalog_publication_keeps_new_generation_noncurrent() { let scope = store .begin_provider_file_publication( catalog.provider, - ProviderFileInventoryObservation::Catalog { + ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: first_update, + metadata: &catalog.metadata, }, MATERIAL_FORMAT, ProviderFilePublicationKind::Replacement, @@ -744,6 +757,7 @@ fn superseded_mutated_catalog_publication_keeps_new_generation_noncurrent() { .unwrap()); drop(scope); } + #[test] fn obsolete_unmutated_catalog_publication_does_not_fence_new_generation() { let temp = tempdir().unwrap(); @@ -769,9 +783,10 @@ fn obsolete_unmutated_catalog_publication_does_not_fence_new_generation() { let scope = store .begin_provider_file_publication( catalog.provider, - ProviderFileInventoryObservation::Catalog { + ProviderFileInventoryObservation::ObservedCatalog { source_format: &catalog.source_format, update: first_update, + metadata: &catalog.metadata, }, MATERIAL_FORMAT, ProviderFilePublicationKind::Replacement, @@ -794,6 +809,7 @@ fn obsolete_unmutated_catalog_publication_does_not_fence_new_generation() { .unwrap()); drop(scope); } + #[test] fn obsolete_unmutated_marker_does_not_block_raw_sql_or_unrelated_archive() { let temp = tempdir().unwrap(); diff --git a/crates/ctx-history-store/src/provider_files/types.rs b/crates/ctx-history-store/src/provider_files/types.rs index dd0d8f5f..aa0ba219 100644 --- a/crates/ctx-history-store/src/provider_files/types.rs +++ b/crates/ctx-history-store/src/provider_files/types.rs @@ -1,10 +1,22 @@ const CATALOG_INVENTORY_FAMILY: &str = "catalog_sessions"; const SOURCE_IMPORT_INVENTORY_FAMILY: &str = "source_import_files"; -const STAGING_SCHEMA: &str = "provider_replacement_stage"; const STAGING_DIR_PREFIX: &str = "stage"; +const STAGING_SEEN_TABLE: &str = "provider_file_publication_seen"; +const STAGING_PRIOR_SOURCES_TABLE: &str = "provider_file_publication_prior_sources"; +const STAGING_BATCH_TABLE: &str = "provider_file_publication_batch"; +const CURRENT_CAPTURE_SOURCE_KIND: &str = "capture_source"; +const PRIOR_CAPTURE_SOURCE_KIND: &str = "prior_capture_source"; +const PRIOR_HISTORY_RECORD_KIND: &str = "prior_history_record"; +const PRIOR_HISTORY_RECORD_CURSOR: &str = "__prior_history_record__"; +const PRIOR_CAPTURE_SOURCE_CURSOR: &str = "__prior_capture_source__"; +const RETIREMENT_RESET_BATCH_CURSOR: &str = "__retirement_reset_batch__"; +const RETIREMENT_RESET_HISTORY_RECORD_CURSOR: &str = "__retirement_reset_history_record__"; +const RETIREMENT_RESET_CAPTURE_SOURCE_CURSOR: &str = "__retirement_reset_capture_source__"; +const RETIREMENT_RESET_SEEN_CURSOR: &str = "__retirement_reset_seen__"; pub const PROVIDER_FILE_PREPARATION_MAX_ROWS: usize = 100_000; pub const PROVIDER_FILE_RECONCILIATION_MAX_ROWS: usize = 100_000; pub const PROVIDER_FILE_CHECKPOINT_RESUME_STATE_MAX_BYTES: usize = 64 * 1024; +pub const PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES: usize = 256 * 1024; const CLEANUP_PHASE_LINKS: i64 = 0; const CLEANUP_PHASE_FILES: i64 = 1; const CLEANUP_PHASE_EDGES: i64 = 2; @@ -21,6 +33,16 @@ const CLEANUP_PHASE_VCS_WORKSPACES: i64 = 12; const CLEANUP_PHASE_AUDIT_LOG: i64 = 13; const CLEANUP_PHASE_COMPLETE: i64 = 14; +fn is_retirement_reset_cursor(cursor: &str) -> bool { + matches!( + cursor, + RETIREMENT_RESET_BATCH_CURSOR + | RETIREMENT_RESET_HISTORY_RECORD_CURSOR + | RETIREMENT_RESET_CAPTURE_SOURCE_CURSOR + | RETIREMENT_RESET_SEEN_CURSOR + ) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProviderFileCheckpointKey<'a> { pub provider: CaptureProvider, @@ -59,9 +81,10 @@ impl ProviderFileCheckpoint { #[derive(Debug, Clone, Copy)] pub enum ProviderFileInventoryObservation<'a> { - Catalog { + ObservedCatalog { source_format: &'a str, update: CatalogSourceIndexUpdate<'a>, + metadata: &'a serde_json::Value, }, SourceImport { source_format: &'a str, @@ -72,69 +95,79 @@ pub enum ProviderFileInventoryObservation<'a> { impl<'a> ProviderFileInventoryObservation<'a> { fn source_format(self) -> &'a str { match self { - Self::Catalog { source_format, .. } | Self::SourceImport { source_format, .. } => { - source_format - } + Self::ObservedCatalog { source_format, .. } + | Self::SourceImport { source_format, .. } => source_format, } } fn source_root(self) -> &'a str { match self { - Self::Catalog { update, .. } => update.source_root, + Self::ObservedCatalog { update, .. } => update.source_root, Self::SourceImport { update, .. } => update.source_root, } } fn source_path(self) -> &'a str { match self { - Self::Catalog { update, .. } => update.source_path, + Self::ObservedCatalog { update, .. } => update.source_path, Self::SourceImport { update, .. } => update.source_path, } } fn file_size_bytes(self) -> u64 { match self { - Self::Catalog { update, .. } => update.file_size_bytes, + Self::ObservedCatalog { update, .. } => update.file_size_bytes, Self::SourceImport { update, .. } => update.file_size_bytes, } } fn file_modified_at_ms(self) -> i64 { match self { - Self::Catalog { update, .. } => update.file_modified_at_ms, + Self::ObservedCatalog { update, .. } => update.file_modified_at_ms, Self::SourceImport { update, .. } => update.file_modified_at_ms, } } fn import_revision(self) -> u32 { match self { - Self::Catalog { update, .. } => update.import_revision, + Self::ObservedCatalog { update, .. } => update.import_revision, Self::SourceImport { update, .. } => update.import_revision, } } fn inventory_generation(self) -> u64 { match self { - Self::Catalog { update, .. } => update.inventory_generation, + Self::ObservedCatalog { update, .. } => update.inventory_generation, Self::SourceImport { update, .. } => update.inventory_generation, } } fn inventory_family(self) -> &'static str { match self { - Self::Catalog { .. } => CATALOG_INVENTORY_FAMILY, + Self::ObservedCatalog { .. } => CATALOG_INVENTORY_FAMILY, Self::SourceImport { .. } => SOURCE_IMPORT_INVENTORY_FAMILY, } } fn metadata_json(self) -> Result> { match self { - Self::Catalog { .. } => Ok(None), + Self::ObservedCatalog { metadata, .. } => serde_json::to_string(metadata) + .map(Some) + .map_err(Into::into), Self::SourceImport { update, .. } => serde_json::to_string(update.metadata) .map(Some) .map_err(Into::into), } } + + fn catalog_update(self) -> Option<(CatalogSourceIndexUpdate<'a>, &'a serde_json::Value)> { + match self { + Self::ObservedCatalog { + update, metadata, .. + } => Some((update, metadata)), + Self::SourceImport { .. } => None, + } + } } #[derive(Debug, Clone, Copy)] @@ -151,6 +184,50 @@ pub enum ProviderFilePublicationKind { Replacement, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderFilePublicationPhase { + Preparing, + Importing, + Reconciling, + ReadyToFinalize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderFilePublicationCompletion { + pub version: u32, + pub payload: serde_json::Value, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderFilePublicationRetirementWork { + pub provider: CaptureProvider, + pub material_source_format: String, + pub material_source_root: String, + pub source_path: String, + pub estimated_bytes: u64, + pub last_attempt_at_ms: i64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderFileInventoryFamily { + Catalog, + SourceImport, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderFilePublicationInventoryOwner { + pub provider: CaptureProvider, + pub inventory_family: ProviderFileInventoryFamily, + pub source_format: String, + pub source_root: String, + pub source_path: String, + pub inventory_generation: u64, + pub file_size_bytes: u64, + pub file_modified_at_ms: i64, + pub import_revision: u32, + pub metadata_json: Option, +} + impl ProviderFilePublicationKind { fn as_str(self) -> &'static str { match self { @@ -192,6 +269,7 @@ pub struct ProviderFilePublicationScope { owner_id: String, staging_id: String, tracks_prior_material: bool, + reuse_staging_state: bool, retires_observation: bool, lifecycle: Arc, _owner_lock: File, @@ -208,6 +286,10 @@ impl ProviderFilePublicationScope { pub fn kind(&self) -> ProviderFilePublicationKind { self.kind } + + pub fn tracks_prior_material(&self) -> bool { + self.tracks_prior_material + } } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -321,6 +403,7 @@ pub struct ProviderFileReconciliationProgress { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ProviderFilePreparationProgress { pub source_ids_staged: usize, + pub rows_processed: usize, pub complete: bool, } @@ -332,6 +415,20 @@ pub enum ProviderFileMaintenanceWarning { }, } +impl std::fmt::Display for ProviderFileMaintenanceWarning { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::StagingCleanupDeferred { + publication_id, + operation, + } => write!( + formatter, + "provider publication {publication_id} staging cleanup deferred during {operation}" + ), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProviderFileFinalizeOutcome { pub reconciliation: ProviderFileReconciliationCounts, @@ -349,12 +446,6 @@ pub(crate) struct ActiveProviderFilePublication { retires_observation: bool, _owner_lock_path: PathBuf, attached: bool, - staging_dir_path: Option, - staging_path: Option, - #[cfg(test)] - staging_file_mode: Option, - #[cfg(test)] - staging_dir_mode: Option, } struct ProviderFileWriteScopeReset<'a> { @@ -379,6 +470,7 @@ pub(crate) enum ProviderFileFaultPoint { BeginAfterStaging, MutationBeforeCommit, PreparationBeforeCommit, + CompletionBeforeCommit, FinalizeBeforeCommit, #[cfg(test)] RetirementFinalizeProcessExit, @@ -394,12 +486,14 @@ struct ReplacementMarker { cleanup_phase: i64, source_cursor: Option, entity_cursor: Option, + completion_payload_json: Option, counts: ProviderFileReconciliationCounts, } struct DurableProviderFilePublication { scope_id: Uuid, staging_id: String, + publication_kind: ProviderFilePublicationKind, inventory_family: &'static str, inventory_source_format: String, inventory_source_root: String, @@ -410,6 +504,8 @@ struct DurableProviderFilePublication { import_revision: u32, metadata_json: Option, mutation_started: bool, + tracks_prior_material: bool, + staging_initialized: bool, } struct ReconciliationBatch { diff --git a/crates/ctx-history-store/src/provider_files/validation.rs b/crates/ctx-history-store/src/provider_files/validation.rs index ae57e841..c80e41ab 100644 --- a/crates/ctx-history-store/src/provider_files/validation.rs +++ b/crates/ctx-history-store/src/provider_files/validation.rs @@ -17,6 +17,28 @@ fn validate_successful_outcome(outcome: ProviderFileImportOutcome<'_>) -> Result Ok(()) } +fn validate_provider_file_completion_outcome( + outcome: ProviderFileImportOutcome<'_>, + completion_kind: ProviderFileCompletionKind, + has_safe_checkpoint: bool, + tracks_prior_material: bool, +) -> Result<()> { + let completed = matches!( + outcome.status, + CatalogIndexedStatus::Indexed | CatalogIndexedStatus::CompletedWithRejections + ); + let terminal_replacement = outcome.status == CatalogIndexedStatus::Rejected + && completion_kind == ProviderFileCompletionKind::Replacement + && !has_safe_checkpoint + && !tracks_prior_material; + if !completed && !terminal_replacement { + return Err(StoreError::InvalidProviderFileCheckpoint( + "publication finalization requires a completed outcome or checkpoint-free rejected replacement", + )); + } + Ok(()) +} + fn validate_observation_identity(observation: ProviderFileInventoryObservation<'_>) -> Result<()> { if observation.source_format().is_empty() || observation.source_root().is_empty() @@ -157,6 +179,82 @@ fn optional_uuid_from_first_column(row: &rusqlite::Row<'_>) -> rusqlite::Result< .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error))) } +fn derive_provider_file_publication_phase( + scope: &ProviderFilePublicationScope, + marker: &ReplacementMarker, +) -> ProviderFilePublicationPhase { + if scope.retires_observation && scope.kind == ProviderFilePublicationKind::Incremental { + return ProviderFilePublicationPhase::ReadyToFinalize; + } + if scope.tracks_prior_material && !marker.preparation_complete { + return ProviderFilePublicationPhase::Preparing; + } + if scope.retires_observation { + return if !scope.tracks_prior_material || marker.cleanup_phase == CLEANUP_PHASE_COMPLETE { + ProviderFilePublicationPhase::ReadyToFinalize + } else { + ProviderFilePublicationPhase::Reconciling + }; + } + if marker.completion_payload_json.is_none() { + return ProviderFilePublicationPhase::Importing; + } + if scope.kind == ProviderFilePublicationKind::Replacement + && scope.tracks_prior_material + && marker.cleanup_phase != CLEANUP_PHASE_COMPLETE + { + ProviderFilePublicationPhase::Reconciling + } else { + ProviderFilePublicationPhase::ReadyToFinalize + } +} + +fn serialize_provider_file_publication_completion( + completion: &ProviderFilePublicationCompletion, +) -> Result { + if completion.version == 0 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + let payload_json = serde_json::to_string(&serde_json::json!({ + "version": completion.version, + "payload": completion.payload, + }))?; + if payload_json.len() > PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + Ok(payload_json) +} + +fn parse_provider_file_publication_completion( + payload_json: &str, +) -> Result { + if payload_json.is_empty() + || payload_json.len() > PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES + { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + let mut envelope = serde_json::from_str::(payload_json)?; + let object = envelope + .as_object_mut() + .ok_or(StoreError::InvalidProviderFilePublicationScope)?; + if object.len() != 2 { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + let version = object + .remove("version") + .and_then(|value| value.as_u64()) + .and_then(|value| u32::try_from(value).ok()) + .filter(|version| *version > 0) + .ok_or(StoreError::InvalidProviderFilePublicationScope)?; + let payload = object + .remove("payload") + .ok_or(StoreError::InvalidProviderFilePublicationScope)?; + if !object.is_empty() { + return Err(StoreError::InvalidProviderFilePublicationScope); + } + Ok(ProviderFilePublicationCompletion { version, payload }) +} + fn parse_provider_file_publication_kind(value: &str) -> Result { match value { "incremental" => Ok(ProviderFilePublicationKind::Incremental), @@ -210,7 +308,3 @@ fn four_optional_uuids_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result) -> rusqlite::Result<(Uuid, Uuid)> { - Ok((parse_uuid_text(row.get(0)?)?, parse_uuid_text(row.get(1)?)?)) -} diff --git a/crates/ctx-history-store/src/provider_files/visibility.rs b/crates/ctx-history-store/src/provider_files/visibility.rs index b1635f41..e22b6ab2 100644 --- a/crates/ctx-history-store/src/provider_files/visibility.rs +++ b/crates/ctx-history-store/src/provider_files/visibility.rs @@ -1,11 +1,7 @@ -pub(crate) fn event_material_visible_predicate(event_alias: &str) -> String { +fn event_material_source_id_sql(event_alias: &str) -> String { format!( r#" - NOT EXISTS ( - SELECT 1 FROM capture_sources AS replacement_source - JOIN provider_file_publications AS replacement - ON {} - WHERE replacement_source.id = COALESCE( + COALESCE( {event_alias}.capture_source_id, ( SELECT event_session.capture_source_id @@ -24,12 +20,62 @@ pub(crate) fn event_material_visible_predicate(event_alias: &str) -> String { WHERE event_run.id = {event_alias}.run_id ) ) - ) - "#, - material_source_matches_replacement_predicate("replacement_source", "replacement") + "# ) } +pub(crate) fn event_material_visible_predicate(event_alias: &str) -> String { + material_source_id_not_replacing_predicate(&event_material_source_id_sql(event_alias)) +} + +impl Store { + pub(crate) fn importer_event_material_visible_predicate(&self, event_alias: &str) -> String { + self.importer_material_visible_predicate(&event_material_source_id_sql(event_alias)) + } + + pub(crate) fn importer_session_material_visible_predicate( + &self, + session_alias: &str, + ) -> String { + self.importer_material_visible_predicate(&format!("{session_alias}.capture_source_id")) + } + + pub(crate) fn importer_capture_source_material_visible_predicate( + &self, + source_alias: &str, + ) -> String { + self.importer_material_visible_predicate(&format!("{source_alias}.id")) + } + + fn importer_material_visible_predicate(&self, source_id_sql: &str) -> String { + let public = material_source_id_not_replacing_predicate(source_id_sql); + let active = self.provider_file_publication.borrow(); + let Some(active) = active.as_ref().filter(|active| { + active.lifecycle.load(Ordering::Acquire) + && !active.retires_observation + && self.provider_file_write_scope.get() == Some(active.scope_id) + }) else { + return public; + }; + let owner = material_source_matches_replacement_owner_predicate( + "active_publication_source", + "active_publication", + ); + format!( + r#" + (({public}) OR EXISTS ( + SELECT 1 + FROM capture_sources AS active_publication_source + JOIN provider_file_publications AS active_publication ON ({owner}) + WHERE active_publication.replacement_id = '{}' + AND active_publication_source.id = {source_id_sql} + )) + "#, + active.scope_id + ) + } +} + pub(crate) fn session_material_visible_predicate(session_alias: &str) -> String { material_source_id_not_replacing_predicate(&format!("{session_alias}.capture_source_id")) } @@ -58,7 +104,16 @@ pub(crate) fn history_record_material_visible_predicate(record_alias: &str) -> S let event_visible = event_material_visible_predicate("record_event"); format!( r#" - ({direct}) AND ( + ({direct}) + AND NOT EXISTS ( + SELECT 1 + FROM provider_file_publications AS record_publication + JOIN {STAGING_SEEN_TABLE} AS record_seen + ON record_seen.replacement_id = record_publication.replacement_id + AND record_seen.entity_kind IN ('history_record', '{PRIOR_HISTORY_RECORD_KIND}') + AND record_seen.entity_id = {record_alias}.id + ) + AND ( NOT EXISTS (SELECT 1 FROM sessions WHERE history_record_id = {record_alias}.id) AND NOT EXISTS (SELECT 1 FROM runs WHERE history_record_id = {record_alias}.id) AND NOT EXISTS (SELECT 1 FROM events WHERE history_record_id = {record_alias}.id) @@ -79,6 +134,39 @@ pub(crate) fn history_record_material_visible_predicate(record_alias: &str) -> S ) } +impl Store { + pub(crate) fn importer_history_record_material_visible_predicate( + &self, + record_alias: &str, + ) -> String { + let public = history_record_material_visible_predicate(record_alias); + let active_scope_id = self + .provider_file_publication + .borrow() + .as_ref() + .filter(|active| { + active.attached + && active.lifecycle.load(Ordering::Acquire) + && self.provider_file_write_scope.get() == Some(active.scope_id) + }) + .map(|active| active.scope_id); + let Some(active_scope_id) = active_scope_id else { + return public; + }; + format!( + r#" + (({public}) OR EXISTS ( + SELECT 1 + FROM provider_file_publication_seen AS active_record_seen + WHERE active_record_seen.replacement_id = '{active_scope_id}' + AND active_record_seen.entity_kind = 'history_record' + AND active_record_seen.entity_id = {record_alias}.id + )) + "#, + ) + } +} + pub(crate) fn summary_material_visible_predicate(summary_alias: &str) -> String { let direct = direct_source_material_visible_predicate(summary_alias, "source_id"); let session_visible = session_material_visible_predicate("summary_session"); @@ -272,9 +360,12 @@ fn material_source_matches_replacement_owner_predicate( pub(crate) fn catalog_material_visible_predicate(catalog_alias: &str) -> String { let effective = effective_provider_file_publication_predicate("replacement"); + let inventory_published = + crate::catalog::catalog_inventory_material_published_predicate(catalog_alias); format!( r#" - NOT EXISTS ( + ({inventory_published}) + AND NOT EXISTS ( SELECT 1 FROM provider_file_publications AS replacement WHERE replacement.inventory_family = '{CATALOG_INVENTORY_FAMILY}' AND replacement.provider = {catalog_alias}.provider @@ -309,17 +400,94 @@ pub(crate) fn effective_provider_file_publication_predicate(publication_alias: & format!("{publication_alias}.mutation_started = 1 OR ({current})") } -pub(crate) fn replacement_observation_current_predicate(replacement_alias: &str) -> String { +fn global_provider_file_publication_id_sql() -> String { + let effective = effective_provider_file_publication_predicate("global_publication"); + format!( + "SELECT global_publication.replacement_id \ + FROM provider_file_publications AS global_publication \ + WHERE {effective} \ + ORDER BY global_publication.started_at_ms, global_publication.replacement_id \ + LIMIT 1" + ) +} + +pub(crate) fn catalog_candidate_is_global_publication(catalog_alias: &str) -> String { + let global_id = global_provider_file_publication_id_sql(); format!( r#" EXISTS ( - SELECT 1 FROM import_inventory_generations AS replacement_inventory - WHERE replacement_inventory.provider = {replacement_alias}.provider - AND replacement_inventory.source_root = {replacement_alias}.inventory_source_root - AND replacement_inventory.inventory_family = {replacement_alias}.inventory_family - AND replacement_inventory.current_generation = {replacement_alias}.inventory_generation + SELECT 1 FROM provider_file_publications AS candidate_publication + WHERE candidate_publication.replacement_id = ({global_id}) + AND candidate_publication.inventory_observation_invalidated = 0 + AND candidate_publication.inventory_family = '{CATALOG_INVENTORY_FAMILY}' + AND candidate_publication.provider = {catalog_alias}.provider + AND candidate_publication.inventory_source_format = {catalog_alias}.source_format + AND candidate_publication.inventory_source_root = {catalog_alias}.source_root + AND candidate_publication.source_path = {catalog_alias}.source_path + AND candidate_publication.file_size_bytes = {catalog_alias}.file_size_bytes + AND candidate_publication.file_modified_at_ms = {catalog_alias}.file_modified_at_ms + AND candidate_publication.import_revision = {catalog_alias}.import_revision + AND (candidate_publication.metadata_json IS NULL + OR candidate_publication.metadata_json IS {catalog_alias}.metadata_json) ) - AND ( + "# + ) +} + +pub(crate) fn source_file_candidate_is_global_publication(source_alias: &str) -> String { + let global_id = global_provider_file_publication_id_sql(); + format!( + r#" + EXISTS ( + SELECT 1 FROM provider_file_publications AS candidate_publication + WHERE candidate_publication.replacement_id = ({global_id}) + AND candidate_publication.inventory_observation_invalidated = 0 + AND candidate_publication.inventory_family = '{SOURCE_IMPORT_INVENTORY_FAMILY}' + AND candidate_publication.provider = {source_alias}.provider + AND candidate_publication.inventory_source_format = {source_alias}.source_format + AND candidate_publication.inventory_source_root = {source_alias}.source_root + AND candidate_publication.source_path = {source_alias}.source_path + AND candidate_publication.file_size_bytes = {source_alias}.file_size_bytes + AND candidate_publication.file_modified_at_ms = {source_alias}.file_modified_at_ms + AND candidate_publication.import_revision = {source_alias}.import_revision + AND candidate_publication.metadata_json IS {source_alias}.metadata_json + ) + "# + ) +} + +fn provider_file_retirement_observation_current_predicate(publication_alias: &str) -> String { + provider_file_observation_current_predicate(publication_alias, false) +} + +pub(crate) fn replacement_observation_current_predicate(replacement_alias: &str) -> String { + provider_file_observation_current_predicate(replacement_alias, true) +} + +fn provider_file_observation_current_predicate( + replacement_alias: &str, + require_generation_match: bool, +) -> String { + let generation_current = if require_generation_match { + format!( + r#" + EXISTS ( + SELECT 1 FROM import_inventory_generations AS replacement_inventory + WHERE replacement_inventory.provider = {replacement_alias}.provider + AND replacement_inventory.source_root = {replacement_alias}.inventory_source_root + AND replacement_inventory.inventory_family = {replacement_alias}.inventory_family + AND replacement_inventory.current_generation = {replacement_alias}.inventory_generation + ) + AND + "# + ) + } else { + String::new() + }; + format!( + r#" + {replacement_alias}.inventory_observation_invalidated = 0 + AND {generation_current} ( ( {replacement_alias}.inventory_family = '{CATALOG_INVENTORY_FAMILY}' AND EXISTS ( @@ -332,6 +500,8 @@ pub(crate) fn replacement_observation_current_predicate(replacement_alias: &str) AND replacement_catalog.file_size_bytes = {replacement_alias}.file_size_bytes AND replacement_catalog.file_modified_at_ms = {replacement_alias}.file_modified_at_ms AND replacement_catalog.import_revision = {replacement_alias}.import_revision + AND ({replacement_alias}.metadata_json IS NULL + OR replacement_catalog.metadata_json IS {replacement_alias}.metadata_json) ) ) OR ( @@ -370,7 +540,7 @@ pub(crate) fn has_fenced_provider_file_publications(conn: &Connection) -> Result .map_err(StoreError::from) } -fn material_owner_predicate( +pub(crate) fn material_owner_predicate( source_alias: &str, provider: &str, source_format: &str, diff --git a/crates/ctx-history-store/src/provider_files/write_scope.rs b/crates/ctx-history-store/src/provider_files/write_scope.rs index 33340093..40acdee6 100644 --- a/crates/ctx-history-store/src/provider_files/write_scope.rs +++ b/crates/ctx-history-store/src/provider_files/write_scope.rs @@ -108,11 +108,28 @@ impl Store { &self, scope: &ProviderFilePublicationScope, writes: impl FnOnce(&Self) -> Result, + ) -> Result { + self.with_provider_file_publication_writes_inner(scope, false, writes) + } + + fn with_provider_file_publication_writes_inner( + &self, + scope: &ProviderFilePublicationScope, + allow_staged_completion: bool, + writes: impl FnOnce(&Self) -> Result, ) -> Result { self.ensure_active_provider_file_publication(scope)?; if scope.retires_observation || self.provider_file_write_scope.get().is_some() { return Err(StoreError::InvalidProviderFilePublicationScope); } + if !allow_staged_completion + && self + .load_replacement_marker(scope)? + .completion_payload_json + .is_some() + { + return Err(StoreError::InvalidProviderFilePublicationScope); + } self.provider_file_write_scope.set(Some(scope.scope_id)); let _reset = ProviderFileWriteScopeReset { scope: &self.provider_file_write_scope, @@ -120,6 +137,38 @@ impl Store { writes(self) } + /// Mutable counterpart for importers whose established API requires a + /// mutable store reference while publishing provider-file material. + pub fn with_provider_file_publication_writes_mut( + &mut self, + scope: &ProviderFilePublicationScope, + writes: impl FnOnce(&mut Self) -> std::result::Result, + ) -> std::result::Result + where + E: From, + { + self.ensure_active_provider_file_publication(scope) + .map_err(E::from)?; + if scope.retires_observation || self.provider_file_write_scope.get().is_some() { + return Err(E::from(StoreError::InvalidProviderFilePublicationScope)); + } + if self + .load_replacement_marker(scope) + .map_err(E::from)? + .completion_payload_json + .is_some() + { + return Err(E::from(StoreError::InvalidProviderFilePublicationScope)); + } + self.provider_file_write_scope.set(Some(scope.scope_id)); + let write_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| writes(self))); + self.provider_file_write_scope.set(None); + match write_result { + Ok(result) => result, + Err(payload) => std::panic::resume_unwind(payload), + } + } + pub(crate) fn with_provider_file_publication_write( &self, write: impl FnOnce() -> Result, @@ -198,75 +247,89 @@ impl Store { completion_kind: ProviderFileCompletionKind, has_safe_checkpoint: bool, ) -> Result<()> { - validate_successful_outcome(outcome)?; - self.ensure_provider_file_observation_is_current(outcome.provider, outcome.observation)?; - let changed = match outcome.observation { - ProviderFileInventoryObservation::Catalog { mut update, .. } => { - let (prior_status, prior_error, prior_event_count) = self.conn.query_row( - r#" + validate_provider_file_completion_outcome( + outcome, + completion_kind, + has_safe_checkpoint, + false, + )?; + self.ensure_provider_file_observation_matches_persisted( + outcome.provider, + outcome.observation, + )?; + let current_inventory_generation = self.current_provider_file_inventory_generation( + outcome.provider, + outcome.observation.source_root(), + outcome.observation.inventory_family(), + )?; + let changed = if let Some((mut update, metadata)) = outcome.observation.catalog_update() { + update.inventory_generation = current_inventory_generation; + let (prior_status, prior_error, prior_event_count) = self.conn.query_row( + r#" SELECT indexed_status, indexed_error, last_imported_event_count FROM catalog_sessions WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 "#, - params![ - outcome.provider.as_str(), - update.source_root, - update.source_path - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - )) - }, - )?; - let preserve_rejections = prior_status == "completed_with_rejections" - && completion_kind != ProviderFileCompletionKind::Replacement; - let status = if preserve_rejections { - CatalogIndexedStatus::CompletedWithRejections - } else { - outcome.status - }; - let error = if preserve_rejections { - prior_error.as_deref().or(outcome.error) - } else { - outcome.error - }; - let prior_event_count = - prior_event_count.map(nonnegative_i64_to_u64).transpose()?; - update.event_count = match completion_kind { - ProviderFileCompletionKind::Replacement => update.event_count, - ProviderFileCompletionKind::AppendDelta => match update.event_count { - Some(delta) => { - Some(prior_event_count.unwrap_or(0).checked_add(delta).ok_or( - StoreError::ProviderFileReconciliationInconsistent { - entity: "catalog event count", - }, - )?) - } - None => prior_event_count, - }, - ProviderFileCompletionKind::RetainCheckpoint => prior_event_count, - }; - let changed = if completion_kind == ProviderFileCompletionKind::RetainCheckpoint { - self.record_catalog_source_import_result_preserving_legacy_cursor( - outcome.provider, - update, - status, - error, - )? - } else { - self.record_catalog_source_import_result( + params![ + outcome.provider.as_str(), + update.source_root, + update.source_path + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + }, + )?; + let preserve_rejections = completion_kind != ProviderFileCompletionKind::Replacement + && (prior_status == "completed_with_rejections" + || (prior_status == "indexed" + && outcome.status == CatalogIndexedStatus::Rejected)); + let status = if preserve_rejections { + CatalogIndexedStatus::CompletedWithRejections + } else { + outcome.status + }; + let error = if preserve_rejections { + prior_error.as_deref().or(outcome.error) + } else { + outcome.error + }; + let prior_event_count = prior_event_count.map(nonnegative_i64_to_u64).transpose()?; + update.event_count = match completion_kind { + ProviderFileCompletionKind::Replacement => update.event_count, + ProviderFileCompletionKind::AppendDelta => match update.event_count { + Some(delta) => Some(prior_event_count.unwrap_or(0).checked_add(delta).ok_or( + StoreError::ProviderFileReconciliationInconsistent { + entity: "catalog event count", + }, + )?), + None => prior_event_count, + }, + ProviderFileCompletionKind::RetainCheckpoint => prior_event_count, + }; + let changed = match completion_kind { + ProviderFileCompletionKind::RetainCheckpoint => self + .record_observed_catalog_source_import_result_preserving_legacy_cursor( outcome.provider, update, + metadata, status, error, - )? - }; - if changed == 1 && has_safe_checkpoint { - self.conn.execute( - r#" + )?, + _ => self.record_observed_catalog_source_import_result( + outcome.provider, + update, + metadata, + status, + error, + )?, + }; + if changed == 1 && has_safe_checkpoint { + self.conn.execute( + r#" UPDATE catalog_sessions SET last_imported_at_ms = ?4, last_imported_file_size_bytes = ?5, @@ -278,49 +341,53 @@ impl Store { AND indexed_file_modified_at_ms = ?6 AND indexed_import_revision = ?9 "#, - params![ - outcome.provider.as_str(), - update.source_root, - update.source_path, - update.indexed_at_ms, - capped_i64(update.file_size_bytes), - update.file_modified_at_ms, - update.file_sha256, - update.event_count.map(capped_i64), - i64::from(update.import_revision), - ], - )?; - } - changed - } - ProviderFileInventoryObservation::SourceImport { update, .. } => { - let (prior_status, prior_error) = self.conn.query_row( - r#" - SELECT indexed_status, indexed_error - FROM source_import_files - WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 - "#, params![ outcome.provider.as_str(), update.source_root, - update.source_path + update.source_path, + update.indexed_at_ms, + capped_i64(update.file_size_bytes), + update.file_modified_at_ms, + update.file_sha256, + update.event_count.map(capped_i64), + i64::from(update.import_revision), ], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), )?; - let preserve_rejections = prior_status == "completed_with_rejections" - && completion_kind != ProviderFileCompletionKind::Replacement; - let status = if preserve_rejections { - CatalogIndexedStatus::CompletedWithRejections - } else { - outcome.status - }; - let error = if preserve_rejections { - prior_error.as_deref().or(outcome.error) - } else { - outcome.error - }; - self.record_source_import_file_result(outcome.provider, update, status, error)? } + changed + } else { + let ProviderFileInventoryObservation::SourceImport { mut update, .. } = + outcome.observation + else { + unreachable!("non-catalog provider observation must be a source import") + }; + update.inventory_generation = current_inventory_generation; + let (prior_status, prior_error) = self.conn.query_row( + r#" + SELECT indexed_status, indexed_error + FROM source_import_files + WHERE provider = ?1 AND source_root = ?2 AND source_path = ?3 + "#, + params![ + outcome.provider.as_str(), + update.source_root, + update.source_path + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + )?; + let preserve_rejections = prior_status == "completed_with_rejections" + && completion_kind != ProviderFileCompletionKind::Replacement; + let status = if preserve_rejections { + CatalogIndexedStatus::CompletedWithRejections + } else { + outcome.status + }; + let error = if preserve_rejections { + prior_error.as_deref().or(outcome.error) + } else { + outcome.error + }; + self.record_source_import_file_result(outcome.provider, update, status, error)? }; if changed != 1 { return Err(provider_file_observation_changed( diff --git a/crates/ctx-history-store/src/records.rs b/crates/ctx-history-store/src/records.rs index ea34f190..13776206 100644 --- a/crates/ctx-history-store/src/records.rs +++ b/crates/ctx-history-store/src/records.rs @@ -138,6 +138,7 @@ impl Store { record.updated_at.to_rfc3339(), ], )?; + self.track_provider_file_publication_history_record(record.id)?; upsert_record_search_projection(&self.conn, record)?; Ok(()) } @@ -149,6 +150,7 @@ impl Store { fn upsert_record_inner(&self, record: &HistoryRecord) -> Result<()> { self.ensure_provider_file_history_record_write_allowed(record.id)?; self.upsert_record_row(record)?; + self.track_provider_file_publication_history_record(record.id)?; upsert_record_search_projection(&self.conn, record)?; Ok(()) } @@ -159,6 +161,10 @@ impl Store { fn delete_orphan_record_inner(&self, record_id: Uuid) -> Result { self.ensure_provider_file_history_record_write_allowed(record_id)?; + self.delete_orphan_record_row(record_id) + } + + pub(crate) fn delete_orphan_record_row(&self, record_id: Uuid) -> Result { let record_id = record_id.to_string(); let deleted = self.conn.execute( r#" @@ -200,7 +206,10 @@ impl Store { self.begin_immediate_batch()?; } for record in records { - if let Err(err) = self.upsert_record_row(record) { + if let Err(err) = self + .upsert_record_row(record) + .and_then(|()| self.track_provider_file_publication_history_record(record.id)) + { if owns_transaction { let _ = self.rollback_batch(); } @@ -264,8 +273,7 @@ impl Store { } pub fn get_record(&self, id: Uuid) -> Result { - let visible = - crate::provider_files::history_record_material_visible_predicate("history_records"); + let visible = self.importer_history_record_material_visible_predicate("history_records"); self.conn .query_row( record_select_sql(&format!("WHERE id = ?1 AND {visible}")).as_str(), @@ -281,8 +289,7 @@ impl Store { } pub fn list_records_page(&self, limit: usize, offset: usize) -> Result> { - let visible = - crate::provider_files::history_record_material_visible_predicate("history_records"); + let visible = self.importer_history_record_material_visible_predicate("history_records"); let mut stmt = self.conn.prepare( record_select_sql(&format!( "WHERE {visible} ORDER BY created_at DESC, id LIMIT ?1 OFFSET ?2" @@ -331,8 +338,7 @@ impl Store { let limit_parameter = values.len(); values.push(Value::Integer(offset as i64)); let offset_parameter = values.len(); - let visible = - crate::provider_files::history_record_material_visible_predicate("history_records"); + let visible = self.importer_history_record_material_visible_predicate("history_records"); let tail = format!( "WHERE ({}) AND {visible} ORDER BY ({coverage}) DESC, created_at DESC, id LIMIT ?{limit_parameter} OFFSET ?{offset_parameter}", predicates.join(") OR (") @@ -356,8 +362,7 @@ impl Store { let has_artifact_search = table_exists(&self.conn, "artifact_search")?; let has_record_scriptgram = record_scriptgram_table_ready(&self.conn)?; let has_event_scriptgram = event_scriptgram_table_ready(&self.conn)?; - let record_visible = - crate::provider_files::history_record_material_visible_predicate("record"); + let record_visible = self.importer_history_record_material_visible_predicate("record"); let event_visible = crate::provider_files::event_material_visible_predicate("event"); let artifact_visible = crate::provider_files::direct_source_material_visible_predicate( "artifact", diff --git a/crates/ctx-history-store/src/schema/ddl.rs b/crates/ctx-history-store/src/schema/ddl.rs index 1180da1c..1e7450a0 100644 --- a/crates/ctx-history-store/src/schema/ddl.rs +++ b/crates/ctx-history-store/src/schema/ddl.rs @@ -219,7 +219,8 @@ CREATE TABLE IF NOT EXISTS catalog_sessions ( last_imported_file_modified_at_ms INTEGER, last_imported_file_sha256 TEXT, last_imported_event_count INTEGER, - metadata_json TEXT NOT NULL DEFAULT '{}' + metadata_json TEXT NOT NULL DEFAULT '{}', + pending_reason TEXT CHECK (pending_reason IS NULL OR pending_reason IN ('fresh_new', 'fresh_changed', 'fresh_append', 'recovery_retry', 'recovery_replacement', 'parser_revision', 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan')) ); CREATE TABLE IF NOT EXISTS source_import_files ( @@ -241,9 +242,19 @@ CREATE TABLE IF NOT EXISTS source_import_files ( indexed_error TEXT, indexed_import_revision INTEGER CHECK (indexed_import_revision > 0), metadata_json TEXT NOT NULL DEFAULT '{}', + pending_reason TEXT CHECK (pending_reason IS NULL OR pending_reason IN ('fresh_new', 'fresh_changed', 'fresh_append', 'recovery_retry', 'recovery_replacement', 'parser_revision', 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan')), PRIMARY KEY (provider, source_root, source_path) ); +CREATE TABLE IF NOT EXISTS import_pending_reason_repairs ( + inventory_family TEXT PRIMARY KEY NOT NULL + CHECK (inventory_family IN ('catalog_sessions', 'source_import_files')), + cursor_provider TEXT, + cursor_source_root TEXT, + cursor_source_path TEXT, + completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0, 1)) +); + CREATE TABLE IF NOT EXISTS provider_file_checkpoints ( provider TEXT NOT NULL, source_format TEXT NOT NULL CHECK (length(source_format) > 0), @@ -279,6 +290,8 @@ CREATE TABLE IF NOT EXISTS provider_file_publications ( import_revision INTEGER NOT NULL CHECK (import_revision > 0), metadata_json TEXT, mutation_started INTEGER NOT NULL DEFAULT 0 CHECK (mutation_started IN (0, 1)), + tracks_prior_material INTEGER NOT NULL DEFAULT 0 CHECK (tracks_prior_material IN (0, 1)), + staging_initialized INTEGER NOT NULL DEFAULT 0 CHECK (staging_initialized IN (0, 1)), preparation_complete INTEGER NOT NULL DEFAULT 0 CHECK (preparation_complete IN (0, 1)), preparation_cursor TEXT, cleanup_phase INTEGER NOT NULL DEFAULT 0 CHECK (cleanup_phase BETWEEN 0 AND 14), @@ -300,9 +313,39 @@ CREATE TABLE IF NOT EXISTS provider_file_publications ( tombstoned_sessions INTEGER NOT NULL DEFAULT 0 CHECK (tombstoned_sessions >= 0), started_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, + completion_payload_json TEXT CHECK ( + completion_payload_json IS NULL OR + length(CAST(completion_payload_json AS BLOB)) BETWEEN 1 AND 262144 + ), + inventory_observation_invalidated INTEGER NOT NULL DEFAULT 0 + CHECK (inventory_observation_invalidated IN (0, 1)), + retirement_started INTEGER NOT NULL DEFAULT 0 CHECK (retirement_started IN (0, 1)), PRIMARY KEY (provider, material_source_format, material_source_root, source_path) ); +CREATE TABLE IF NOT EXISTS provider_file_publication_seen ( + replacement_id TEXT NOT NULL, + entity_kind TEXT NOT NULL, + entity_id TEXT NOT NULL, + PRIMARY KEY (replacement_id, entity_kind, entity_id), + FOREIGN KEY (replacement_id) REFERENCES provider_file_publications(replacement_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS provider_file_publication_prior_sources ( + replacement_id TEXT NOT NULL, + source_id TEXT NOT NULL, + PRIMARY KEY (replacement_id, source_id), + FOREIGN KEY (replacement_id) REFERENCES provider_file_publications(replacement_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS provider_file_publication_batch ( + replacement_id TEXT NOT NULL, + source_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + PRIMARY KEY (replacement_id, source_id, entity_id), + FOREIGN KEY (replacement_id) REFERENCES provider_file_publications(replacement_id) ON DELETE CASCADE +); + CREATE TABLE IF NOT EXISTS semantic_replacement_revision ( singleton INTEGER PRIMARY KEY NOT NULL CHECK (singleton = 1), current_revision INTEGER NOT NULL CHECK (current_revision >= 0) diff --git a/crates/ctx-history-store/src/schema/fixtures/pending_reason_v49_rows.sql b/crates/ctx-history-store/src/schema/fixtures/pending_reason_v49_rows.sql new file mode 100644 index 00000000..b3684b9b --- /dev/null +++ b/crates/ctx-history-store/src/schema/fixtures/pending_reason_v49_rows.sql @@ -0,0 +1,44 @@ +INSERT INTO catalog_sessions ( + source_path, provider, source_format, source_root, external_session_id, + agent_type, file_size_bytes, file_modified_at_ms, import_revision, + cataloged_at_ms, indexed_at_ms, indexed_file_size_bytes, + indexed_file_modified_at_ms, indexed_status, indexed_error, + indexed_import_revision +) VALUES ( + '/fixture/codex/failed.jsonl', 'codex', 'codex_session_jsonl', + '/fixture/codex', 'failed-codex', 'primary', 128, 1000, 1, + 1000, 1001, 128, 1000, 'failed', 'legacy retry', 1 +); + +INSERT INTO source_import_files ( + provider, source_format, source_root, source_path, file_size_bytes, + file_modified_at_ms, import_revision, observed_at_ms, indexed_at_ms, + indexed_file_size_bytes, indexed_file_modified_at_ms, indexed_status, + indexed_error, indexed_import_revision, metadata_json +) VALUES ( + 'pi', 'pi_session_jsonl', '/fixture/pi', '/fixture/pi/failed.jsonl', + 96, 2000, 1, 2000, 2001, 96, 2000, 'failed', 'legacy retry', 1, '{}' +), ( + 'tabnine', 'tabnine_cli_recording_jsonl', '/fixture/tabnine', + '/fixture/tabnine/revision.jsonl', 64, 3000, 2, 3000, 3001, + 64, 3000, 'indexed', NULL, 1, '{}' +), ( + 'qwen_code', 'qwen_code_json', '/fixture/qwen', + '/fixture/qwen/missing.json', 80, 4000, 1, 4000, 4001, + 80, 4000, 'indexed', NULL, 1, '{"inventory_unit":"logical_import_unit"}' +), ( + 'hermes', 'hermes_state_sqlite', '/fixture/hermes', + '/fixture/hermes/state.db', 72, 5000, 1, 5000, 5001, + 72, 5000, 'indexed', NULL, 1, '{"inventory_unit":"source_root"}' +); + +INSERT INTO capture_sources ( + id, kind, provider, machine_id, raw_source_path, source_format, + source_root, started_at_ms, fidelity +) VALUES ( + 'qwen-sibling', 'provider_import', 'qwen_code', 'fixture-machine', + '/fixture/qwen/sibling.json', 'qwen_code_json', '/fixture/qwen', 4001, 'imported' +), ( + 'hermes-root', 'provider_import', 'hermes', 'fixture-machine', + '/fixture/hermes/sibling.db', 'hermes_state_sqlite', '/fixture/hermes', 5001, 'imported' +); diff --git a/crates/ctx-history-store/src/schema/indexes.rs b/crates/ctx-history-store/src/schema/indexes.rs index e9b95478..996e4eeb 100644 --- a/crates/ctx-history-store/src/schema/indexes.rs +++ b/crates/ctx-history-store/src/schema/indexes.rs @@ -1,7 +1,7 @@ pub(crate) const INDEXES_SQL: &str = r#" CREATE INDEX IF NOT EXISTS idx_capture_sources_external_session_id ON capture_sources(provider, external_session_id); CREATE INDEX IF NOT EXISTS idx_capture_sources_provider_source_identity ON capture_sources(provider, source_format, source_identity); -CREATE INDEX IF NOT EXISTS idx_capture_sources_provider_material_owner ON capture_sources(provider, source_format, raw_source_path, source_root, id); +CREATE INDEX IF NOT EXISTS idx_capture_sources_provider_material_owner ON capture_sources(provider, source_format, source_root, raw_source_path, external_session_id, id); CREATE UNIQUE INDEX IF NOT EXISTS idx_provider_file_publications_owner ON provider_file_publications(owner_id); CREATE INDEX IF NOT EXISTS idx_provider_file_publications_fence ON provider_file_publications(mutation_started, provider, material_source_format, material_source_root, source_path); @@ -105,3 +105,52 @@ CREATE INDEX IF NOT EXISTS idx_reconcile_history_records_source_id ON history_re CREATE INDEX IF NOT EXISTS idx_reconcile_vcs_workspaces_source_id ON vcs_workspaces(source_id, id); CREATE INDEX IF NOT EXISTS idx_reconcile_audit_log_source_id ON audit_log(source_id, id); "#; + +pub(crate) const PENDING_WORK_INDEXES_SQL: &str = r#" +-- Fresh stores have no legacy rows to classify. Real upgrades retain v51's +-- incomplete ledger until bounded maintenance advances it. +UPDATE import_pending_reason_repairs +SET completed = 1 +WHERE completed = 0 + AND cursor_provider IS NULL + AND cursor_source_root IS NULL + AND cursor_source_path IS NULL + AND ( + (inventory_family = 'catalog_sessions' + AND NOT EXISTS (SELECT 1 FROM catalog_sessions)) + OR + (inventory_family = 'source_import_files' + AND NOT EXISTS (SELECT 1 FROM source_import_files)) + ); + +DROP INDEX IF EXISTS idx_catalog_sessions_pending_reason; +DROP INDEX IF EXISTS idx_source_import_files_pending_reason; +DROP INDEX IF EXISTS idx_catalog_sessions_pending_fresh; +DROP INDEX IF EXISTS idx_catalog_sessions_pending_recovery; +DROP INDEX IF EXISTS idx_source_import_files_pending_fresh; +DROP INDEX IF EXISTS idx_source_import_files_pending_recovery; + +CREATE INDEX IF NOT EXISTS idx_catalog_sessions_pending_fresh_attempt +ON catalog_sessions(provider, source_root, indexed_at_ms, source_path) +WHERE is_stale = 0 + AND pending_reason IN ('fresh_new', 'fresh_changed', 'fresh_append'); +CREATE INDEX IF NOT EXISTS idx_catalog_sessions_pending_recovery_attempt +ON catalog_sessions(provider, source_root, indexed_at_ms, source_path) +WHERE is_stale = 0 + AND pending_reason IN ( + 'recovery_retry', 'recovery_replacement', 'parser_revision', + 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan' + ); + +CREATE INDEX IF NOT EXISTS idx_source_import_files_pending_fresh_attempt +ON source_import_files(provider, source_root, indexed_at_ms, source_path) +WHERE is_stale = 0 + AND pending_reason IN ('fresh_new', 'fresh_changed', 'fresh_append'); +CREATE INDEX IF NOT EXISTS idx_source_import_files_pending_recovery_attempt +ON source_import_files(provider, source_root, indexed_at_ms, source_path) +WHERE is_stale = 0 + AND pending_reason IN ( + 'recovery_retry', 'recovery_replacement', 'parser_revision', + 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan' + ); +"#; diff --git a/crates/ctx-history-store/src/schema/migrations.rs b/crates/ctx-history-store/src/schema/migrations.rs index bff6d6f7..10971153 100644 --- a/crates/ctx-history-store/src/schema/migrations.rs +++ b/crates/ctx-history-store/src/schema/migrations.rs @@ -7,7 +7,9 @@ use crate::schema::ddl::{ }; use crate::schema::fts::{create_fts_tables_if_supported, drop_fts_table_if_exists}; use crate::schema::indexes::INDEXES_SQL; -use crate::schema::rebuild::rebuild_v44_current_schema_tables; +use crate::schema::rebuild::{ + rebuild_table_from_current_schema, rebuild_v44_current_schema_tables, +}; use crate::schema::scriptgram::migrate_to_v45; use crate::schema::views::{ create_stable_sql_views, drop_stable_sql_views, stable_sql_views_exist, @@ -91,6 +93,18 @@ pub(crate) fn run_migrations(conn: &Connection, user_version: i64) -> Result<()> if user_version < 50 { migrate_to_v50(conn)?; } + if user_version < 51 { + migrate_to_v51(conn)?; + } + if user_version < 52 { + migrate_to_v52(conn)?; + } + if user_version < 53 { + migrate_to_v53(conn)?; + } + if user_version < 54 { + migrate_to_v54(conn)?; + } Ok(()) } @@ -832,7 +846,7 @@ fn migrate_to_v49(conn: &Connection) -> Result<()> { } } -fn migrate_to_v50(conn: &Connection) -> Result<()> { +pub(super) fn migrate_to_v50(conn: &Connection) -> Result<()> { conn.execute_batch("BEGIN IMMEDIATE;")?; let migration = (|| -> Result<()> { if stable_sql_views_exist(conn)? { @@ -862,6 +876,271 @@ fn migrate_to_v50(conn: &Connection) -> Result<()> { } } +pub(super) fn migrate_to_v51(conn: &Connection) -> Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let migration = (|| -> Result<()> { + let legacy_provider_staging = + !table_has_column(conn, "provider_file_publications", "staging_initialized")?; + let legacy_prior_material_scope = + !table_has_column(conn, "provider_file_publications", "tracks_prior_material")?; + if legacy_provider_staging || legacy_prior_material_scope { + let recreate_views = stable_sql_views_exist(conn)?; + if recreate_views { + drop_stable_sql_views(conn)?; + } + if legacy_prior_material_scope { + conn.execute_batch( + "ALTER TABLE provider_file_publications ADD COLUMN tracks_prior_material INTEGER \ + NOT NULL DEFAULT 0 CHECK (tracks_prior_material IN (0, 1));", + )?; + } + if legacy_provider_staging { + conn.execute_batch( + "ALTER TABLE provider_file_publications ADD COLUMN staging_initialized INTEGER \ + NOT NULL DEFAULT 0 CHECK (staging_initialized IN (0, 1));", + )?; + } + rebuild_table_from_current_schema(conn, "provider_file_publications")?; + if recreate_views { + create_stable_sql_views(conn)?; + } + } + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS provider_file_publication_seen ( + replacement_id TEXT NOT NULL, + entity_kind TEXT NOT NULL, + entity_id TEXT NOT NULL, + PRIMARY KEY (replacement_id, entity_kind, entity_id), + FOREIGN KEY (replacement_id) REFERENCES provider_file_publications(replacement_id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS provider_file_publication_prior_sources ( + replacement_id TEXT NOT NULL, + source_id TEXT NOT NULL, + PRIMARY KEY (replacement_id, source_id), + FOREIGN KEY (replacement_id) REFERENCES provider_file_publications(replacement_id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS provider_file_publication_batch ( + replacement_id TEXT NOT NULL, + source_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + PRIMARY KEY (replacement_id, source_id, entity_id), + FOREIGN KEY (replacement_id) REFERENCES provider_file_publications(replacement_id) ON DELETE CASCADE + ); + "#, + )?; + if legacy_prior_material_scope { + // v50 sidecar state is unavailable after restart. Once mutation + // began, classify conservatively so recovery reconciles any rows + // committed before the crash, even without an append checkpoint. + conn.execute_batch( + r#" + UPDATE provider_file_publications AS publication + SET tracks_prior_material = CASE + WHEN publication.mutation_started = 0 THEN 0 + ELSE 1 + END; + "#, + )?; + } + if legacy_provider_staging { + // v50 staging lived in an unlinked Unix sidecar and cannot be trusted + // after restart. Keep mutation fencing, but rewind every cursor whose + // corresponding staged rows must be reconstructed in the main database. + conn.execute_batch( + r#" + UPDATE provider_file_publications + SET staging_initialized = 0, + preparation_complete = CASE + WHEN publication_kind = 'incremental' THEN 1 ELSE 0 + END, + preparation_cursor = NULL, + cleanup_phase = 0, + cleanup_source_cursor = NULL, + cleanup_entity_cursor = NULL, + removed_artifacts = 0, + removed_summaries = 0, + removed_history_record_links = 0, + removed_history_records = 0, + removed_history_record_tags = 0, + removed_record_edges = 0, + removed_audit_log_entries = 0, + removed_vcs_workspaces = 0, + removed_vcs_changes = 0, + removed_events = 0, + removed_runs = 0, + removed_files_touched = 0, + removed_session_edges = 0, + tombstoned_sessions = 0; + "#, + )?; + } + if !table_has_column(conn, "catalog_sessions", "pending_reason")? { + conn.execute_batch( + "ALTER TABLE catalog_sessions ADD COLUMN pending_reason TEXT \ + CHECK (pending_reason IS NULL OR pending_reason IN \ + ('fresh_new', 'fresh_changed', 'fresh_append', 'recovery_retry', \ + 'recovery_replacement', 'parser_revision', 'missing_material', \ + 'abandoned_publication', 'legacy', 'explicit_rescan'));", + )?; + } + if !table_has_column(conn, "source_import_files", "pending_reason")? { + conn.execute_batch( + "ALTER TABLE source_import_files ADD COLUMN pending_reason TEXT \ + CHECK (pending_reason IS NULL OR pending_reason IN \ + ('fresh_new', 'fresh_changed', 'fresh_append', 'recovery_retry', \ + 'recovery_replacement', 'parser_revision', 'missing_material', \ + 'abandoned_publication', 'legacy', 'explicit_rescan'));", + )?; + } + conn.execute_batch( + r#" + -- Existing inventory rows are classified by bounded maintenance. These + -- cursors make that work durable without rewriting the corpus here. + CREATE TABLE IF NOT EXISTS import_pending_reason_repairs ( + inventory_family TEXT PRIMARY KEY NOT NULL + CHECK (inventory_family IN ('catalog_sessions', 'source_import_files')), + cursor_provider TEXT, + cursor_source_root TEXT, + cursor_source_path TEXT, + completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0, 1)) + ); + + INSERT OR IGNORE INTO import_pending_reason_repairs (inventory_family) + VALUES ('catalog_sessions'), ('source_import_files'); + "#, + )?; + conn.execute_batch("PRAGMA user_version = 51;")?; + Ok(()) + })(); + + match migration { + Ok(()) => { + conn.execute_batch("COMMIT;")?; + Ok(()) + } + Err(err) => { + if let Err(rollback_err) = conn.execute_batch("ROLLBACK;") { + return Err(StoreError::Sql(rollback_err)); + } + Err(err) + } + } +} + +pub(super) fn migrate_to_v52(conn: &Connection) -> Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let migration = (|| -> Result<()> { + if !table_has_column( + conn, + "provider_file_publications", + "completion_payload_json", + )? { + conn.execute_batch( + r#" + ALTER TABLE main.provider_file_publications + ADD COLUMN completion_payload_json TEXT CHECK ( + completion_payload_json IS NULL OR + length(CAST(completion_payload_json AS BLOB)) BETWEEN 1 AND 262144 + ); + "#, + )?; + } + conn.execute_batch("PRAGMA user_version = 52;")?; + Ok(()) + })(); + + match migration { + Ok(()) => { + conn.execute_batch("COMMIT;")?; + Ok(()) + } + Err(err) => { + if let Err(rollback_err) = conn.execute_batch("ROLLBACK;") { + return Err(StoreError::Sql(rollback_err)); + } + Err(err) + } + } +} + +pub(super) fn migrate_to_v53(conn: &Connection) -> Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let migration = (|| -> Result<()> { + if !table_has_column( + conn, + "provider_file_publications", + "inventory_observation_invalidated", + )? { + conn.execute_batch( + r#" + ALTER TABLE main.provider_file_publications + ADD COLUMN inventory_observation_invalidated INTEGER NOT NULL DEFAULT 0 + CHECK (inventory_observation_invalidated IN (0, 1)); + "#, + )?; + } + if !table_has_column(conn, "provider_file_publications", "retirement_started")? { + conn.execute_batch( + r#" + ALTER TABLE main.provider_file_publications + ADD COLUMN retirement_started INTEGER NOT NULL DEFAULT 0 + CHECK (retirement_started IN (0, 1)); + "#, + )?; + } + // v51 originally classified some mutated incremental publications as + // not tracking prior material. Recovery must reconcile conservatively + // after any mutation, even when no append checkpoint survived. + conn.execute_batch( + r#" + UPDATE provider_file_publications + SET tracks_prior_material = 1 + WHERE mutation_started != 0 + AND tracks_prior_material = 0; + "#, + )?; + conn.execute_batch("PRAGMA user_version = 53;")?; + Ok(()) + })(); + + match migration { + Ok(()) => { + conn.execute_batch("COMMIT;")?; + Ok(()) + } + Err(err) => { + if let Err(rollback_err) = conn.execute_batch("ROLLBACK;") { + return Err(StoreError::Sql(rollback_err)); + } + Err(err) + } + } +} + +fn migrate_to_v54(conn: &Connection) -> Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let migration = (|| -> Result<()> { + drop_stable_sql_views(conn)?; + create_stable_sql_views(conn)?; + conn.execute_batch("PRAGMA user_version = 54;")?; + Ok(()) + })(); + + match migration { + Ok(()) => { + conn.execute_batch("COMMIT;")?; + Ok(()) + } + Err(err) => { + if let Err(rollback_err) = conn.execute_batch("ROLLBACK;") { + return Err(StoreError::Sql(rollback_err)); + } + Err(err) + } + } +} + fn backfill_capture_source_identity_columns(conn: &Connection) -> Result<()> { if !table_exists(conn, "capture_sources")? { return Ok(()); diff --git a/crates/ctx-history-store/src/schema/mod.rs b/crates/ctx-history-store/src/schema/mod.rs index 743a59dd..dfa2912d 100644 --- a/crates/ctx-history-store/src/schema/mod.rs +++ b/crates/ctx-history-store/src/schema/mod.rs @@ -11,7 +11,7 @@ pub(crate) mod views; use rusqlite::Connection; use crate::connection::configure_connection; -use crate::schema::indexes::INDEXES_SQL; +use crate::schema::indexes::{INDEXES_SQL, PENDING_WORK_INDEXES_SQL}; use crate::{Result, Store, StoreError, SCHEMA_VERSION}; pub(crate) use fts::create_fts_tables_if_supported; @@ -24,6 +24,7 @@ pub(crate) fn migrate_to_latest(conn: &Connection) -> Result<()> { migrations::run_migrations(conn, user_version)?; create_fts_tables_if_supported(conn)?; conn.execute_batch(INDEXES_SQL)?; + conn.execute_batch(PENDING_WORK_INDEXES_SQL)?; Ok(()) } diff --git a/crates/ctx-history-store/src/schema/tests.rs b/crates/ctx-history-store/src/schema/tests.rs index d3d6fb44..48003ffc 100644 --- a/crates/ctx-history-store/src/schema/tests.rs +++ b/crates/ctx-history-store/src/schema/tests.rs @@ -1,4 +1,8 @@ -use std::{collections::BTreeSet, fs}; +use std::{ + collections::BTreeSet, + fs, + sync::{Arc, Mutex}, +}; use chrono::{DateTime, Utc}; use ctx_history_core::{ @@ -6,13 +10,18 @@ use ctx_history_core::{ Event, EventRole, EventType, Fidelity, Session, SessionHistoryArchive, SessionStatus, SyncMetadata, SyncState, Visibility, }; -use rusqlite::{params, Connection}; +use rusqlite::{ + hooks::{AuthAction, AuthContext, Authorization}, + params, Connection, +}; use uuid::Uuid; +use crate::provider_files::PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES; use crate::schema::ddl::{table_exists, table_has_column, CREATE_TABLES_SQL}; use crate::schema::fts::FTS_TABLES_SQL; use crate::schema::indexes::INDEXES_SQL; use crate::schema::migrations::{ + migrate_to_v50, migrate_to_v51, migrate_to_v52, migrate_to_v53, rebuild_capture_sources_provider_check, rebuild_catalog_sessions_provider_check, }; use crate::{Store, SCHEMA_VERSION}; @@ -1145,6 +1154,42 @@ fn schema_v22_adds_forgecode_provider_checks() { .unwrap(); } +#[test] +fn v54_migration_recreates_stable_views_with_inventory_publication_fencing() { + let temp = tempdir(); + let path = temp.path().join("v54-stable-views.sqlite"); + drop(Store::open(&path).unwrap()); + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "DROP VIEW ctx_sources;\ + CREATE VIEW ctx_sources AS SELECT 1 AS stale;\ + PRAGMA user_version = 53;", + ) + .unwrap(); + } + + let store = Store::open(&path).unwrap(); + let view_sql: String = store + .conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type = 'view' AND name = 'ctx_sources'", + [], + |row| row.get(0), + ) + .unwrap(); + + assert!(view_sql.contains("import_inventory_generations")); + assert!(!view_sql.contains("SELECT 1 AS stale")); + assert_eq!( + store + .conn + .query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0)) + .unwrap(), + SCHEMA_VERSION + ); +} + #[test] fn schema_v23_adds_mistral_vibe_provider_checks() { let temp = tempdir(); @@ -1444,4 +1489,5 @@ fn schema_v46_adds_mimocode_provider_checks() { "/tmp/mimocode/mimocode.db", ); } + include!("tests/import_contract.rs"); diff --git a/crates/ctx-history-store/src/schema/tests/import_contract.rs b/crates/ctx-history-store/src/schema/tests/import_contract.rs index 4bf9d3ff..b8087467 100644 --- a/crates/ctx-history-store/src/schema/tests/import_contract.rs +++ b/crates/ctx-history-store/src/schema/tests/import_contract.rs @@ -346,6 +346,864 @@ fn real_schema_v49_fixture_adds_provider_file_contract_tables() { assert_schema_object_parity(&store.conn, &fresh.conn); } +#[test] +fn real_schema_v49_fixture_stages_pending_reason_repairs() { + let temp = tempdir(); + let path = temp.path().join("pending-reasons.sqlite"); + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch(include_str!("../fixtures/schema_v49.sql")) + .unwrap(); + conn.execute_batch(include_str!("../fixtures/pending_reason_v49_rows.sql")) + .unwrap(); + } + + let store = Store::open(&path).unwrap(); + let repairs = store + .conn + .prepare( + "SELECT inventory_family, cursor_provider, cursor_source_root, \ + cursor_source_path, completed \ + FROM import_pending_reason_repairs ORDER BY inventory_family", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, bool>(4)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + repairs, + vec![ + ("catalog_sessions".into(), None, None, None, false), + ("source_import_files".into(), None, None, None, false), + ] + ); + + let pending_reason_count: usize = store + .conn + .query_row( + "SELECT (SELECT COUNT(*) FROM catalog_sessions WHERE pending_reason IS NOT NULL) + \ + (SELECT COUNT(*) FROM source_import_files WHERE pending_reason IS NOT NULL)", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(pending_reason_count, 0); +} + +#[test] +fn v51_migration_preserves_rows_and_repair_progress_when_retried() { + let temp = tempdir(); + let path = temp.path().join("v51-failure-reasons.sqlite"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch(CREATE_TABLES_SQL).unwrap(); + for (index, prior_reason) in [ + (1, Some("fresh_append")), + (2, Some("recovery_retry")), + (3, Some("fresh_changed")), + (4, Some("recovery_replacement")), + (5, None), + ] { + conn.execute( + r#" + INSERT INTO source_import_files ( + provider, source_format, source_root, source_path, + file_size_bytes, file_modified_at_ms, observed_at_ms, + indexed_status, pending_reason + ) VALUES ('pi', 'pi_session_jsonl', ?1, ?2, 1, 1, 1, 'failed', ?3) + "#, + params![ + format!("/fixture/failure-{index}"), + format!("/fixture/failure-{index}/session.jsonl"), + prior_reason, + ], + ) + .unwrap(); + } + + migrate_to_v51(&conn).unwrap(); + let reasons = conn + .prepare("SELECT pending_reason FROM source_import_files ORDER BY source_root") + .unwrap() + .query_map([], |row| row.get::<_, Option>(0)) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + reasons, + vec![ + Some("fresh_append".into()), + Some("recovery_retry".into()), + Some("fresh_changed".into()), + Some("recovery_replacement".into()), + None, + ] + ); + + conn.execute( + r#" + UPDATE import_pending_reason_repairs + SET cursor_provider = 'pi', cursor_source_root = '/fixture/failure-5', + cursor_source_path = '/fixture/failure-5/session.jsonl', completed = 1 + WHERE inventory_family = 'source_import_files' + "#, + [], + ) + .unwrap(); + migrate_to_v51(&conn).unwrap(); + + let repair = conn + .query_row( + r#" + SELECT cursor_provider, cursor_source_root, cursor_source_path, completed + FROM import_pending_reason_repairs + WHERE inventory_family = 'source_import_files' + "#, + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, bool>(3)?, + )) + }, + ) + .unwrap(); + assert_eq!( + repair, + ( + "pi".into(), + "/fixture/failure-5".into(), + "/fixture/failure-5/session.jsonl".into(), + true, + ) + ); +} + +#[test] +fn v51_legacy_publications_reconstruct_without_sidecars_and_match_fresh_schema() { + let temp = tempdir(); + let path = temp.path().join("v51-publication-staging.sqlite"); + { + let conn = Connection::open(&path).unwrap(); + let legacy_sql = CREATE_TABLES_SQL + .replace( + " tracks_prior_material INTEGER NOT NULL DEFAULT 0 CHECK (tracks_prior_material IN (0, 1)),\n", + "", + ) + .replace( + " staging_initialized INTEGER NOT NULL DEFAULT 0 CHECK (staging_initialized IN (0, 1)),\n", + "", + ) + .replace( + ",\n pending_reason TEXT CHECK (pending_reason IS NULL OR pending_reason IN ('fresh_new', 'fresh_changed', 'fresh_append', 'recovery_retry', 'recovery_replacement', 'parser_revision', 'missing_material', 'abandoned_publication', 'legacy', 'explicit_rescan'))", + "", + ); + assert!(!legacy_sql.contains("staging_initialized")); + assert!(!legacy_sql.contains("tracks_prior_material")); + conn.execute_batch(&legacy_sql).unwrap(); + assert!(!table_has_column(&conn, "catalog_sessions", "pending_reason").unwrap()); + assert!(!table_has_column(&conn, "source_import_files", "pending_reason").unwrap()); + conn.execute_batch("PRAGMA user_version = 49;").unwrap(); + migrate_to_v50(&conn).unwrap(); + conn.execute_batch( + r#" + INSERT INTO provider_file_publications ( + replacement_id, owner_id, publication_kind, staging_id, provider, + inventory_family, inventory_source_format, inventory_source_root, + source_path, material_source_format, material_source_root, + inventory_generation, file_size_bytes, file_modified_at_ms, + import_revision, mutation_started, preparation_complete, preparation_cursor, + cleanup_phase, cleanup_source_cursor, cleanup_entity_cursor, + removed_events, started_at_ms, updated_at_ms + ) VALUES + ( + 'legacy-mutated', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'replacement', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'codex', + 'catalog_sessions', 'codex_session_jsonl', '/legacy/inventory', + '/legacy/mutated.jsonl', 'codex_session_jsonl', '/legacy/material', + 1, 10, 20, 1, 1, 1, 'prior-source', 4, 'prior-source', 'event-cursor', + 7, 30, 30 + ), + ( + 'legacy-unmutated', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'replacement', 'dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', 'codex', + 'catalog_sessions', 'codex_session_jsonl', '/legacy/inventory', + '/legacy/unmutated.jsonl', 'codex_session_jsonl', '/legacy/material', + 1, 10, 20, 1, 0, 1, 'prior-source', 0, NULL, NULL, + 0, 31, 31 + ), + ( + 'legacy-mutated-incremental', 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'incremental', 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', 'codex', + 'catalog_sessions', 'codex_session_jsonl', '/legacy/inventory', + '/legacy/incremental.jsonl', 'codex_session_jsonl', '/legacy/material', + 1, 10, 20, 1, 1, 1, NULL, 0, NULL, NULL, + 0, 32, 32 + ); + "#, + ) + .unwrap(); + } + + let upgraded = Store::open(&path).unwrap(); + let mut statement = upgraded + .conn + .prepare( + "SELECT replacement_id, mutation_started, tracks_prior_material, staging_initialized, \ + preparation_complete, preparation_cursor, cleanup_phase, \ + cleanup_source_cursor, cleanup_entity_cursor, removed_events \ + FROM provider_file_publications ORDER BY replacement_id", + ) + .unwrap(); + let publications = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, bool>(2)?, + row.get::<_, bool>(3)?, + row.get::<_, bool>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, i64>(9)?, + )) + }) + .unwrap(); + let publications = publications.collect::>>().unwrap(); + assert_eq!( + publications, + vec![ + ( + "legacy-mutated".into(), + true, + true, + false, + false, + None, + 0, + None, + None, + 0, + ), + ( + "legacy-mutated-incremental".into(), + true, + true, + false, + true, + None, + 0, + None, + None, + 0, + ), + ( + "legacy-unmutated".into(), + false, + false, + false, + false, + None, + 0, + None, + None, + 0, + ), + ] + ); + for table in [ + "provider_file_publication_seen", + "provider_file_publication_prior_sources", + "provider_file_publication_batch", + ] { + let count: i64 = upgraded + .conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(count, 0, "{table}"); + } + let default_value: String = upgraded + .conn + .query_row( + "SELECT dflt_value FROM pragma_table_info('provider_file_publications') \ + WHERE name = 'staging_initialized'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(default_value, "0"); + + let fresh = Store::open(temp.path().join("fresh-v51.sqlite")).unwrap(); + assert_schema_object_parity(&upgraded.conn, &fresh.conn); +} + +#[test] +fn v51_migration_defers_material_classification_to_bounded_repair() { + let temp = tempdir(); + let path = temp.path().join("v51-material-owners.sqlite"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch(CREATE_TABLES_SQL).unwrap(); + conn.execute_batch( + r#" + INSERT INTO catalog_sessions ( + source_path, provider, source_format, source_root, external_session_id, + agent_type, file_size_bytes, file_modified_at_ms, import_revision, + cataloged_at_ms, indexed_at_ms, indexed_file_size_bytes, + indexed_file_modified_at_ms, indexed_status, indexed_import_revision + ) VALUES + ('/fixture/catalog/correct.jsonl', 'codex', 'codex_session_jsonl', + '/fixture/catalog', 'correct', 'primary', 1, 1, 1, 1, 2, 1, 1, 'indexed', 1), + ('/fixture/catalog/unowned.jsonl', 'codex', 'codex_session_jsonl', + '/fixture/catalog', 'unowned', 'primary', 1, 1, 1, 1, 2, 1, 1, 'indexed', 1), + ('/fixture/catalog/wrong-format.jsonl', 'codex', 'codex_session_jsonl', + '/fixture/catalog', 'wrong-format', 'primary', 1, 1, 1, 1, 2, 1, 1, 'indexed', 1), + ('/fixture/catalog/wrong-root.jsonl', 'codex', 'codex_session_jsonl', + '/fixture/catalog', 'wrong-root', 'primary', 1, 1, 1, 1, 2, 1, 1, 'indexed', 1); + + INSERT INTO source_import_files ( + provider, source_format, source_root, source_path, file_size_bytes, + file_modified_at_ms, import_revision, observed_at_ms, indexed_at_ms, + indexed_file_size_bytes, indexed_file_modified_at_ms, indexed_status, + indexed_import_revision, metadata_json + ) VALUES + ('pi', 'pi_session_jsonl', '/fixture/source', '/fixture/source/correct.jsonl', + 1, 1, 1, 1, 2, 1, 1, 'indexed', 1, '{"inventory_unit":"logical_import_unit"}'), + ('mistral_vibe', 'mistral_vibe_session_jsonl_tree', '/fixture/source', + '/fixture/source/mistral/messages.jsonl', + 1, 1, 1, 1, 2, 1, 1, 'indexed', 1, '{"inventory_unit":"logical_import_unit"}'), + ('pi', 'pi_session_jsonl', '/fixture/source', '/fixture/source/wrong-format.jsonl', + 1, 1, 1, 1, 2, 1, 1, 'indexed', 1, '{"inventory_unit":"logical_import_unit"}'), + ('pi', 'pi_session_jsonl', '/fixture/source', '/fixture/source/wrong-root.jsonl', + 1, 1, 1, 1, 2, 1, 1, 'indexed', 1, '{"inventory_unit":"logical_import_unit"}'); + + INSERT INTO capture_sources ( + id, kind, provider, machine_id, raw_source_path, source_format, + source_root, external_session_id, started_at_ms, fidelity + ) VALUES + ('catalog-correct', 'provider_import', 'codex', 'fixture', + '/fixture/catalog/correct.jsonl', 'codex_session_jsonl', + '/fixture/catalog', 'correct', 1, 'imported'), + ('catalog-wrong-format', 'provider_import', 'codex', 'fixture', + '/fixture/catalog/wrong-format.jsonl', 'codex_session_jsonl_tree', + '/fixture/catalog', 'wrong-format', 1, 'imported'), + ('catalog-wrong-root', 'provider_import', 'codex', 'fixture', + '/fixture/catalog/wrong-root.jsonl', 'codex_session_jsonl', + '/fixture/catalog/other', 'wrong-root', 1, 'imported'), + ('source-correct', 'provider_import', 'pi', 'fixture', + '/fixture/source/correct.jsonl', 'pi_session_jsonl', + '/fixture/source', NULL, 1, 'imported'), + ('source-mistral', 'provider_import', 'mistral_vibe', 'fixture', + '/fixture/source/mistral/messages.jsonl', 'mistral_vibe_session_jsonl', + '/fixture/source/mistral/messages.jsonl', NULL, 1, 'imported'), + ('source-wrong-format', 'provider_import', 'pi', 'fixture', + '/fixture/source/wrong-format.jsonl', 'pi_session_json', + '/fixture/source', NULL, 1, 'imported'), + ('source-wrong-root', 'provider_import', 'pi', 'fixture', + '/fixture/source/wrong-root.jsonl', 'pi_session_jsonl', + '/fixture/source/other', NULL, 1, 'imported'); + + INSERT INTO sessions ( + id, capture_source_id, provider, external_session_id, agent_type, + status, fidelity, started_at_ms, created_at_ms, updated_at_ms + ) VALUES + ('session-correct', 'catalog-correct', 'codex', 'correct', + 'primary', 'imported', 'imported', 1, 1, 1), + ('session-unowned', NULL, 'codex', 'unowned', + 'primary', 'imported', 'imported', 1, 1, 1), + ('session-wrong-format', 'catalog-wrong-format', 'codex', 'wrong-format', + 'primary', 'imported', 'imported', 1, 1, 1), + ('session-wrong-root', 'catalog-wrong-root', 'codex', 'wrong-root', + 'primary', 'imported', 'imported', 1, 1, 1); + "#, + ) + .unwrap(); + + migrate_to_v51(&conn).unwrap(); + + let catalog_reasons = conn + .prepare("SELECT source_path, pending_reason FROM catalog_sessions ORDER BY source_path") + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + catalog_reasons, + vec![ + ("/fixture/catalog/correct.jsonl".into(), None), + ("/fixture/catalog/unowned.jsonl".into(), None), + ("/fixture/catalog/wrong-format.jsonl".into(), None), + ("/fixture/catalog/wrong-root.jsonl".into(), None), + ] + ); + let source_reasons = conn + .prepare("SELECT source_path, pending_reason FROM source_import_files ORDER BY source_path") + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!( + source_reasons, + vec![ + ("/fixture/source/correct.jsonl".into(), None), + ("/fixture/source/mistral/messages.jsonl".into(), None), + ("/fixture/source/wrong-format.jsonl".into(), None), + ("/fixture/source/wrong-root.jsonl".into(), None), + ] + ); + let incomplete_repairs: usize = conn + .query_row( + "SELECT COUNT(*) FROM import_pending_reason_repairs WHERE completed = 0", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(incomplete_repairs, 2); +} + +#[test] +fn v51_migration_does_not_rewrite_inventory_rows_or_churn_indexes() { + let temp = tempdir(); + let path = temp.path().join("v51-no-rebuild.sqlite"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch(include_str!("../fixtures/schema_v49.sql")) + .unwrap(); + migrate_to_v50(&conn).unwrap(); + let rootpages_before = ["catalog_sessions", "source_import_files"].map(|table| { + conn.query_row( + "SELECT rootpage FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + |row| row.get::<_, i64>(0), + ) + .unwrap() + }); + let total_changes_before = conn.total_changes(); + + let observed = Arc::new(Mutex::new(Vec::new())); + let callback_observed = Arc::clone(&observed); + conn.authorizer(Some(move |context: AuthContext<'_>| { + let (description, forbidden) = match context.action { + AuthAction::AlterTable { table_name, .. } + if matches!(table_name, "catalog_sessions" | "source_import_files") => + { + (format!("alter:{table_name}"), false) + } + AuthAction::Update { table_name, .. } + if matches!(table_name, "catalog_sessions" | "source_import_files") => + { + (format!("forbidden-update:{table_name}"), true) + } + AuthAction::CreateIndex { table_name, .. } + if matches!( + table_name, + "capture_sources" | "catalog_sessions" | "source_import_files" + ) => + { + (format!("forbidden-index:{table_name}"), true) + } + AuthAction::CreateTable { table_name } + | AuthAction::DropTable { table_name } + | AuthAction::Insert { table_name } + if matches!( + table_name, + "catalog_sessions" + | "catalog_sessions_new" + | "source_import_files" + | "source_import_files_new" + ) => + { + (format!("forbidden:{table_name}"), true) + } + _ => return Authorization::Allow, + }; + callback_observed.lock().unwrap().push(description); + if forbidden { + Authorization::Deny + } else { + Authorization::Allow + } + })); + + migrate_to_v51(&conn).unwrap(); + let observed = observed.lock().unwrap(); + assert!(observed + .iter() + .any(|action| action == "alter:catalog_sessions")); + assert!(observed + .iter() + .any(|action| action == "alter:source_import_files")); + assert!(!observed + .iter() + .any(|action| action.starts_with("forbidden"))); + drop(observed); + assert_eq!(conn.total_changes() - total_changes_before, 2); + + let rootpages_after = ["catalog_sessions", "source_import_files"].map(|table| { + conn.query_row( + "SELECT rootpage FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + |row| row.get::<_, i64>(0), + ) + .unwrap() + }); + assert_eq!(rootpages_after, rootpages_before); +} + +#[test] +fn v52_migration_adds_bounded_completion_without_row_or_index_churn() { + let temp = tempdir(); + let path = temp.path().join("v52-publication-completion.sqlite"); + let conn = Connection::open(&path).unwrap(); + let legacy_sql = CREATE_TABLES_SQL + .replace( + " completion_payload_json TEXT CHECK (\n completion_payload_json IS NULL OR\n length(CAST(completion_payload_json AS BLOB)) BETWEEN 1 AND 262144\n ),\n", + "", + ) + .replace( + " inventory_observation_invalidated INTEGER NOT NULL DEFAULT 0\n CHECK (inventory_observation_invalidated IN (0, 1)),\n", + "", + ) + .replace( + " retirement_started INTEGER NOT NULL DEFAULT 0 CHECK (retirement_started IN (0, 1)),\n", + "", + ); + assert!(!legacy_sql.contains("completion_payload_json")); + assert!(!legacy_sql.contains("inventory_observation_invalidated")); + assert!(!legacy_sql.contains("retirement_started")); + conn.execute_batch(&legacy_sql).unwrap(); + conn.execute_batch(INDEXES_SQL).unwrap(); + conn.execute_batch("PRAGMA user_version = 51;").unwrap(); + conn.execute_batch( + r#" + INSERT INTO provider_file_publications ( + replacement_id, owner_id, publication_kind, staging_id, provider, + inventory_family, inventory_source_format, inventory_source_root, + source_path, material_source_format, material_source_root, + inventory_generation, file_size_bytes, file_modified_at_ms, + import_revision, preparation_complete, started_at_ms, updated_at_ms + ) VALUES ( + 'v51-publication', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'replacement', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'claude', 'source_import_files', 'claude_projects_jsonl_tree', + '/history/claude/projects', '/history/claude/projects/a.jsonl', + 'claude_projects_jsonl', '/history/claude/projects', + 1, 20, 100, 7, 1, 105, 105 + ); + "#, + ) + .unwrap(); + let schema_objects = [ + "provider_file_publications", + "idx_provider_file_publications_owner", + "idx_provider_file_publications_fence", + ]; + let rootpages_before = schema_objects.map(|name| { + conn.query_row( + "SELECT rootpage FROM sqlite_schema WHERE name = ?1", + [name], + |row| row.get::<_, i64>(0), + ) + .unwrap() + }); + let total_changes_before = conn.total_changes(); + let observed = Arc::new(Mutex::new(Vec::new())); + let callback_observed = Arc::clone(&observed); + conn.authorizer(Some(move |context: AuthContext<'_>| { + let (description, forbidden) = match context.action { + AuthAction::AlterTable { table_name, .. } + if table_name == "provider_file_publications" => + { + (format!("alter:{table_name}"), false) + } + AuthAction::CreateIndex { table_name, .. } + if table_name == "provider_file_publications" => + { + (format!("forbidden-index:{table_name}"), true) + } + AuthAction::CreateTable { table_name } | AuthAction::DropTable { table_name } + if matches!( + table_name, + "provider_file_publications" | "provider_file_publications_new" + ) => + { + (format!("forbidden-table:{table_name}"), true) + } + AuthAction::Insert { table_name } + | AuthAction::Update { table_name, .. } + | AuthAction::Delete { table_name } + if table_name == "provider_file_publications" => + { + (format!("forbidden-row-write:{table_name}"), true) + } + _ => return Authorization::Allow, + }; + callback_observed.lock().unwrap().push(description); + if forbidden { + Authorization::Deny + } else { + Authorization::Allow + } + })); + + migrate_to_v52(&conn).unwrap(); + assert_eq!(conn.total_changes(), total_changes_before); + assert!(table_has_column( + &conn, + "provider_file_publications", + "completion_payload_json" + ) + .unwrap()); + let row: (String, Option) = conn + .query_row( + "SELECT replacement_id, completion_payload_json \ + FROM provider_file_publications", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(row, ("v51-publication".into(), None)); + let rootpages_after = schema_objects.map(|name| { + conn.query_row( + "SELECT rootpage FROM sqlite_schema WHERE name = ?1", + [name], + |row| row.get::<_, i64>(0), + ) + .unwrap() + }); + assert_eq!(rootpages_after, rootpages_before); + let observed = observed.lock().unwrap(); + assert!(observed + .iter() + .any(|action| action == "alter:provider_file_publications")); + assert!(!observed + .iter() + .any(|action| action.starts_with("forbidden"))); + drop(observed); + drop(conn); + + let conn = Connection::open(&path).unwrap(); + let maximum_json = format!( + "\"{}\"", + "x".repeat(PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES - 2) + ); + assert_eq!( + maximum_json.len(), + PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES + ); + conn.execute( + "UPDATE provider_file_publications SET completion_payload_json = ?1", + [&maximum_json], + ) + .unwrap(); + let oversized_json = format!( + "\"{}\"", + "x".repeat(PROVIDER_FILE_PUBLICATION_COMPLETION_MAX_BYTES - 1) + ); + assert!(conn + .execute( + "UPDATE provider_file_publications SET completion_payload_json = ?1", + [&oversized_json], + ) + .is_err()); + migrate_to_v52(&conn).unwrap(); + assert_eq!( + conn.query_row( + "SELECT completion_payload_json FROM provider_file_publications", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(), + maximum_json + ); + + drop(conn); + let upgraded = Store::open(&path).unwrap(); + let fresh = Store::open(temp.path().join("fresh-v52.sqlite")).unwrap(); + let publication_schema = |conn: &Connection| { + schema_object_signature(conn) + .into_iter() + .filter(|(_, name, _)| { + matches!( + name.as_str(), + "provider_file_publications" + | "idx_provider_file_publications_owner" + | "idx_provider_file_publications_fence" + ) + }) + .collect::>() + }; + assert_eq!( + publication_schema(&upgraded.conn), + publication_schema(&fresh.conn) + ); +} + +#[test] +fn v53_migration_adds_durable_retirement_state_without_row_or_index_churn() { + let temp = tempdir(); + let path = temp.path().join("v53-publication-retirement.sqlite"); + let conn = Connection::open(&path).unwrap(); + let legacy_sql = CREATE_TABLES_SQL + .replace( + " inventory_observation_invalidated INTEGER NOT NULL DEFAULT 0\n CHECK (inventory_observation_invalidated IN (0, 1)),\n", + "", + ) + .replace( + " retirement_started INTEGER NOT NULL DEFAULT 0 CHECK (retirement_started IN (0, 1)),\n", + "", + ); + assert!(!legacy_sql.contains("retirement_started")); + assert!(!legacy_sql.contains("inventory_observation_invalidated")); + conn.execute_batch(&legacy_sql).unwrap(); + conn.execute_batch(INDEXES_SQL).unwrap(); + conn.execute_batch("PRAGMA user_version = 52;").unwrap(); + conn.execute_batch( + r#" + INSERT INTO provider_file_publications ( + replacement_id, owner_id, publication_kind, staging_id, provider, + inventory_family, inventory_source_format, inventory_source_root, + source_path, material_source_format, material_source_root, + inventory_generation, file_size_bytes, file_modified_at_ms, + import_revision, preparation_complete, started_at_ms, updated_at_ms + ) VALUES ( + 'v52-publication', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'replacement', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'claude', 'source_import_files', 'claude_projects_jsonl_tree', + '/history/claude/projects', '/history/claude/projects/a.jsonl', + 'claude_projects_jsonl', '/history/claude/projects', + 1, 20, 100, 7, 1, 105, 105 + ); + "#, + ) + .unwrap(); + let rootpage_before: i64 = conn + .query_row( + "SELECT rootpage FROM sqlite_schema WHERE name = 'provider_file_publications'", + [], + |row| row.get(0), + ) + .unwrap(); + let total_changes_before = conn.total_changes(); + + migrate_to_v53(&conn).unwrap(); + assert_eq!(conn.total_changes(), total_changes_before); + assert!(table_has_column(&conn, "provider_file_publications", "retirement_started").unwrap()); + assert!(table_has_column( + &conn, + "provider_file_publications", + "inventory_observation_invalidated", + ) + .unwrap()); + assert!(!conn + .query_row( + "SELECT retirement_started FROM provider_file_publications", + [], + |row| row.get::<_, bool>(0), + ) + .unwrap()); + assert!(!conn + .query_row( + "SELECT inventory_observation_invalidated FROM provider_file_publications", + [], + |row| row.get::<_, bool>(0), + ) + .unwrap()); + assert_eq!( + conn.query_row( + "SELECT rootpage FROM sqlite_schema WHERE name = 'provider_file_publications'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + rootpage_before + ); + migrate_to_v53(&conn).unwrap(); + drop(conn); + + let upgraded = Store::open(&path).unwrap(); + let fresh = Store::open(temp.path().join("fresh-v53.sqlite")).unwrap(); + assert_eq!( + schema_object_signature(&upgraded.conn) + .into_iter() + .find(|(_, name, _)| name == "provider_file_publications"), + schema_object_signature(&fresh.conn) + .into_iter() + .find(|(_, name, _)| name == "provider_file_publications") + ); +} + +#[test] +fn v53_migration_repairs_mutated_publications_that_lost_prior_material_scope() { + let conn = Connection::open_in_memory().unwrap(); + let legacy_sql = CREATE_TABLES_SQL + .replace( + " inventory_observation_invalidated INTEGER NOT NULL DEFAULT 0\n CHECK (inventory_observation_invalidated IN (0, 1)),\n", + "", + ) + .replace( + " retirement_started INTEGER NOT NULL DEFAULT 0 CHECK (retirement_started IN (0, 1)),\n", + "", + ); + conn.execute_batch(&legacy_sql).unwrap(); + conn.execute_batch(INDEXES_SQL).unwrap(); + conn.execute_batch("PRAGMA user_version = 52;").unwrap(); + conn.execute_batch( + r#" + INSERT INTO provider_file_publications ( + replacement_id, owner_id, publication_kind, staging_id, provider, + inventory_family, inventory_source_format, inventory_source_root, + source_path, material_source_format, material_source_root, + inventory_generation, file_size_bytes, file_modified_at_ms, + import_revision, mutation_started, tracks_prior_material, + preparation_complete, started_at_ms, updated_at_ms + ) VALUES ( + 'v52-mutated-incremental', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'incremental', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'codex', 'source_import_files', 'codex_sessions_root', + '/history/codex/sessions', '/history/codex/sessions/a.jsonl', + 'codex_session_jsonl', '/history/codex/sessions', + 1, 20, 100, 7, 1, 0, 1, 105, 105 + ); + "#, + ) + .unwrap(); + + migrate_to_v53(&conn).unwrap(); + + assert!(conn + .query_row( + "SELECT tracks_prior_material FROM provider_file_publications", + [], + |row| row.get::<_, bool>(0), + ) + .unwrap()); +} + fn schema_object_signature(conn: &Connection) -> Vec<(String, String, String)> { conn.prepare( r#" diff --git a/crates/ctx-history-store/src/sessions.rs b/crates/ctx-history-store/src/sessions.rs index 78649970..5f5aca30 100644 --- a/crates/ctx-history-store/src/sessions.rs +++ b/crates/ctx-history-store/src/sessions.rs @@ -85,7 +85,7 @@ impl Store { pub fn get_session(&self, id: Uuid) -> Result { let tail = format!( "WHERE id = ?1 AND {}", - crate::provider_files::session_material_visible_predicate("sessions") + self.importer_session_material_visible_predicate("sessions") ); self.conn .query_row( @@ -112,7 +112,7 @@ impl Store { provider: CaptureProvider, external_session_id: &str, ) -> Result> { - let visible = crate::provider_files::session_material_visible_predicate("sessions"); + let visible = self.importer_session_material_visible_predicate("sessions"); self.conn .query_row( session_select_sql(&format!( @@ -132,7 +132,7 @@ impl Store { provider: CaptureProvider, external_session_id: &str, ) -> Result> { - let visible = crate::provider_files::session_material_visible_predicate("sessions"); + let visible = self.importer_session_material_visible_predicate("sessions"); self.conn .query_row( session_select_sql(&format!( @@ -158,7 +158,7 @@ impl Store { ) -> Result> { let tail = format!( "WHERE provider = ?1 AND external_session_id = ?2 AND {} ORDER BY started_at_ms DESC LIMIT ?3", - crate::provider_files::session_material_visible_predicate("sessions") + self.importer_session_material_visible_predicate("sessions") ); let mut stmt = self.conn.prepare(session_select_sql(&tail).as_str())?; let rows = stmt.query_map( diff --git a/crates/ctx-history-store/src/sources.rs b/crates/ctx-history-store/src/sources.rs index 45a6877b..47eb6a2d 100644 --- a/crates/ctx-history-store/src/sources.rs +++ b/crates/ctx-history-store/src/sources.rs @@ -68,12 +68,12 @@ impl Store { serde_json::to_string(&source.sync.metadata)?, ], )?; + self.track_provider_file_publication_capture_source(source.id)?; Ok(()) } pub fn get_capture_source(&self, id: Uuid) -> Result { - let visible = - crate::provider_files::capture_source_material_visible_predicate("capture_sources"); + let visible = self.importer_capture_source_material_visible_predicate("capture_sources"); self.conn .query_row( &format!("SELECT id, kind, provider, machine_id, process_id, cwd, raw_source_path, source_format, source_root, source_identity, external_session_id, started_at_ms, ended_at_ms, fidelity, visibility, sync_state, sync_version, metadata_json FROM capture_sources WHERE id = ?1 AND {visible}"), @@ -110,8 +110,7 @@ impl Store { provider: CaptureProvider, external_session_id: &str, ) -> Result> { - let visible = - crate::provider_files::capture_source_material_visible_predicate("capture_sources"); + let visible = self.importer_capture_source_material_visible_predicate("capture_sources"); self.conn .query_row( &format!("SELECT id, kind, provider, machine_id, process_id, cwd, raw_source_path, source_format, source_root, source_identity, external_session_id, started_at_ms, ended_at_ms, fidelity, visibility, sync_state, sync_version, metadata_json FROM capture_sources WHERE provider = ?1 AND external_session_id = ?2 AND {visible} ORDER BY started_at_ms DESC LIMIT 1"), diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 413cdc78..50355b33 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -69,8 +69,11 @@ ctx daemon enable semantic is enabled. The daemon may acquire the local embedding model for semantic indexing. A looping daemon keeps the embedding model resident after cold start and performs recent-work freshness checks before settling into idle - loops; cloud sync remains disabled with `enabled: false` and - `network_allowed: false`. + loops. Native refresh charges measured source reads, logical write work, and + observed FTS maintenance WAL size to an internal 8 MiB/s I/O budget with a 1 + MiB burst; SQLite and filesystem write amplification mean this is not a hard + physical-device bandwidth cap. Cloud sync remains disabled with + `enabled: false` and `network_allowed: false`. - `daemon disable` and `daemon enable` update `[daemon].enabled` in `config.toml`. The prerelease default is disabled, so daemon maintenance is an explicit opt-in; `daemon run --force` overrides a disabled config for explicit @@ -267,6 +270,13 @@ or index failures abort the command. A source with only rejected records is a failure; a source with valid content and rejections completes with an explicit `completed_with_rejections` outcome. +Foreground import charges source reads, logical write work, and observed FTS +maintenance WAL size to an internal bounded I/O budget (64 MiB/s with an 8 MiB +burst). This has no command-line or configuration option: oversized units still +make durable progress, while pacing occurs between bounded batches and slices. +SQLite and filesystem write amplification mean the budget is not a hard +physical-device bandwidth cap. + When `[daemon].enabled` is true, `import` may opportunistically start a short one-pass ctx-owned maintenance profile after the foreground import finishes. The daemon work is local-only: bounded native provider-history refresh plus diff --git a/docs/contracts/json.md b/docs/contracts/json.md index 4872c298..195d04a9 100644 --- a/docs/contracts/json.md +++ b/docs/contracts/json.md @@ -20,24 +20,46 @@ Writes local storage and returns: - `data_root`; - `database_path`; - `config_path`; -- `mode`, either `ready` or `catalog_only`; +- `mode`, one of `ready`, `partial`, `background`, or `catalog_only`; - `indexed_items`; - `sources`; - `inventory`; - `catalog`; - `catalog_sources`; - `import`; +- `background_indexing`; - `network_required: false`; - `repo_writes: false`. -`import.ran` is true for the default setup path and false for -`ctx setup --catalog-only`. When it runs, `import.outcome`, +`mode` describes how setup left indexing work: + +- `catalog_only` means `--catalog-only` inventoried history without importing + it. `import.ran` is false with `import.reason: "catalog_only"`; +- `background` means setup selected its daemon-backed, inventory-only path; it + does not by itself assert that a backlog exists. `import.ran` is false with + `import.reason: "background"`; +- `partial` means the foreground import ran but its final import totals still + report fresh or recovery work pending; +- `ready` means the foreground import ran and reports no pending import work. + +When the foreground import runs, `import.outcome`, `import.failure_scope`, `import.failure_type`, `import.totals`, and `import.sources` use the same semantics and shapes as `ctx import --json`. Setup still uses top-level schema version 1; these nested fields are additive. An all-failed foreground setup import prints the complete JSON result and exits nonzero. Mixed success and source failures remain successful. +`background_indexing.enabled` is true when daemon-backed setup leaves native +inventory work pending or has supported semantic indexing enabled. The object +also includes `units`, `source_bytes`, `semantic_enabled`, +`semantic_supported`, nullable `lexical_estimate_seconds`, +`semantic_estimate_seconds`, `semantic_estimate_backend`, and +`semantic_cpu_fallback_estimate_seconds`, plus stable follow-up command strings +in `status_command`, `watch_command`, and `wait_command`. JSON setup never +starts the daemon itself: when background indexing is enabled, +`daemon_autostart.status` is `skipped` with reason `json_output`; otherwise it +is `not_needed` with reason `not_requested`. + `inventory` reports the shared local-history inventory across all native sources. It includes `sources`, `units`, `source_files`, `source_bytes`, `source_import_files`, `indexed_source_import_files`, @@ -238,12 +260,33 @@ corruption, locks, unreadable source files, and malformed source data are source failures and do not stop independent sources. Failed sources can include up to five rejection details when every content record was rejected. -`totals` and each source row include file, byte, session, event, edge, skipped, -and `rejected_records` counts. Rejection details are exposed as `rejections` -(bounded to five entries); `failed_sources` remains the count of source-level -failures. `sources_completed_with_rejections` counts sources that committed -accepted content while rejecting other records. `resume_mode` is currently `idempotent_rescan` when -`--resume` is passed and `normal_scan` otherwise. +`totals` includes these work-progress fields: + +- `durable_progress`, true when the invocation committed import or maintenance + progress, including durable progress that did not complete a whole unit; +- `fresh_units_processed`, the exact number of new, changed, or appended units + completed by this invocation; +- `recovery_units_processed`, the exact number of retry, replacement, + parser-revision, missing-material, abandoned-publication, legacy, or explicit + rescan units completed by this invocation; +- `fresh_units_pending` and `recovery_units_pending`, bounded end-of-run backlog + counts; +- `fresh_units_pending_exact` and `recovery_units_pending_exact`, which state + whether the corresponding pending count is exact. + +Pending counts are capped at 256. An `*_pending_exact` value of true means the +reported count is exact. False means the reported value is 256 and the actual +backlog is at least 256; consumers must not interpret it as exactly 256. +Processed counts are not capped and describe only work completed during the +current invocation. + +`totals` and each source row also include file, byte, session, event, edge, +skipped, and `rejected_records` counts. Rejection details are exposed as +`rejections` (bounded to five entries); `failed_sources` remains the count of +source-level failures. `sources_completed_with_rejections` counts sources that +committed accepted content while rejecting other records. `resume_mode` is +currently `idempotent_rescan` when `--resume` is passed and `normal_scan` +otherwise. Non-JSON native imports that target discovered/default provider sources may opportunistically start the ctx-owned background daemon maintenance profile diff --git a/docs/provider-import-policy.md b/docs/provider-import-policy.md index 804ec111..07c669d7 100644 --- a/docs/provider-import-policy.md +++ b/docs/provider-import-policy.md @@ -60,6 +60,29 @@ Content transactions use the shared 64-unit/8 MiB bounds with required WAL checkpoints. Event-search merge suppression may span a whole source, including manifested per-file imports, but its final compaction remains bounded. +Import execution also applies an internal cross-platform token-bucket I/O +pacer; it is policy, not a product flag. Exhaustive foreground setup/import and +wait refresh charge source reads and logical write work at 64 MiB/s with an 8 +MiB burst, bounded interactive refresh uses 32 MiB/s with a 4 MiB burst, and +daemon refresh uses 8 MiB/s with a 1 MiB burst. These are logical work budgets, +not an operating-system guarantee on physical disk bandwidth: SQLite journaling, +index maintenance, filesystem caching, and device write amplification can make +observed device traffic differ. FTS maintenance precharges its logical merge and +checkpoint budget, then supplements that charge to cover both the already-written +SQLite WAL and its imminent checkpoint copy. The nominal precharge is subtracted +from this observed two-pass target, so an amplified merge delays that checkpoint +and the next step proportionally without charging the nominal work twice. Bounded +transactions, checkpoints, and FTS merge steps constrain individual maintenance +operations around that portable budget. An ordinary writable store open +reasserts merge suppression but leaves stale FTS recovery marked; a paced import +or daemon maintenance path resumes it in bounded slices. + +The daemon retains its bucket across refresh passes, and idle credit never +exceeds one burst. Sleeps are issued in at most 25 ms quanta. A first unit may +exceed the 8 MiB transaction/slice target, but it is committed and then charged +by its actual durable or observed byte count, preserving progress and retry +fairness instead of rejecting the unit. + ## Import Unit Observation Inventory tracks the logical unit an adapter reads, not merely the file passed @@ -99,6 +122,11 @@ super-journal still exists. A child journal whose super-journal has been deleted is non-hot post-commit residue under SQLite's recovery rules; relative native super-journal names are resolved beside the child journal. Observation retries a bounded number of times and never modifies provider-owned files. +WAL checksum validation charges each header/page read to the active pacer. +Snapshot copying uses bounded chunks and charges both source reads and temporary +destination writes, so a large stable generation progresses without an +unaccounted copy burst. Normal streamed imports likewise pause only between +bounded durable transaction batches and scheduler slices, not per byte. Providers whose adapters normalize a whole document or root may remain whole-source replacement units. Native one-shot discovery still scans the diff --git a/scripts/check-loc-exceptions.tsv b/scripts/check-loc-exceptions.tsv index 07fbbba7..b5a13ca1 100644 --- a/scripts/check-loc-exceptions.tsv +++ b/scripts/check-loc-exceptions.tsv @@ -2,18 +2,18 @@ path max_lines kind reason review_after crates/ctx-history-capture/src/tests/native_json.rs 2820 test native JSON import regression matrix with shared fixtures; split after provider fixture consolidation lands 2026-10-01 crates/ctx-history-capture/src/tests/native_providers.rs 2435 test native provider parser matrix with cross-provider fixture setup; split after provider fixture consolidation lands 2026-10-01 crates/ctx-cli/src/integrations/mcp.rs 1623 source MCP config writer matrix spans many agent formats; split after the initial integrations installer surface stabilizes 2026-10-01 -crates/ctx-history-capture/src/tests/support.rs 2091 test shared capture test fixture support plus hermetic fixture roots; splitting now would add indirection without narrowing callers 2026-10-01 +crates/ctx-history-capture/src/tests/support.rs 2087 test shared capture test fixture support plus hermetic fixture roots; splitting now would add indirection without narrowing callers 2026-10-01 crates/ctx-history-capture/src/provider/providers/opencode.rs 1446 source OpenCode/Kilo provider import compatibility spans legacy and message-part transcript formats; split after provider parser boundaries settle 2026-10-01 crates/ctx-history-capture/src/tests/codex.rs 2240 test Codex parser regression matrix covers oversized JSONL and session semantics; split after fixture helpers are shared 2026-10-01 crates/ctx-history-capture/src/tests/native_sqlite.rs 1754 test native SQLite provider regression matrix shares setup across formats; split after fixture helpers are shared 2026-10-01 crates/ctx-history-capture/src/provider/native.rs 1272 source native provider discovery dispatcher remains centralized until provider registry extraction lands 2026-10-01 crates/ctx-history-capture/src/provider/importer.rs 1118 source native provider import orchestration remains centralized until importer phase extraction lands 2026-10-01 crates/ctx-cli/tests/native_providers.rs 1897 test native provider public CLI scenario matrix; fixtures are already modular and splitting would scatter end-to-end coverage 2026-10-01 -crates/ctx-history-store/src/schema/migrations.rs 1200 source schema migration history is append-only; split only after migration module boundaries are consolidated 2026-10-01 -crates/ctx-history-capture/src/provider/codex/session.rs 1096 source Codex session parser recently grew oversized-record semantics; keep bounded while follow-up parser split is scoped 2026-10-01 +crates/ctx-history-store/src/schema/migrations.rs 1479 source schema migration history is append-only; split only after migration module boundaries are consolidated 2026-10-01 +crates/ctx-history-capture/src/provider/codex/session.rs 1113 source Codex session parser recently grew oversized-record semantics; keep bounded while follow-up parser split is scoped 2026-10-01 crates/ctx-history-store/src/search/projections.rs 2108 source search projection eligibility and embedding document helpers are temporarily over cap after semantic projection integration; split with next search-store cleanup 2026-10-01 crates/ctx-cli/src/main.rs 1112 source public CLI argument and dispatch surface is temporarily over cap after daemon/status/semantic flags; split command args with next CLI modularization 2026-10-01 -crates/ctx-cli/src/semantic/daemon.rs 2315 source daemon lifecycle, query service, and semantic scheduler are integrated for prerelease; split after daemon service boundaries settle 2026-10-01 +crates/ctx-cli/src/semantic/daemon.rs 2371 source daemon lifecycle, query service, and semantic scheduler are integrated for prerelease; split after daemon service boundaries settle 2026-10-01 crates/ctx-cli/src/semantic/embedding_backend.rs 1361 source unified ONNX and Core ML embedding execution remains together while the cross-platform backend contract settles 2026-10-01 crates/ctx-cli/src/semantic/model_acquisition.rs 1424 source model download, cache validation, and atomic installation remain together while distribution paths settle 2026-10-01 crates/ctx-cli/src/semantic/model_bundle.rs 1445 source model bundle manifests, validation, cache markers, and atomic publication remain together while the bundle format settles 2026-10-01