diff --git a/.gitignore b/.gitignore index 7373a8d4..c9b83c34 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ /Cargo.lock /target **/target -**/data **/results /.idea diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..0714f108 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,82 @@ +# CLAUDE.md + +Guidance for Claude Code when working in the hudi-rs repo. + +## Toolchain + +- **Rust:** `1.88` (edition 2024), pinned in `rust-toolchain.toml` and `Cargo.toml` (`workspace.package.rust-version`). Rustup auto-installs this when you run cargo in the repo. +- **Python:** managed via [uv](https://github.com/astral-sh/uv). See `python/pyproject.toml` for the supported version range. +- **Maturin:** used to build the Python bindings; version is pinned in `python/pyproject.toml` (`requires = ...`). + +## Workspace layout + +Cargo workspace (`resolver = "2"`) with these members: + +- `crates/core` — core Hudi logic (`hudi-core`) +- `crates/datafusion` — DataFusion integration +- `crates/hudi` — umbrella crate published as `hudi` on crates.io +- `crates/test` — shared test utilities +- `cpp` — C++ bindings via cxx +- `python` — PyO3/Maturin bindings, published as `hudi` on PyPI +- `benchmark/tpch` — TPC-H benchmark harness + +## Build & test (Rust) + +Prefer the Makefile targets; they wrap the canonical commands. + +```shell +# All Rust tests (workspace, all targets, all features) +make test-rust +# equivalent: cargo test --no-fail-fast --all-targets --all-features --workspace + +# Single crate +cargo test -p hudi-core + +# Single test +cargo test -p hudi-core table::tests::hudi_table_get_schema + +# Format / lint (matches CI) +make format-rust # cargo fmt --all +make check-rust # cargo clippy --all-targets --all-features --workspace --no-deps -- -D warnings + # + cargo fmt --all -- --check +``` + +Note: Rust commands are invoked through `./build-wrapper.sh` in the Makefile. Running `cargo` directly works fine for local dev. + +## Build & test (Python) + +```shell +make setup-venv # uv venv .venv +source .venv/bin/activate +make develop # maturin develop --extras=devel,datafusion --features datafusion,testing +make test-python # uv run pytest -s python +pytest python/tests/test_table_read.py -s -k "test_read_table_has_correct_schema" +make check-python # ruff format --check, ruff check, mypy +``` + +## Before opening a PR + +```shell +make format check test +``` + +This runs format + check + test for both Rust and Python. Same checks CI runs. + +## Coverage + +```shell +make coverage-rust # HTML report at ./cov-reports/tarpaulin-report.html +make coverage-check # fails if below COV_THRESHOLD (default 60) +``` +Requires `cargo install cargo-tarpaulin`. + +## Benchmarks + +TPC-H harness lives in `benchmark/tpch`. See `make tpch-generate`, `make tpch-create-tables`, `make bench-tpch`. + +## PR conventions (from CONTRIBUTING.md) + +- Title must follow [conventional commits](https://www.conventionalcommits.org). +- Keep diffs under ~1000 lines when feasible. +- New features / bug fixes require tests. +- Prefer self-explanatory code over comments; only comment non-obvious logic. diff --git a/cpp/Cargo.toml b/cpp/Cargo.toml index 00af632e..353455c5 100644 --- a/cpp/Cargo.toml +++ b/cpp/Cargo.toml @@ -32,12 +32,27 @@ name = "hudi" crate-type = ["cdylib", "rlib"] [dependencies] -hudi = { path = "../crates/hudi", version = "0.5.0-dev" } +hudi-dep = { package = "hudi", path = "../crates/hudi", version = "0.5.0-dev" } hudi-test = { path = "../crates/test", version = "0.5.0-dev" } cxx = { version = "1.0" } +apache-avro = { workspace = true } arrow = { workspace = true , features = ["ffi"]} arrow-array = { workspace = true , features = ["ffi"]} +arrow-schema = { workspace = true } tokio = { workspace = true } +env_logger = { version = "0.11" } +log = { version = "0.4" } + +[dev-dependencies] +arrow = { workspace = true } +arrow-array = { workspace = true } +arrow-cast = { workspace = true } +arrow-ord = { workspace = true } +arrow-schema = { workspace = true } +arrow-select = { workspace = true } +hudi-test = { path = "../crates/test", version = "0.5.0-dev" } +parquet = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [build-dependencies] cxx-build = { version = "1.0" } diff --git a/cpp/src/context.rs b/cpp/src/context.rs new file mode 100644 index 00000000..c5704dbf --- /dev/null +++ b/cpp/src/context.rs @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Rich Rust types that mirror the Substrait `HudiReadOptions` proto hierarchy +//! 1-to-1. The FFI layer ([`crate::ffi::FfiReaderContext`]) is a flat CXX +//! struct; this module reconstructs the proper nesting using [`Option`], +//! [`Vec`], [`HashMap`], and [`Box`]. +//! +//! ## Proto → Rust mapping +//! +//! | Proto message | Rust struct | +//! |---------------------------------------------------|-------------------------------| +//! | `HudiReadOptions.HoodieFileGroupReaderContext` | [`FileGroupReaderContext`] | +//! | `HudiReadOptions.BaseFile` | [`BaseFile`] | +//! | `HudiReadOptions.LogFile` | [`LogFile`] | +//! | `HudiReadOptions.HoodieSchema` | [`HoodieSchema`] | +//! | `HudiReadOptions.InstantRange` | [`InstantRange`] | +//! | `HudiReadOptions.ReaderContext` | [`ReaderContext`] | + +use crate::ffi::FfiReaderContext; +use std::collections::HashMap; + +// ── Proto message: BaseFile ───────────────────────────────────────────────── + +/// Mirrors `HudiReadOptions.BaseFile` proto. +/// +/// ```proto +/// message BaseFile { +/// string path = 1; +/// string file_name = 2; +/// int64 file_size = 3; +/// string file_id = 4; +/// string commit_time = 5; +/// BaseFile bootstrap_base_file = 6; // recursive +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct BaseFile { + pub path: String, + pub file_name: String, + pub file_size: i64, + pub file_id: String, + pub commit_time: String, + pub bootstrap_base_file: Option>, +} + +// ── Proto message: LogFile ────────────────────────────────────────────────── + +/// Mirrors `HudiReadOptions.LogFile` proto. +/// +/// ```proto +/// message LogFile { +/// string path_str = 1; +/// string file_id = 2; +/// string delta_commit_time = 3; +/// int32 log_version = 4; +/// string log_write_token = 5; +/// string file_extension = 6; +/// string suffix = 7; +/// int64 file_size = 8; +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct LogFile { + pub path_str: String, + pub file_id: String, + pub delta_commit_time: String, + pub log_version: i32, + pub log_write_token: String, + pub file_extension: String, + pub suffix: String, + pub file_size: i64, +} + +// ── Proto message: HoodieSchema ───────────────────────────────────────────── + +/// Mirrors `HudiReadOptions.HoodieSchema` proto. +/// +/// ```proto +/// message HoodieSchema { +/// string avro_schema_json = 1; +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct HoodieSchema { + pub avro_schema_json: String, +} + +// ── Proto message: InstantRange ───────────────────────────────────────────── + +/// Mirrors `HudiReadOptions.InstantRange` proto. +/// +/// ```proto +/// message InstantRange { +/// string start_instant = 1; +/// string end_instant = 2; +/// string range_type = 3; +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct InstantRange { + pub start_instant: String, + pub end_instant: String, + pub range_type: String, +} + +// ── Proto message: ReaderContext ───────────────────────────────────────────── + +/// Mirrors `HudiReadOptions.ReaderContext` proto (inner hudi-rs reader context). +/// +/// ```proto +/// message ReaderContext { +/// string table_path = 1; +/// string latest_commit_time = 2; +/// string base_file_format = 3; +/// bool has_log_files = 4; +/// bool has_bootstrap_base_file = 5; +/// bool needs_bootstrap_merge = 6; +/// bool should_merge_use_record_position = 7; +/// bool enable_logical_timestamp_field_repair = 8; +/// string iterator_mode = 9; +/// string merge_mode = 10; +/// string merge_strategy_id = 11; +/// InstantRange instant_range = 12; +/// map table_config = 13; +/// map hoodie_reader_config = 14; +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct ReaderContext { + pub table_path: String, + pub latest_commit_time: String, + pub base_file_format: String, + pub has_log_files: bool, + pub has_bootstrap_base_file: bool, + pub needs_bootstrap_merge: bool, + pub should_merge_use_record_position: bool, + pub enable_logical_timestamp_field_repair: bool, + pub iterator_mode: String, + pub merge_mode: String, + pub merge_strategy_id: String, + pub instant_range: Option, + pub table_config: HashMap, + pub hoodie_reader_config: HashMap, +} + +// ── Proto message: HoodieFileGroupReaderContext ─────────────────────────────── + +/// Mirrors `HudiReadOptions.HoodieFileGroupReaderContext` proto (outer context). +/// +/// ```proto +/// message HoodieFileGroupReaderContext { +/// string table_path = 1; +/// string partition_path = 2; +/// string latest_commit_time = 3; +/// int64 start = 4; +/// int64 length = 5; +/// bool should_use_record_position = 6; +/// bool allow_inflight_instants = 7; +/// bool emit_delete = 8; +/// bool sort_output = 9; +/// map props = 10; +/// BaseFile base_file = 11; +/// repeated LogFile log_files = 12; +/// HoodieSchema data_schema = 13; +/// HoodieSchema requested_schema = 14; +/// ReaderContext reader_context = 15; +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct FileGroupReaderContext { + // ── outer primitives ──────────────────────────────────────────────── + pub table_path: String, + pub partition_path: String, + pub latest_commit_time: String, + pub start: i64, + pub length: i64, + pub should_use_record_position: bool, + pub allow_inflight_instants: bool, + pub emit_delete: bool, + pub sort_output: bool, + + // ── props (TypedProperties → Map) ─────────────────── + pub props: HashMap, + + // ── base_file ──────────────────────────────────────────────────────── + pub base_file: Option, + + // ── log_files ──────────────────────────────────────────────────────── + pub log_files: Vec, + + // ── schemas ────────────────────────────────────────────────────────── + pub data_schema: Option, + pub requested_schema: Option, + + // ── reader_context (inner hudi-rs ReaderContext) ───────────────────── + pub reader_context: ReaderContext, +} + +// ── Conversion: FfiReaderContext → FileGroupReaderContext ──────────────────── + +/// Zips parallel key/value vectors into a [`HashMap`]. +fn zip_kv(keys: Vec, values: Vec) -> HashMap { + keys.into_iter().zip(values).collect() +} + +impl From for FileGroupReaderContext { + fn from(ffi: FfiReaderContext) -> Self { + // ── inner ReaderContext (proto field 15) ───────────────────────── + let reader_context = ReaderContext { + table_path: ffi.table_path.clone(), + latest_commit_time: ffi.latest_commit_time.clone(), + base_file_format: ffi.base_file_format, + has_log_files: ffi.has_log_files, + has_bootstrap_base_file: ffi.has_bootstrap_base_file, + needs_bootstrap_merge: ffi.needs_bootstrap_merge, + should_merge_use_record_position: ffi.should_merge_use_record_position, + enable_logical_timestamp_field_repair: ffi.enable_logical_timestamp_field_repair, + iterator_mode: ffi.iterator_mode, + merge_mode: ffi.merge_mode, + merge_strategy_id: ffi.merge_strategy_id, + instant_range: if ffi.has_instant_range { + Some(InstantRange { + start_instant: ffi.instant_range_start, + end_instant: ffi.instant_range_end, + range_type: ffi.instant_range_type, + }) + } else { + None + }, + table_config: zip_kv(ffi.table_config_keys, ffi.table_config_values), + hoodie_reader_config: zip_kv( + ffi.hoodie_reader_config_keys, + ffi.hoodie_reader_config_values, + ), + }; + + // ── base_file (proto field 11) ────────────────────────────────── + let base_file = if ffi.has_base_file { + let bootstrap = if ffi.base_file_has_bootstrap { + Some(Box::new(BaseFile { + path: ffi.base_file_bootstrap_path, + file_name: ffi.base_file_bootstrap_file_name, + file_size: ffi.base_file_bootstrap_file_size, + file_id: ffi.base_file_bootstrap_file_id, + commit_time: ffi.base_file_bootstrap_commit_time, + bootstrap_base_file: None, // proto recursion stops at depth 1 + })) + } else { + None + }; + Some(BaseFile { + path: ffi.base_file_path, + file_name: ffi.base_file_file_name, + file_size: ffi.base_file_file_size, + file_id: ffi.base_file_file_id, + commit_time: ffi.base_file_commit_time, + bootstrap_base_file: bootstrap, + }) + } else { + None + }; + + // ── log_files (proto field 12) ────────────────────────────────── + let log_files: Vec = ffi + .log_file_details + .into_iter() + .map(|lf| LogFile { + path_str: lf.path_str, + file_id: lf.file_id, + delta_commit_time: lf.delta_commit_time, + log_version: lf.log_version, + log_write_token: lf.log_write_token, + file_extension: lf.file_extension, + suffix: lf.suffix, + file_size: lf.file_size, + }) + .collect(); + + // ── schemas (proto fields 13–14) ──────────────────────────────── + let data_schema = if ffi.data_schema_json.is_empty() { + None + } else { + Some(HoodieSchema { + avro_schema_json: ffi.data_schema_json, + }) + }; + let requested_schema = if ffi.requested_schema_json.is_empty() { + None + } else { + Some(HoodieSchema { + avro_schema_json: ffi.requested_schema_json, + }) + }; + + FileGroupReaderContext { + table_path: ffi.table_path, + partition_path: ffi.partition_path.clone(), + latest_commit_time: ffi.latest_commit_time, + start: ffi.start, + length: ffi.length, + should_use_record_position: ffi.should_use_record_position, + allow_inflight_instants: ffi.allow_inflight_instants, + emit_delete: ffi.emit_delete, + sort_output: ffi.sort_output, + props: zip_kv(ffi.props_keys, ffi.props_values), + base_file, + log_files, + data_schema, + requested_schema, + reader_context, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ffi::{FfiLogFile, FfiReaderContext}; + + #[test] + fn test_empty_context_converts() { + let ffi = FfiReaderContext::default(); + let ctx: FileGroupReaderContext = ffi.into(); + assert!(ctx.base_file.is_none()); + assert!(ctx.log_files.is_empty()); + assert!(ctx.data_schema.is_none()); + assert!(ctx.requested_schema.is_none()); + assert!(ctx.props.is_empty()); + assert!(ctx.reader_context.instant_range.is_none()); + assert!(ctx.reader_context.table_config.is_empty()); + assert!(ctx.reader_context.hoodie_reader_config.is_empty()); + } + + #[test] + fn test_full_context_converts() { + let ffi = FfiReaderContext { + table_path: "s3://bucket/table".into(), + latest_commit_time: "20240101120000000".into(), + base_file_format: "PARQUET".into(), + has_log_files: true, + has_bootstrap_base_file: false, + needs_bootstrap_merge: false, + should_merge_use_record_position: true, + enable_logical_timestamp_field_repair: false, + iterator_mode: "ENGINE_RECORD".into(), + merge_mode: "COMMIT_TIME_ORDERING".into(), + merge_strategy_id: "".into(), + has_instant_range: true, + instant_range_start: "20240101000000000".into(), + instant_range_end: "20240101120000000".into(), + instant_range_type: "CLOSED_CLOSED".into(), + table_config_keys: vec!["hoodie.table.name".into()], + table_config_values: vec!["test".into()], + hoodie_reader_config_keys: vec!["hoodie.read.key".into()], + hoodie_reader_config_values: vec!["val".into()], + props_keys: vec!["hoodie.props.key".into()], + props_values: vec!["propval".into()], + data_schema_json: r#"{"type":"record"}"#.into(), + requested_schema_json: r#"{"type":"record"}"#.into(), + partition_path: "year=2024".into(), + base_file_name: "abc.parquet".into(), + log_file_names: vec![".abc.log.1".into()], + start: 0, + length: 1024, + should_use_record_position: true, + allow_inflight_instants: false, + emit_delete: false, + sort_output: false, + has_base_file: true, + base_file_path: "s3://bucket/table/year=2024/abc.parquet".into(), + base_file_file_name: "abc.parquet".into(), + base_file_file_size: 1024, + base_file_file_id: "abc".into(), + base_file_commit_time: "20240101120000000".into(), + base_file_has_bootstrap: false, + base_file_bootstrap_path: "".into(), + base_file_bootstrap_file_name: "".into(), + base_file_bootstrap_file_size: 0, + base_file_bootstrap_file_id: "".into(), + base_file_bootstrap_commit_time: "".into(), + log_file_details: vec![FfiLogFile { + path_str: ".abc.log.1".into(), + file_id: "abc".into(), + delta_commit_time: "20240101120000000".into(), + log_version: 1, + log_write_token: "1-0-1".into(), + file_extension: ".log".into(), + suffix: "".into(), + file_size: 512, + }], + }; + + let ctx: FileGroupReaderContext = ffi.into(); + assert_eq!(ctx.table_path, "s3://bucket/table"); + assert_eq!(ctx.partition_path, "year=2024"); + assert_eq!(ctx.latest_commit_time, "20240101120000000"); + assert_eq!(ctx.start, 0); + assert_eq!(ctx.length, 1024); + assert!(ctx.should_use_record_position); + + // props + assert_eq!(ctx.props.get("hoodie.props.key").unwrap(), "propval"); + + // BaseFile + let bf = ctx.base_file.as_ref().unwrap(); + assert_eq!(bf.file_id, "abc"); + assert!(bf.bootstrap_base_file.is_none()); + + // LogFile + assert_eq!(ctx.log_files.len(), 1); + assert_eq!(ctx.log_files[0].log_version, 1); + + // Schemas + assert!(ctx.data_schema.is_some()); + assert!(ctx.requested_schema.is_some()); + + // ReaderContext (inner) + assert_eq!(ctx.reader_context.table_path, "s3://bucket/table"); + assert!(ctx.reader_context.has_log_files); + let ir = ctx.reader_context.instant_range.unwrap(); + assert_eq!(ir.range_type, "CLOSED_CLOSED"); + assert_eq!( + ctx.reader_context + .table_config + .get("hoodie.table.name") + .unwrap(), + "test" + ); + } +} diff --git a/cpp/src/lib.rs b/cpp/src/lib.rs index eddf6776..3d670305 100644 --- a/cpp/src/lib.rs +++ b/cpp/src/lib.rs @@ -16,16 +16,129 @@ * specific language governing permissions and limitations * under the License. */ +pub mod context; mod util; -use crate::util::create_raw_pointer_for_record_batches; -use cxx::{CxxString, CxxVector}; -use hudi::file_group::FileGroup; -use hudi::file_group::file_slice::FileSlice; -use hudi::file_group::reader::FileGroupReader; +/// Re-export core types for integration tests and downstream consumers. +pub use hudi_dep as hudi_core; + +use crate::context::FileGroupReaderContext; +use crate::util::{create_raw_pointer_for_record_batches, free_arrow_stream}; +use hudi_dep::avro_to_arrow::to_arrow_schema; +use hudi_dep::config::HudiConfigs; +use hudi_dep::config::table::HudiTableConfig; +use hudi_dep::config::util::split_hudi_options_from_others; +use hudi_dep::file_group::reader::HoodieFileGroupReader as CoreFileGroupReader; +use hudi_dep::file_group::reader::input_split::InputSplit; +use hudi_dep::file_group::reader::reader_context::ReaderContext; +use hudi_dep::file_group::reader::reader_parameters::ReaderParameters; +use hudi_dep::file_group::reader::record_context::RecordContext; +use hudi_dep::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; +use hudi_dep::storage::Storage; +use hudi_dep::timeline::selector::InstantRange; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::OnceLock; + +static LOGGER: OnceLock<()> = OnceLock::new(); + +/// Initialize env_logger exactly once for the lifetime of the loaded shared library. +fn init_logger() { + LOGGER.get_or_init(|| { + match std::env::var("RUST_LOG") { + Ok(_) => {} + Err(_) => unsafe { std::env::set_var("RUST_LOG", "debug") }, + } + let _ = env_logger::try_init(); + }); +} #[cxx::bridge] mod ffi { + /// Mirrors `HudiReadOptions.LogFile` proto (CXX-safe, used in `Vec`). + #[derive(Default)] + struct FfiLogFile { + path_str: String, + file_id: String, + delta_commit_time: String, + log_version: i32, + log_write_token: String, + file_extension: String, + suffix: String, + file_size: i64, + } + + /// FFI-safe flat representation of `HudiReadOptions.HoodieFileGroupReaderContext`. + /// + /// Contains everything the file group reader needs: + /// - Table path (used as `hoodie.base.path` for Storage) + /// - Partition, commit time, reader flags + /// - Config maps (table_config, props, hoodie_reader_config) + /// - Base file / log file details + /// - Schemas, merge mode, instant range + /// - Split-level transport fields (base_file_name, log_file_names) + #[derive(Default)] + struct FfiReaderContext { + // ── outer primitives (HoodieFileGroupReaderContext fields 1–9) ── + table_path: String, + partition_path: String, + latest_commit_time: String, + start: i64, + length: i64, + should_use_record_position: bool, + allow_inflight_instants: bool, + emit_delete: bool, + sort_output: bool, + + // ── props (proto field 10) ─────────────────────────────────────── + props_keys: Vec, + props_values: Vec, + + // ── BaseFile (proto field 11, flattened) ───────────────────────── + has_base_file: bool, + base_file_path: String, + base_file_file_name: String, + base_file_file_size: i64, + base_file_file_id: String, + base_file_commit_time: String, + base_file_has_bootstrap: bool, + base_file_bootstrap_path: String, + base_file_bootstrap_file_name: String, + base_file_bootstrap_file_size: i64, + base_file_bootstrap_file_id: String, + base_file_bootstrap_commit_time: String, + + // ── LogFile[] (proto field 12) ──────────────────────────────────── + log_file_details: Vec, + + // ── HoodieSchema (proto fields 13–14, inlined) ─────────────────── + data_schema_json: String, + requested_schema_json: String, + + // ── ReaderContext (proto field 15, flattened) ───────────────────── + base_file_format: String, + has_log_files: bool, + has_bootstrap_base_file: bool, + needs_bootstrap_merge: bool, + should_merge_use_record_position: bool, + enable_logical_timestamp_field_repair: bool, + iterator_mode: String, + merge_mode: String, + merge_strategy_id: String, + has_instant_range: bool, + instant_range_start: String, + instant_range_end: String, + instant_range_type: String, + table_config_keys: Vec, + table_config_values: Vec, + hoodie_reader_config_keys: Vec, + hoodie_reader_config_values: Vec, + + // ── file-slice split fields (not in proto — transport only) ────── + base_file_name: String, + log_file_names: Vec, + } + unsafe extern "C++" { include!("arrow/c/abi.h"); @@ -33,95 +146,339 @@ mod ffi { } extern "Rust" { - type HudiFileGroupReader; - fn new_file_group_reader_with_options( - base_uri: &CxxString, - options: &CxxVector, - ) -> Result>; - - type HudiFileSlice; - fn new_file_slice_from_file_names( - partition_path: &CxxString, - base_file_name: &CxxString, - log_file_names: &CxxVector, - ) -> Result>; - - fn read_file_slice_by_base_file_path( - self: &HudiFileGroupReader, - relative_path: &CxxString, - ) -> Result<*mut ArrowArrayStream>; - - fn read_file_slice( - self: &HudiFileGroupReader, - file_slice: &HudiFileSlice, - ) -> Result<*mut ArrowArrayStream>; + type HoodieFileGroupReader; + + /// Create a file group reader from the full context. + /// `table_path` inside `ctx` is used as the storage base URI. + fn new_file_group_reader_with_context( + ctx: FfiReaderContext, + ) -> Result>; + + /// Read the file group and return merged results as an ArrowArrayStream. + fn get_closable_iterator(self: &HoodieFileGroupReader) -> Result<*mut ArrowArrayStream>; + + /// Free an ArrowArrayStream that was returned by `get_closable_iterator`. + /// + /// This drops the stream (invoking the Arrow release callback to free + /// internal buffers) and deallocates the struct itself through Rust's + /// allocator. The caller must not use `ptr` after this call. + unsafe fn hudi_free_arrow_stream(ptr: *mut ArrowArrayStream); } } -pub struct HudiFileGroupReader { - inner: FileGroupReader, +pub struct HoodieFileGroupReader { + // ── 1:1 with Java HoodieFileGroupReader fields ───────────────── + reader_context: Arc, + storage: Arc, + props: HashMap, + reader_parameters: ReaderParameters, + input_split: InputSplit, + partition_path_fields: Option>, + + // ── Rust-only ────────────────────────────────────────────────── rt: tokio::runtime::Runtime, } -pub fn new_file_group_reader_with_options( - base_uri: &CxxString, - options: &CxxVector, -) -> std::result::Result, String> { - let base_uri = base_uri - .to_str() - .map_err(|e| format!("Failed to convert CxxString to str: {e}"))?; - - let mut opt_vec = Vec::new(); - for opt in options.iter() { - let opt_str = opt - .to_str() - .map_err(|e| format!("Failed to convert CxxString to str: {e}"))?; - if let Some((key, value)) = opt_str.split_once('=') { - opt_vec.push((key, value)) - } +/// Creates a `HoodieFileGroupReader` from a full `FfiReaderContext`. +pub fn new_file_group_reader_with_context( + ctx: ffi::FfiReaderContext, +) -> std::result::Result, String> { + init_logger(); + + // Capture transport-only file names before ctx is consumed by .into(). + let base_file_name = ctx.base_file_name.clone(); + let log_file_names: Vec = ctx.log_file_names.iter().cloned().collect(); + + // Convert flat FFI struct → nested Rust types (intermediate, not stored). + let fgrc: FileGroupReaderContext = ctx.into(); + + log::debug!( + "new_file_group_reader_with_context: \ + table_path={table_path} partition_path={partition_path} \ + latest_commit_time={latest_commit_time} start={start} length={length} \ + should_use_record_position={surp} allow_inflight_instants={aii} \ + emit_delete={ed} sort_output={so} \ + props_count={props_count} base_file={base_file} log_files_count={log_files_count} \ + has_data_schema={has_data_schema} has_requested_schema={has_req_schema}", + table_path = fgrc.table_path, + partition_path = fgrc.partition_path, + latest_commit_time = fgrc.latest_commit_time, + start = fgrc.start, + length = fgrc.length, + surp = fgrc.should_use_record_position, + aii = fgrc.allow_inflight_instants, + ed = fgrc.emit_delete, + so = fgrc.sort_output, + props_count = fgrc.props.len(), + base_file = fgrc + .base_file + .as_ref() + .map(|bf| bf.file_name.as_str()) + .unwrap_or(""), + log_files_count = fgrc.log_files.len(), + has_data_schema = fgrc.data_schema.is_some(), + has_req_schema = fgrc.requested_schema.is_some(), + ); + { + let rc = &fgrc.reader_context; + log::debug!( + "new_file_group_reader_with_context: reader_context \ + table_path={table_path} latest_commit_time={latest_commit_time} \ + base_file_format={base_file_format} has_log_files={has_log_files} \ + needs_bootstrap_merge={nbm} should_merge_use_record_position={smurp} \ + iterator_mode={iterator_mode} merge_mode={merge_mode} \ + merge_strategy_id={merge_strategy_id} has_instant_range={has_instant_range} \ + table_config_count={table_config_count} hoodie_reader_config_count={hrc_count}", + table_path = rc.table_path, + latest_commit_time = rc.latest_commit_time, + base_file_format = rc.base_file_format, + has_log_files = rc.has_log_files, + nbm = rc.needs_bootstrap_merge, + smurp = rc.should_merge_use_record_position, + iterator_mode = rc.iterator_mode, + merge_mode = rc.merge_mode, + merge_strategy_id = rc.merge_strategy_id, + has_instant_range = rc.instant_range.is_some(), + table_config_count = rc.table_config.len(), + hrc_count = rc.hoodie_reader_config.len(), + ); + } + log::debug!( + "new_file_group_reader_with_context: split base_file_name={base_file_name} \ + log_file_names={log_file_names:?}", + ); + + // ── 1. Build merged props ─────────────────────────────────────── + // Order: hoodie.base.path + table_config < props < hoodie_reader_config + let mut options: Vec<(String, String)> = Vec::new(); + options.push(( + HudiTableConfig::BasePath.as_ref().to_string(), + fgrc.table_path.clone(), + )); + for (k, v) in &fgrc.reader_context.table_config { + options.push((k.clone(), v.clone())); + } + for (k, v) in &fgrc.props { + options.push((k.clone(), v.clone())); } + for (k, v) in &fgrc.reader_context.hoodie_reader_config { + options.push((k.clone(), v.clone())); + } + + // ── 2. Create Storage (needs temporary HudiConfigs) ───────────── + let (hudi_opts, storage_opts) = split_hudi_options_from_others(options); + let props: HashMap = hudi_opts; + let hudi_configs = Arc::new(HudiConfigs::new(props.clone())); + let storage = Storage::new(Arc::new(storage_opts), hudi_configs) + .map_err(|e| format!("Failed to create Storage: {e}"))?; + + // ── 3. Build ReaderParameters ─────────────────────────────────── + let reader_parameters = ReaderParameters { + use_record_position: fgrc.should_use_record_position, + emit_delete: fgrc.emit_delete, + sort_output: fgrc.sort_output, + allow_inflight_instants: fgrc.allow_inflight_instants, + }; + // ── 4. Build InputSplit ───────────────────────────────────────── + let base_file_path = if base_file_name.is_empty() { + None + } else if fgrc.partition_path.is_empty() { + Some(base_file_name) + } else { + Some(format!("{}/{}", fgrc.partition_path, base_file_name)) + }; + let log_file_paths: Vec = log_file_names + .into_iter() + .map(|name| { + if fgrc.partition_path.is_empty() { + name + } else { + format!("{}/{}", fgrc.partition_path, name) + } + }) + .collect(); + let input_split = InputSplit::new(base_file_path, None, log_file_paths, fgrc.partition_path); + + // ── 5. Convert FFI ReaderContext → core ReaderContext ──────────── + let ffi_rc = fgrc.reader_context; + let instant_range = ffi_rc.instant_range.map(|ir| { + let timezone = ffi_rc + .table_config + .get(HudiTableConfig::TimelineTimezone.as_ref()) + .cloned() + .unwrap_or_else(|| "utc".to_string()); + let (start_inclusive, end_inclusive) = match ir.range_type.as_str() { + "CLOSED_CLOSED" => (true, true), + "OPEN_CLOSED" => (false, true), + "CLOSED_OPEN" => (true, false), + _ => (false, true), // default + }; + let start = if ir.start_instant.is_empty() { + None + } else { + Some(ir.start_instant) + }; + let end = if ir.end_instant.is_empty() { + None + } else { + Some(ir.end_instant) + }; + InstantRange::new(timezone, start, end, start_inclusive, end_inclusive) + }); + // RecordContext is constructed from table_config, matching Java's + // RecordContext(tableConfig, typeConverter) pattern. The table_config + // carries hoodie.populate.meta.fields, hoodie.table.precombine.field, + // and hoodie.table.recordkey.fields — RecordContext derives everything + // from these. + let partition_path = input_split.partition_path.clone(); + let record_context = RecordContext::new(&ffi_rc.table_config, partition_path); + + // ── Extract partition path fields from table config ───────────── + // Mirrors Java's: tableConfig.getPartitionFields() which reads + // "hoodie.table.partition.fields", splits on ",", and strips + // custom key-generator partition type suffixes (split on ":"). + // e.g. "date:TIMESTAMP,region:SIMPLE" → ["date", "region"] + let partition_path_fields: Option> = ffi_rc + .table_config + .get("hoodie.table.partition.fields") + .map(|v| { + v.split(',') + .map(|s| s.trim().split(':').next().unwrap_or("").to_string()) + .filter(|s| !s.is_empty()) + .collect() + }); + + // ── 6. Build schema handler from Avro schemas passed via FFI ── + // Set on ReaderContext to match Java's HoodieReaderContext.schemaHandler. + // + // The `data_schema` from the Scala planning layer is the pruned table + // data schema (full table columns minus "op" and partition columns). + // This serves as both `table_schema` (field lookup source for mandatory + // fields) and `data_schema` (base file reading schema). + let schema_handler = { + let mut handler = FileGroupReaderSchemaHandler::new(); + if let Some(hs) = fgrc.data_schema.as_ref() { + if let Ok(arrow_schema) = avro_json_to_arrow_schema(&hs.avro_schema_json) { + let schema_ref = Arc::new(arrow_schema); + handler = handler + .with_table_schema(schema_ref.clone()) + .with_data_schema(schema_ref); + } + } + if let Some(hs) = fgrc.requested_schema.as_ref() { + if let Ok(arrow_schema) = avro_json_to_arrow_schema(&hs.avro_schema_json) { + handler = handler.with_requested_schema(Arc::new(arrow_schema)); + } + } + handler + }; + + let core_reader_context = Arc::new(ReaderContext { + table_path: ffi_rc.table_path, + latest_commit_time: ffi_rc.latest_commit_time, + base_file_format: ffi_rc.base_file_format, + has_log_files: ffi_rc.has_log_files, + has_bootstrap_base_file: ffi_rc.has_bootstrap_base_file, + needs_bootstrap_merge: ffi_rc.needs_bootstrap_merge, + should_merge_use_record_position: ffi_rc.should_merge_use_record_position, + enable_logical_timestamp_field_repair: ffi_rc.enable_logical_timestamp_field_repair, + iterator_mode: ffi_rc.iterator_mode, + merge_mode: ffi_rc.merge_mode, + merge_strategy_id: ffi_rc.merge_strategy_id, + instant_range, + record_context, + schema_handler, + table_config: ffi_rc.table_config, + hoodie_reader_config: ffi_rc.hoodie_reader_config, + // Mirrors Java HoodieReaderContext.keyFilterOpt — FFI bridge always + // initializes to None (FFI integration is out of scope per spec §2). + key_filter_opt: None, + }); + + // ── 7. Build tokio runtime ────────────────────────────────────── let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|e| format!("Failed to create tokio runtime: {e}"))?; - let reader = rt - .block_on(FileGroupReader::new_with_options(base_uri, opt_vec)) - .map_err(|e| format!("Failed to create FileGroupReader: {e}"))?; - Ok(Box::new(HudiFileGroupReader { inner: reader, rt })) + + Ok(Box::new(HoodieFileGroupReader { + reader_context: core_reader_context.clone(), + storage, + props, + reader_parameters, + input_split, + partition_path_fields, + rt, + })) } -impl HudiFileGroupReader { - pub fn read_file_slice_by_base_file_path( +impl HoodieFileGroupReader { + /// Construct a `HoodieFileGroupReader` directly (for testing without FFI). + pub fn new( + reader_context: Arc, + storage: Arc, + props: HashMap, + reader_parameters: ReaderParameters, + input_split: InputSplit, + partition_path_fields: Option>, + ) -> std::result::Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("Failed to create tokio runtime: {e}"))?; + Ok(Self { + reader_context, + storage, + props, + reader_parameters, + input_split, + partition_path_fields, + rt, + }) + } + + /// Runs the full 3-phase merge and returns the resulting `RecordBatch` and its schema. + pub fn read_record_batch( &self, - relative_path: &CxxString, - ) -> std::result::Result<*mut ffi::ArrowArrayStream, String> { - let relative_path = relative_path - .to_str() - .map_err(|e| format!("Failed to convert CxxString to str: {e}"))?; + ) -> std::result::Result<(arrow_array::RecordBatch, arrow_schema::SchemaRef), String> { + log::debug!( + "read_record_batch: partition={} base_file={:?} log_files={} \ + latest_instant_time={} ordering_fields={:?} merge_mode={}", + self.input_split.partition_path, + self.input_split.base_file_path, + self.input_split.log_file_paths.len(), + self.reader_context.latest_commit_time, + self.reader_context.ordering_field_names(), + self.reader_context.merge_mode.as_str(), + ); + + let mut reader = CoreFileGroupReader::builder() + .with_reader_context(self.reader_context.clone()) + .with_storage(self.storage.clone()) + .with_input_split(self.input_split.clone()) + .with_reader_parameters(self.reader_parameters.clone()) + .build() + .map_err(|e| format!("Failed to build file group reader: {e}"))?; let record_batch = self .rt - .block_on(self.inner.read_file_slice_by_base_file_path(relative_path)) - .map_err(|e| format!("Failed to read file batch: {e}"))?; + .block_on(reader.read()) + .map_err(|e| format!("Failed to read file group: {e}"))?; let schema = record_batch.schema(); - Ok(create_raw_pointer_for_record_batches( - vec![record_batch], - schema, - )) - } + log::debug!( + "read_record_batch: merge complete, {} rows, {} cols", + record_batch.num_rows(), + record_batch.num_columns(), + ); - pub fn read_file_slice( - &self, - file_slice: &HudiFileSlice, - ) -> std::result::Result<*mut ffi::ArrowArrayStream, String> { - let record_batch = self - .rt - .block_on(self.inner.read_file_slice(&file_slice.inner)) - .map_err(|e| format!("Failed to read file slice: {e}"))?; - let schema = record_batch.schema(); + Ok((record_batch, schema)) + } + /// Uses the core `HoodieFileGroupReader` for the full 3-phase merge. + pub fn get_closable_iterator(&self) -> std::result::Result<*mut ffi::ArrowArrayStream, String> { + let (record_batch, schema) = self.read_record_batch()?; Ok(create_raw_pointer_for_record_batches( vec![record_batch], schema, @@ -129,43 +486,19 @@ impl HudiFileGroupReader { } } -pub struct HudiFileSlice { - inner: FileSlice, +/// Free an `ArrowArrayStream` returned by [`HoodieFileGroupReader::get_closable_iterator`]. +/// +/// # Safety +/// `ptr` must have been returned by `get_closable_iterator` and must not be +/// used after this call. +unsafe fn hudi_free_arrow_stream(ptr: *mut ffi::ArrowArrayStream) { + unsafe { free_arrow_stream(ptr) }; } -pub fn new_file_slice_from_file_names( - partition_path: &CxxString, - base_file_name: &CxxString, - log_file_names: &CxxVector, -) -> std::result::Result, String> { - let partition_path = partition_path - .to_str() - .map_err(|e| format!("Failed to convert CxxString to str: {e}"))?; - let base_file_name = base_file_name - .to_str() - .map_err(|e| format!("Failed to convert CxxString to str: {e}"))?; - - let log_file_names = log_file_names - .iter() - .map(|name| { - name.to_str() - .map_err(|e| format!("Failed to convert CxxString to str: {e}")) - }) - .collect::, _>>()?; - - let mut file_group = FileGroup::new_with_base_file_name(base_file_name, partition_path) - .map_err(|e| format!("Failed to create FileGroup: {e}"))?; - file_group - .add_log_files_from_names(&log_file_names) - .map_err(|e| format!("Failed to add files to FileGroup: {e}"))?; - - let (_, file_slice) = file_group - .file_slices - .iter() - .next() - .ok_or_else(|| format!("Failed to get file slice from FileGroup: {file_group:?}"))?; - - Ok(Box::new(HudiFileSlice { - inner: file_slice.clone(), - })) +/// Convert an Avro schema JSON string to an Arrow Schema. +fn avro_json_to_arrow_schema(avro_json: &str) -> std::result::Result { + let sanitized = avro_json.trim().replace("\\:", ":"); + let avro_schema = apache_avro::Schema::parse_str(&sanitized) + .map_err(|e| format!("Failed to parse Avro schema: {e}"))?; + to_arrow_schema(&avro_schema).map_err(|e| format!("Failed to convert Avro→Arrow: {e}")) } diff --git a/cpp/src/util.rs b/cpp/src/util.rs index 5a30cf3d..767a0e11 100644 --- a/cpp/src/util.rs +++ b/cpp/src/util.rs @@ -28,6 +28,26 @@ pub fn create_raw_pointer_for_record_batches( let batches = batches.into_iter().map(Ok); let batch_iterator = RecordBatchIterator::new(batches, schema); let ffi_array_stream = FFI_ArrowArrayStream::new(Box::new(batch_iterator)); + // Heap-allocate and leak into a raw pointer. free_arrow_stream() below + // is the only correct way to reclaim this allocation. let raw_ptr = Box::into_raw(Box::new(ffi_array_stream)); raw_ptr as *mut ffi::ArrowArrayStream } + +/// Free an `ArrowArrayStream` that was heap-allocated by +/// [`create_raw_pointer_for_record_batches`] via `Box::into_raw`. +/// +/// This reclaims the memory through Rust's own allocator, making it safe +/// regardless of which global allocator the Rust library was compiled with. +/// The caller must not use `ptr` after this call. +pub unsafe fn free_arrow_stream(ptr: *mut ffi::ArrowArrayStream) { + if !ptr.is_null() { + // Reconstruct the original Box. This is the + // exact type that Box::into_raw() produced in create_raw_pointer_for_record_batches(); + // the pointer was only cast to *mut ffi::ArrowArrayStream (layout-compatible + // CXX opaque type) for transport across the FFI boundary. + // Dropping the Box invokes FFI_ArrowArrayStream's Drop impl (which calls + // the Arrow release callback) and then frees the heap allocation. + unsafe { drop(Box::from_raw(ptr as *mut FFI_ArrowArrayStream)) }; + } +} diff --git a/cpp/tests/read_record_batch_tests.rs b/cpp/tests/read_record_batch_tests.rs new file mode 100644 index 00000000..7e18102a --- /dev/null +++ b/cpp/tests/read_record_batch_tests.rs @@ -0,0 +1,764 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! E2E integration tests for `HoodieFileGroupReader::read_record_batch`. +//! +//! These mirror every test in `crates/core/tests/file_group_reader_tests.rs` +//! but go through the cpp crate's `HoodieFileGroupReader` (the struct that +//! backs the C++ FFI) instead of the core `HoodieFileGroupReader` directly. +//! This validates the full construction path (ReaderContext, InputSplit, +//! Storage, ReaderParameters) that the FFI consumer uses. +//! +//! All tests are synchronous (`#[test]`) because `HoodieFileGroupReader` +//! owns a `tokio::runtime::Runtime` and calls `block_on` internally — nesting +//! a second runtime under `#[tokio::test]` would panic. + +use hudi::HoodieFileGroupReader; +use hudi::hudi_core::config::HudiConfigs; +use hudi::hudi_core::config::table::HudiTableConfig; +use hudi::hudi_core::file_group::reader::input_split::InputSplit; +use hudi::hudi_core::file_group::reader::reader_context::ReaderContext; +use hudi::hudi_core::file_group::reader::reader_parameters::ReaderParameters; +use hudi::hudi_core::file_group::reader::record_context::RecordContext; +use hudi::hudi_core::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; +use hudi::hudi_core::storage::Storage; +use hudi::hudi_core::table::builder::OptionResolver; +use hudi_test::QuickstartTripsTable; + +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use std::collections::HashMap; +use std::fs::File; +use std::sync::Arc; + +// ============================================================================= +// Helpers +// ============================================================================= + +/// Build Storage and merged props from table path using a temporary runtime. +fn create_storage_and_props(table_path: &str) -> (Arc, HashMap) { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build setup runtime"); + rt.block_on(async { + let empty_opts: Vec<(&str, &str)> = vec![]; + let mut resolver = OptionResolver::new_with_options(table_path, empty_opts); + resolver.resolve_options().await.expect("resolve options"); + let hudi_configs = Arc::new(HudiConfigs::new(resolver.hudi_options.clone())); + let storage = + Storage::new(Arc::new(resolver.storage_options), hudi_configs).expect("create storage"); + (storage, resolver.hudi_options) + }) +} + +/// Build the full `ReaderContext` the same way `new_file_group_reader_with_context` does. +fn build_reader_context( + table_path: &str, + partition: &str, + has_log_files: bool, + table_config: HashMap, +) -> Arc { + let record_context = RecordContext::new(&table_config, partition.to_string()); + Arc::new(ReaderContext { + table_path: table_path.to_string(), + latest_commit_time: "99991231235959999".to_string(), + base_file_format: String::new(), + has_log_files, + has_bootstrap_base_file: false, + needs_bootstrap_merge: false, + should_merge_use_record_position: false, + enable_logical_timestamp_field_repair: false, + iterator_mode: String::new(), + merge_mode: "COMMIT_TIME_ORDERING".to_string(), + merge_strategy_id: String::new(), + instant_range: None, + record_context, + schema_handler: FileGroupReaderSchemaHandler::new(), + table_config, + hoodie_reader_config: HashMap::new(), + key_filter_opt: None, + }) +} + +/// Build an `InputSplit` from partition, base file, and log files. +fn build_input_split(partition: &str, base_file: &str, log_files: Vec<&str>) -> InputSplit { + let base_path = if base_file.is_empty() { + None + } else if partition.is_empty() { + Some(base_file.to_string()) + } else { + Some(format!("{}/{}", partition, base_file)) + }; + let log_paths: Vec = log_files + .iter() + .map(|lf| { + if partition.is_empty() { + lf.to_string() + } else { + format!("{}/{}", partition, lf) + } + }) + .collect(); + InputSplit::new(base_path, None, log_paths, partition.to_string()) +} + +/// Construct a `HoodieFileGroupReader` (cpp crate) and call `read_record_batch`. +fn read_record_batch( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + storage: Arc, + props: HashMap, +) -> (arrow_array::RecordBatch, SchemaRef) { + let has_log_files = !log_files.is_empty(); + let input_split = build_input_split(partition, base_file, log_files); + + let mut table_config: HashMap = HashMap::new(); + table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + + let reader_context = build_reader_context(table_path, partition, has_log_files, table_config); + + let reader = HoodieFileGroupReader::new( + reader_context, + storage, + props, + ReaderParameters::default(), + input_split, + None, + ) + .expect("build HoodieFileGroupReader"); + + reader.read_record_batch().expect("read_record_batch") +} + +/// Extract (id, name, age) tuples from a RecordBatch, sorted by id. +fn extract_id_name_age(batch: &arrow_array::RecordBatch) -> Vec<(i32, String, i32)> { + QuickstartTripsTable::id_name_age(batch) +} + +/// Extract (id, name, price) tuples from a RecordBatch, sorted by id. +fn extract_id_name_price(batch: &arrow_array::RecordBatch) -> Vec<(i32, String, f64)> { + QuickstartTripsTable::id_name_price(batch) +} + +/// Extract (id, name) tuples from a RecordBatch, sorted by id. +/// Used by tests that exercise column projection where `age` has been +/// projected away. +fn extract_id_name(batch: &arrow_array::RecordBatch) -> Vec<(i32, String)> { + use arrow_array::{Int32Array, StringArray}; + let ids = batch + .column_by_name("id") + .expect("id column") + .as_any() + .downcast_ref::() + .expect("id Int32Array"); + let names = batch + .column_by_name("name") + .expect("name column") + .as_any() + .downcast_ref::() + .expect("name StringArray"); + let mut out: Vec<(i32, String)> = ids + .iter() + .zip(names.iter()) + .map(|(i, n)| (i.unwrap(), n.unwrap().to_string())) + .collect(); + out.sort_by_key(|(id, _)| *id); + out +} + +/// Read gold parquet data from disk. +fn read_gold_parquet(gold_dir: &str) -> arrow_array::RecordBatch { + let entries: Vec<_> = std::fs::read_dir(gold_dir) + .expect("gold_data dir should exist") + .filter_map(|e| e.ok()) + .filter(|e| e.path().extension().map_or(false, |ext| ext == "parquet")) + .collect(); + assert!(!entries.is_empty(), "no parquet files in {gold_dir}"); + let file = File::open(entries[0].path()).expect("open gold parquet"); + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .expect("build parquet reader") + .build() + .expect("build reader"); + let batches: Vec<_> = reader.map(|b| b.expect("read batch")).collect(); + arrow::compute::concat_batches(&batches[0].schema(), &batches).expect("concat gold batches") +} + +/// Sort a RecordBatch by the "key" column (String type, ascending). +fn sort_by_key(batch: &arrow_array::RecordBatch) -> arrow_array::RecordBatch { + let key_col = batch + .column(batch.schema().index_of("key").expect("key column")) + .clone(); + let indices = arrow_ord::sort::sort_to_indices(&key_col, None, None).expect("sort"); + let columns: Vec<_> = batch + .columns() + .iter() + .map(|col| arrow_select::take::take(col, &indices, None).expect("take")) + .collect(); + arrow_array::RecordBatch::try_new(batch.schema(), columns).expect("sorted batch") +} + +/// Compare actual reader output against gold data (non-hoodie columns). +fn assert_matches_gold( + actual: &arrow_array::RecordBatch, + gold: &arrow_array::RecordBatch, + table_name: &str, +) { + let actual_sorted = sort_by_key(actual); + let gold_sorted = sort_by_key(gold); + + assert_eq!( + actual_sorted.num_rows(), + gold_sorted.num_rows(), + "[{table_name}] row count mismatch: actual={} gold={}", + actual_sorted.num_rows(), + gold_sorted.num_rows(), + ); + + let user_cols: Vec = gold_sorted + .schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .filter(|n| !n.starts_with("_hoodie_")) + .collect(); + + for col_name in &user_cols { + let actual_idx = actual_sorted.schema().index_of(col_name); + let gold_idx = gold_sorted.schema().index_of(col_name).unwrap(); + if actual_idx.is_err() { + continue; + } + let actual_col = actual_sorted.column(actual_idx.unwrap()); + let gold_col = gold_sorted.column(gold_idx); + + let is_timestamp = matches!( + actual_col.data_type(), + arrow_schema::DataType::Timestamp(_, _) + ) || matches!( + gold_col.data_type(), + arrow_schema::DataType::Timestamp(_, _) + ); + if is_timestamp && actual_col.data_type() != gold_col.data_type() { + continue; + } + + for row in 0..actual_sorted.num_rows() { + let actual_str = arrow_cast::display::array_value_to_string(actual_col, row) + .unwrap_or_else(|_| "".to_string()); + let gold_str = arrow_cast::display::array_value_to_string(gold_col, row) + .unwrap_or_else(|_| "".to_string()); + assert_eq!( + actual_str, gold_str, + "[{table_name}] mismatch at row={row} col={col_name}: \ + actual={actual_str} gold={gold_str}", + ); + } + } +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=sf (2 base rows, 1 update in log) +// ============================================================================= + +#[test] +fn test_read_record_batch_sf_merge() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + storage, + props, + ); + + let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(col_names.contains(&"id"), "schema should contain 'id'"); + assert!(col_names.contains(&"name"), "schema should contain 'name'"); + assert!(col_names.contains(&"age"), "schema should contain 'age'"); + + let records = extract_id_name_age(&batch); + assert_eq!(records.len(), 2, "sf partition should have 2 rows"); + assert_eq!(records[0], (1, "Alice-V2".to_string(), 31)); + assert_eq!(records[1], (2, "Bob".to_string(), 25)); +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=nyc +// ============================================================================= + +#[test] +fn test_read_record_batch_nyc_merge() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "city=nyc", + "cae699a1-42f7-4226-9bd3-7f0e49496028-0_3-13-36_20260408053032350.parquet", + vec![".cae699a1-42f7-4226-9bd3-7f0e49496028-0_20260408053037787.log.1_3-27-76"], + storage, + props, + ); + + let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(col_names.contains(&"id")); + assert!(col_names.contains(&"name")); + assert!(col_names.contains(&"age")); + + let records = extract_id_name_age(&batch); + assert_eq!(records.len(), 2, "nyc partition should have 2 rows"); + assert_eq!(records[0], (3, "Carol-V2".to_string(), 36)); + assert_eq!(records[1], (4, "Dave".to_string(), 28)); +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=chi +// ============================================================================= + +#[test] +fn test_read_record_batch_chi_merge() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "city=chi", + "8fb566bd-0f0d-45ee-9f4b-391c1ceb9dda-0_2-13-35_20260408053032350.parquet", + vec![".8fb566bd-0f0d-45ee-9f4b-391c1ceb9dda-0_20260408053037787.log.1_2-27-75"], + storage, + props, + ); + + let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(col_names.contains(&"id")); + assert!(col_names.contains(&"name")); + assert!(col_names.contains(&"age")); + + let records = extract_id_name_age(&batch); + assert_eq!(records.len(), 2, "chi partition should have 2 rows"); + assert_eq!(records[0], (5, "Eve-V2".to_string(), 33)); + assert_eq!(records[1], (6, "Frank".to_string(), 40)); +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=la +// ============================================================================= + +#[test] +fn test_read_record_batch_la_merge() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "city=la", + "31d2005b-4c79-46f4-aca5-6519809b2503-0_1-13-34_20260408053032350.parquet", + vec![".31d2005b-4c79-46f4-aca5-6519809b2503-0_20260408053037787.log.1_1-27-74"], + storage, + props, + ); + + let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(col_names.contains(&"id")); + assert!(col_names.contains(&"name")); + assert!(col_names.contains(&"age")); + + let records = extract_id_name_age(&batch); + assert_eq!(records.len(), 2, "la partition should have 2 rows"); + assert_eq!(records[0], (7, "Grace-V2".to_string(), 28)); + assert_eq!(records[1], (8, "Hank".to_string(), 45)); +} + +// ============================================================================= +// v9 MOR: Base-only read (no log files) +// ============================================================================= + +#[test] +fn test_read_record_batch_base_only() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![], // no log files + storage, + props, + ); + + let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(col_names.contains(&"id")); + assert!(col_names.contains(&"name")); + assert!(col_names.contains(&"age")); + + let records = extract_id_name_age(&batch); + assert_eq!(records.len(), 2, "base-only should have 2 rows"); + assert_eq!(records[0], (1, "Alice".to_string(), 30)); + assert_eq!(records[1], (2, "Bob".to_string(), 25)); +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: Non-partitioned, 2 log files (delete + update) +// ============================================================================= + +#[test] +fn test_read_record_batch_nonpart_multi_log() { + let table_path = QuickstartTripsTable::V9MorNonpart3Commits.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "", + "960a29a0-0f78-401d-85b1-1cbc44b34121-0_0-846-1597_20260409002001492.parquet", + vec![ + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002002957.log.1_0-868-1644", + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002003963.log.1_0-890-1691", + ], + storage, + props, + ); + + let col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert!(col_names.contains(&"id")); + assert!(col_names.contains(&"name")); + assert!(col_names.contains(&"price")); + + let records = extract_id_name_price(&batch); + assert_eq!(records.len(), 4, "should have 4 rows after merge"); + assert_eq!(records[0], (3, "C".to_string(), 30.0)); + assert_eq!(records[1], (4, "D2".to_string(), 45.0)); + assert_eq!(records[2], (5, "E2".to_string(), 55.0)); + assert_eq!(records[3], (6, "F2".to_string(), 65.0)); +} + +// ============================================================================= +// MOR File Slice Layout: Log compaction (5 log files, no base) +// ============================================================================= + +#[test] +fn test_read_record_batch_log_compaction() { + let table_path = QuickstartTripsTable::MorLayoutLogCompaction.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutLogCompaction.path_to_mor_avro_gold(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "", + "", + vec![ + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030511461.log.1_0-16-23", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030518232.log.1_0-30-49", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030519923.log.1_0-44-78", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030521407.log.1_0-58-110", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030522412.log.1_0-67-128", + ], + storage, + props, + ); + + assert!(!schema.fields().is_empty(), "schema should have columns"); + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&batch, &gold, "table_log_compaction"); +} + +// ============================================================================= +// MOR File Slice Layout: Log only (3 log files) +// ============================================================================= + +#[test] +fn test_read_record_batch_log_only() { + let table_path = QuickstartTripsTable::MorLayoutLogOnly.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutLogOnly.path_to_mor_avro_gold(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "", + "", + vec![ + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030525348.log.1_0-102-176", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030527298.log.1_0-116-202", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030528554.log.1_0-130-231", + ], + storage, + props, + ); + + assert!(!schema.fields().is_empty(), "schema should have columns"); + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&batch, &gold, "table_log_only"); +} + +// ============================================================================= +// MOR File Slice Layout: Mixed column types (base + 2 log files) +// ============================================================================= + +#[test] +fn test_read_record_batch_mixed_column_types() { + let table_path = QuickstartTripsTable::MorLayoutColumnProjection.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutColumnProjection.path_to_mor_avro_gold(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "", + "78076137-b2c4-410d-8473-3d6366ae0985-0_0-165-279_20260409030530945.parquet", + vec![ + ".78076137-b2c4-410d-8473-3d6366ae0985-0_20260409030532996.log.1_0-179-305", + ".78076137-b2c4-410d-8473-3d6366ae0985-0_20260409030534379.log.1_0-193-334", + ], + storage, + props, + ); + + assert!(!schema.fields().is_empty(), "schema should have columns"); + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&batch, &gold, "table_column_projection"); +} + +// ============================================================================= +// MOR File Slice Layout: All data types (base + 3 log files) +// ============================================================================= + +#[test] +fn test_read_record_batch_all_data_types() { + let table_path = QuickstartTripsTable::MorLayoutAllDataTypes.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutAllDataTypes.path_to_mor_avro_gold(); + let (storage, props) = create_storage_and_props(&table_path); + + let (batch, schema) = read_record_batch( + &table_path, + "", + "c887c1e8-5fb9-475e-8171-769c5cf10c61-0_0-240-395_20260409030537482.parquet", + vec![ + ".c887c1e8-5fb9-475e-8171-769c5cf10c61-0_20260409030539332.log.1_0-254-421", + ".c887c1e8-5fb9-475e-8171-769c5cf10c61-0_20260409030540482.log.1_0-268-450", + ".c887c1e8-5fb9-475e-8171-769c5cf10c61-0_20260409030541620.log.1_0-282-482", + ], + storage, + props, + ); + + assert!(!schema.fields().is_empty(), "schema should have columns"); + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&batch, &gold, "table_all_data_types"); +} + +// ============================================================================= +// Column projection via schema handler (FFI shape) +// +// Verifies the FFI projection path: when a `FileGroupReaderSchemaHandler` +// is set on `ReaderContext` and the core builder is called without +// builder-level schema args, the core builder reuses the handler from +// `ReaderContext` (mod.rs `else` branch in `HoodieFileGroupReader::new`) +// and `OutputConverter` projects the merged batch from `required_schema` +// down to `requested_schema`. +// +// Mirrors the Java `HoodieFileGroupReader` constructor (lines 119-122) +// and `next()` projection loop (lines 264-265). For v9 MOR +// COMMIT_TIME_ORDERING, `required_schema = requested_schema ∪ {record_key, +// _hoodie_is_deleted (if in table schema), …}` — no precombine field is +// added (see `SchemaHandlerTestBase.testMor` line 121 in hudi-internal). +// ============================================================================= + +#[test] +fn test_read_record_batch_column_projection() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let partition = "city=sf"; + let base_file = "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet"; + let log_files = vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"]; + + let input_split = build_input_split(partition, base_file, log_files); + + // Read data schema from base file parquet metadata using a temporary runtime. + let data_schema: SchemaRef = { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build schema-read runtime"); + let base_rel = format!("{}/{}", partition, base_file); + let s = storage.clone(); + Arc::new( + rt.block_on(s.get_parquet_file_schema(&base_rel)) + .expect("read parquet schema"), + ) + }; + + // Request only id and name → projection should drop everything else. + let requested_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + ])); + + let schema_handler = FileGroupReaderSchemaHandler::new() + .with_data_schema(data_schema) + .with_requested_schema(requested_schema); + + let mut table_config: HashMap = HashMap::new(); + table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + let record_context = RecordContext::new(&table_config, partition.to_string()); + + let reader_context = Arc::new(ReaderContext { + table_path: table_path.clone(), + latest_commit_time: "99991231235959999".to_string(), + base_file_format: String::new(), + has_log_files: true, + has_bootstrap_base_file: false, + needs_bootstrap_merge: false, + should_merge_use_record_position: false, + enable_logical_timestamp_field_repair: false, + iterator_mode: String::new(), + merge_mode: "COMMIT_TIME_ORDERING".to_string(), + merge_strategy_id: String::new(), + instant_range: None, + record_context, + schema_handler, + table_config, + hoodie_reader_config: HashMap::new(), + key_filter_opt: None, + }); + + let reader = HoodieFileGroupReader::new( + reader_context, + storage, + props, + ReaderParameters::default(), + input_split, + None, + ) + .expect("build HoodieFileGroupReader"); + + let (batch, schema) = reader.read_record_batch().expect("read_record_batch"); + + // Output schema must be exactly the requested fields, in order. + let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!( + field_names, + vec!["id", "name"], + "output schema is the requested projection", + ); + assert_eq!(batch.num_columns(), 2); + + // Merge correctness: id 1 was upserted to "Alice-V2"; id 2 untouched. + let records = extract_id_name(&batch); + assert_eq!(records.len(), 2, "sf partition should have 2 rows"); + assert_eq!(records[0], (1, "Alice-V2".to_string())); + assert_eq!(records[1], (2, "Bob".to_string())); +} + +// Companion to `test_read_record_batch_column_projection` — same FFI shape +// (schema handler on `ReaderContext`, no builder schema args), but +// `requested_schema` covers all `data_schema` fields. In that case +// `required_schema == requested_schema`, `getOutputConverter()` returns +// `None`, and every column flows through unprojected. +#[test] +fn test_read_record_batch_no_projection_when_requested_equals_data() { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (storage, props) = create_storage_and_props(&table_path); + + let partition = "city=sf"; + let base_file = "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet"; + let log_files = vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"]; + + let input_split = build_input_split(partition, base_file, log_files); + + let data_schema: SchemaRef = { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build schema-read runtime"); + let base_rel = format!("{}/{}", partition, base_file); + let s = storage.clone(); + Arc::new( + rt.block_on(s.get_parquet_file_schema(&base_rel)) + .expect("read parquet schema"), + ) + }; + let expected_field_count = data_schema.fields().len(); + + // requested_schema == data_schema → required == requested → no projection. + let schema_handler = FileGroupReaderSchemaHandler::new() + .with_data_schema(data_schema.clone()) + .with_requested_schema(data_schema); + + let mut table_config: HashMap = HashMap::new(); + table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + let record_context = RecordContext::new(&table_config, partition.to_string()); + + let reader_context = Arc::new(ReaderContext { + table_path: table_path.clone(), + latest_commit_time: "99991231235959999".to_string(), + base_file_format: String::new(), + has_log_files: true, + has_bootstrap_base_file: false, + needs_bootstrap_merge: false, + should_merge_use_record_position: false, + enable_logical_timestamp_field_repair: false, + iterator_mode: String::new(), + merge_mode: "COMMIT_TIME_ORDERING".to_string(), + merge_strategy_id: String::new(), + instant_range: None, + record_context, + schema_handler, + table_config, + hoodie_reader_config: HashMap::new(), + key_filter_opt: None, + }); + + let reader = HoodieFileGroupReader::new( + reader_context, + storage, + props, + ReaderParameters::default(), + input_split, + None, + ) + .expect("build HoodieFileGroupReader"); + + let (batch, _schema) = reader.read_record_batch().expect("read_record_batch"); + + assert_eq!( + batch.num_columns(), + expected_field_count, + "no-projection path should keep every data_schema column", + ); + // Sanity: data columns are still present and merge result is correct. + let records = extract_id_name_age(&batch); + assert_eq!(records.len(), 2); + assert_eq!(records[0], (1, "Alice-V2".to_string(), 31)); + assert_eq!(records[1], (2, "Bob".to_string(), 25)); +} diff --git a/crates/core/schemas/HoodieDeleteRecord.avsc b/crates/core/schemas/HoodieDeleteRecord.avsc index 4755f459..4f4d0213 100644 --- a/crates/core/schemas/HoodieDeleteRecord.avsc +++ b/crates/core/schemas/HoodieDeleteRecord.avsc @@ -27,18 +27,103 @@ "doc": "Ordering value determining the order of merging on the same key", "type": [ "null", - "int", - "long", - "float", - "double", - "bytes", - "string", - { "type": "bytes", "logicalType": "decimal", "precision": 30, "scale": 15 }, - { "type": "int", "logicalType": "date" }, - { "type": "int", "logicalType": "time-millis" }, - { "type": "long", "logicalType": "time-micros" }, - { "type": "long", "logicalType": "timestamp-millis" }, - { "type": "long", "logicalType": "timestamp-micros" } + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "BooleanWrapper", + "fields": [{ "type": "boolean", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "IntWrapper", + "fields": [{ "type": "int", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "LongWrapper", + "fields": [{ "type": "long", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "FloatWrapper", + "fields": [{ "type": "float", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "DoubleWrapper", + "fields": [{ "type": "double", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "BytesWrapper", + "fields": [{ "type": "bytes", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "StringWrapper", + "fields": [{ "type": "string", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "DateWrapper", + "fields": [{ "type": "int", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "DecimalWrapper", + "fields": [{ "type": { "type": "bytes", "logicalType": "decimal", "precision": 30, "scale": 15 }, "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "TimeMicrosWrapper", + "fields": [{ "type": { "type": "long", "logicalType": "time-micros" }, "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "TimestampMicrosWrapper", + "fields": [{ "type": "long", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "ArrayWrapper", + "fields": [ + { + "name": "wrappedValues", + "type": [ + "null", + { + "type": "array", + "items": [ + "null", + "org.apache.hudi.avro.model.BooleanWrapper", + "org.apache.hudi.avro.model.IntWrapper", + "org.apache.hudi.avro.model.LongWrapper", + "org.apache.hudi.avro.model.FloatWrapper", + "org.apache.hudi.avro.model.DoubleWrapper", + "org.apache.hudi.avro.model.BytesWrapper", + "org.apache.hudi.avro.model.StringWrapper", + "org.apache.hudi.avro.model.DateWrapper", + "org.apache.hudi.avro.model.DecimalWrapper", + "org.apache.hudi.avro.model.TimeMicrosWrapper", + "org.apache.hudi.avro.model.TimestampMicrosWrapper" + ] + } + ], + "default": null + } + ] + } ], "default": null } diff --git a/crates/core/schemas/HoodieDeleteRecordList.avsc b/crates/core/schemas/HoodieDeleteRecordList.avsc index 19d79674..91c87478 100644 --- a/crates/core/schemas/HoodieDeleteRecordList.avsc +++ b/crates/core/schemas/HoodieDeleteRecordList.avsc @@ -37,18 +37,116 @@ "doc": "Ordering value determining the order of merging on the same key", "type": [ "null", - "int", - "long", - "float", - "double", - "bytes", - "string", - { "type": "bytes", "logicalType": "decimal", "precision": 30, "scale": 15 }, - { "type": "int", "logicalType": "date" }, - { "type": "int", "logicalType": "time-millis" }, - { "type": "long", "logicalType": "time-micros" }, - { "type": "long", "logicalType": "timestamp-millis" }, - { "type": "long", "logicalType": "timestamp-micros" } + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "BooleanWrapper", + "doc": "A record wrapping boolean type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "boolean", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "IntWrapper", + "doc": "A record wrapping int type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "int", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "LongWrapper", + "doc": "A record wrapping long type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "long", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "FloatWrapper", + "doc": "A record wrapping float type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "float", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "DoubleWrapper", + "doc": "A record wrapping double type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "double", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "BytesWrapper", + "doc": "A record wrapping bytes type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "bytes", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "StringWrapper", + "doc": "A record wrapping string type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "string", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "DateWrapper", + "doc": "A record wrapping Date logical type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "int", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "DecimalWrapper", + "doc": "A record wrapping Decimal logical type to be able to be used it w/in Avro's Union", + "fields": [{ "type": { "type": "bytes", "logicalType": "decimal", "precision": 30, "scale": 15 }, "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "TimeMicrosWrapper", + "doc": "A record wrapping Time-micros logical type to be able to be used it w/in Avro's Union", + "fields": [{ "type": { "type": "long", "logicalType": "time-micros" }, "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "TimestampMicrosWrapper", + "doc": "A record wrapping Timestamp-micros logical type to be able to be used it w/in Avro's Union", + "fields": [{ "type": "long", "name": "value" }] + }, + { + "namespace": "org.apache.hudi.avro.model", + "type": "record", + "name": "ArrayWrapper", + "doc": "A record containing a list of wrapped values.", + "fields": [ + { + "name": "wrappedValues", + "doc": "An array where each element is one of the supported wrapper types.", + "type": [ + "null", + { + "type": "array", + "items": [ + "null", + "org.apache.hudi.avro.model.BooleanWrapper", + "org.apache.hudi.avro.model.IntWrapper", + "org.apache.hudi.avro.model.LongWrapper", + "org.apache.hudi.avro.model.FloatWrapper", + "org.apache.hudi.avro.model.DoubleWrapper", + "org.apache.hudi.avro.model.BytesWrapper", + "org.apache.hudi.avro.model.StringWrapper", + "org.apache.hudi.avro.model.DateWrapper", + "org.apache.hudi.avro.model.DecimalWrapper", + "org.apache.hudi.avro.model.TimeMicrosWrapper", + "org.apache.hudi.avro.model.TimestampMicrosWrapper" + ] + } + ], + "default": null + } + ] + } ], "default": null } diff --git a/crates/core/src/avro_to_arrow/arrow_array_reader.rs b/crates/core/src/avro_to_arrow/arrow_array_reader.rs index eed1edd7..d30bb3ee 100644 --- a/crates/core/src/avro_to_arrow/arrow_array_reader.rs +++ b/crates/core/src/avro_to_arrow/arrow_array_reader.rs @@ -27,9 +27,9 @@ use apache_avro::{ types::Value, }; use arrow::array::{ - Array, ArrayBuilder, ArrayData, ArrayDataBuilder, ArrayRef, BooleanBuilder, LargeStringArray, - ListBuilder, NullArray, OffsetSizeTrait, PrimitiveArray, StringArray, StringBuilder, - StringDictionaryBuilder, make_array, + Array, ArrayBuilder, ArrayData, ArrayDataBuilder, ArrayRef, BooleanBuilder, Decimal128Array, + LargeStringArray, ListBuilder, MapArray, NullArray, OffsetSizeTrait, PrimitiveArray, + StringArray, StringBuilder, StringDictionaryBuilder, StructArray, make_array, }; use arrow::array::{BinaryArray, FixedSizeBinaryArray, GenericListArray}; use arrow::buffer::{Buffer, MutableBuffer}; @@ -435,8 +435,10 @@ impl>> AvroArrowArrayReader { let mut bool_nulls = MutableBuffer::new(num_bytes).with_bitset(num_bytes, true); let mut curr_index = 0; rows.iter().for_each(|v| { + let v = maybe_resolve_union(v); if let Value::Array(vs) = v { vs.iter().for_each(|value| { + let value = maybe_resolve_union(value); if let Value::Boolean(child) = value { // if valid boolean, append value if *child { @@ -542,8 +544,22 @@ impl>> AvroArrowArrayReader { }) .collect(); + // Check if schema_lookup covers the nested struct fields. + // If not (e.g., list items from Avro log data), build directly from records. let sub_parent_field_name = format!("{}.{}", parent_field_name, list_field.name()); - let arrays = self.build_struct_array(&rows, &sub_parent_field_name, fields)?; + let first_child_path = format!("{}.{}", sub_parent_field_name, fields[0].name()); + let has_lookup = self.schema_lookup.contains_key(&first_child_path); + let arrays = if has_lookup { + self.build_struct_array(&rows, &sub_parent_field_name, fields)? + } else { + // Build arrays directly from Record field names + let avro_values: Vec = + rows.iter().map(|r| Value::Record(r.to_vec())).collect(); + fields + .iter() + .map(|f| build_struct_field_from_avro_values(&avro_values, f)) + .collect::>>()? + }; let data_type = DataType::Struct(fields.clone()); ArrayDataBuilder::new(data_type) .len(rows.len()) @@ -673,6 +689,25 @@ impl>> AvroArrowArrayReader { *size, )?) as ArrayRef } + DataType::Decimal128(precision, scale) => { + let precision = *precision; + let scale = *scale; + let values: Vec> = rows + .iter() + .map(|row| { + let maybe_value = self.field_lookup(&field_path, row); + maybe_value.and_then(|v| resolve_decimal(v)) + }) + .collect(); + Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(precision, scale) + .map_err(|e| SchemaError(format!("Decimal128 error: {e}")))?, + ) as ArrayRef + } + DataType::Map(map_field, _sorted_keys) => { + self.build_map_array(rows, &field_path, map_field)? + } DataType::List(list_field) => { match list_field.data_type() { DataType::Dictionary(key_ty, _) => { @@ -743,6 +778,125 @@ impl>> AvroArrowArrayReader { arrays } + /// Build a MapArray from Avro Map values. + /// + /// Avro maps are `Value::Map(HashMap)`. The Arrow Map type + /// has a struct child with "key" and "value" fields. + fn build_map_array( + &self, + rows: RecordSlice, + field_path: &str, + entries_field: &arrow::datatypes::FieldRef, + ) -> ArrowResult { + // entries_field is a Struct field with children: key (String) and value (any type) + let entries_struct = match entries_field.data_type() { + DataType::Struct(fields) => fields, + other => { + return Err(SchemaError(format!( + "Map entries field should be Struct, got {other}" + ))); + } + }; + + // Collect all map entries flattened, tracking offsets + let mut offsets: Vec = vec![0]; + let mut all_keys: Vec = Vec::new(); + let mut all_values: Vec = Vec::new(); + let mut null_bitmap: Vec = Vec::new(); + + for row in rows { + let maybe_value = self.field_lookup(field_path, row); + match maybe_value.map(maybe_resolve_union) { + Some(Value::Map(map)) => { + null_bitmap.push(true); + for (k, v) in map { + all_keys.push(k.clone()); + all_values.push(v.clone()); + } + offsets.push(*offsets.last().unwrap() + map.len() as i32); + } + None | Some(Value::Null) => { + null_bitmap.push(false); + offsets.push(*offsets.last().unwrap()); + } + other => { + return Err(SchemaError(format!( + "Expected Map value for {field_path}, got {other:?}" + ))); + } + } + } + + // Build key array (always String) + let key_array: ArrayRef = Arc::new(StringArray::from(all_keys)); + + // Build value array + let value_field = &entries_struct[1]; // "value" field + let value_array = match value_field.data_type() { + DataType::Struct(value_struct_fields) => { + // Build struct array directly from Avro Record values. + // Can't use build_struct_array because schema_lookup doesn't cover map value fields. + let len = all_values.len(); + let child_arrays: ArrowResult> = value_struct_fields + .iter() + .map(|sf| build_struct_field_from_avro_values(&all_values, sf)) + .collect(); + let child_arrays = child_arrays?; + let data = ArrayDataBuilder::new(DataType::Struct(value_struct_fields.clone())) + .len(len) + .child_data(child_arrays.into_iter().map(|a| a.to_data()).collect()) + .build()?; + make_array(data) + } + DataType::Utf8 | DataType::LargeUtf8 => { + let strings: Vec> = all_values + .iter() + .map(|v| resolve_string(maybe_resolve_union(v)).ok().flatten()) + .collect(); + Arc::new(StringArray::from(strings)) as ArrayRef + } + DataType::Int32 => { + let vals: Vec> = all_values + .iter() + .map(|v| { + let v = maybe_resolve_union(v); + match v { + Value::Int(i) => Some(*i), + Value::Long(l) => Some(*l as i32), + _ => None, + } + }) + .collect(); + Arc::new(arrow_array::Int32Array::from(vals)) as ArrayRef + } + other => { + return Err(SchemaError(format!( + "Map value type {other} not yet supported in build_map_array" + ))); + } + }; + + // Build the entries struct array (key, value) + let entries_data = ArrayDataBuilder::new(DataType::Struct(entries_struct.clone())) + .len(key_array.len()) + .child_data(vec![key_array.to_data(), value_array.to_data()]) + .build()?; + let entries_array = make_array(entries_data); + + // Build offsets buffer + let offsets_buffer = Buffer::from_slice_ref(&offsets); + let null_buffer: arrow::buffer::NullBuffer = null_bitmap.into(); + + let map_data = ArrayDataBuilder::new(DataType::Map(entries_field.clone(), false)) + .len(rows.len()) + .add_buffer(offsets_buffer) + .null_bit_buffer(Some(null_buffer.inner().inner().clone())) + .child_data(vec![entries_array.to_data()]) + .build()?; + + Ok(make_array(map_data)) + } + /// Read the primitive list's values into ArrayData fn read_primitive_list_values(&self, rows: &[&Value]) -> ArrayData where @@ -877,6 +1031,138 @@ fn resolve_fixed(v: &Value, size: usize) -> Option> { } } +/// Build a single field's array from a list of Avro Values that are Records. +/// +/// Extracts the named field from each Record value and builds the appropriate Arrow array. +fn build_struct_field_from_avro_values( + values: &[Value], + field: &arrow::datatypes::FieldRef, +) -> ArrowResult { + let field_name = field.name(); + let extracted: Vec> = values + .iter() + .map(|v| { + let v = maybe_resolve_union(v); + match v { + Value::Record(fields) => fields + .iter() + .find(|(n, _)| n == field_name) + .map(|(_, val)| val), + _ => None, + } + }) + .collect(); + + match field.data_type() { + DataType::Int32 => { + let arr: Vec> = extracted + .iter() + .map(|v| { + v.and_then(|v| { + let v = maybe_resolve_union(v); + match v { + Value::Int(i) => Some(*i), + Value::Long(l) => Some(*l as i32), + _ => None, + } + }) + }) + .collect(); + Ok(Arc::new(arrow_array::Int32Array::from(arr)) as ArrayRef) + } + DataType::Int64 => { + let arr: Vec> = extracted + .iter() + .map(|v| { + v.and_then(|v| { + let v = maybe_resolve_union(v); + match v { + Value::Long(l) => Some(*l), + Value::Int(i) => Some(*i as i64), + _ => None, + } + }) + }) + .collect(); + Ok(Arc::new(arrow_array::Int64Array::from(arr)) as ArrayRef) + } + DataType::Utf8 | DataType::LargeUtf8 => { + let arr: Vec> = extracted + .iter() + .map(|v| v.and_then(|v| resolve_string(maybe_resolve_union(v)).ok().flatten())) + .collect(); + Ok(Arc::new(StringArray::from(arr)) as ArrayRef) + } + DataType::Float64 => { + let arr: Vec> = extracted + .iter() + .map(|v| { + v.and_then(|v| { + let v = maybe_resolve_union(v); + match v { + Value::Double(d) => Some(*d), + Value::Float(f) => Some(*f as f64), + _ => None, + } + }) + }) + .collect(); + Ok(Arc::new(arrow_array::Float64Array::from(arr)) as ArrayRef) + } + DataType::Boolean => { + let arr: Vec> = extracted + .iter() + .map(|v| { + v.and_then(|v| { + let v = maybe_resolve_union(v); + match v { + Value::Boolean(b) => Some(*b), + _ => None, + } + }) + }) + .collect(); + Ok(Arc::new(arrow_array::BooleanArray::from(arr)) as ArrayRef) + } + other => Err(SchemaError(format!( + "Map struct field type {other} not yet supported in build_struct_field_from_avro_values" + ))), + } +} + +/// Resolve an Avro decimal value (Fixed or Bytes) to i128. +/// +/// Avro stores decimals as big-endian two's complement bytes in Fixed or Bytes. +fn resolve_decimal(v: &Value) -> Option { + let v = if let Value::Union(_, b) = v { b } else { v }; + let bytes = match v { + Value::Fixed(_, bytes) => bytes.as_slice(), + Value::Bytes(bytes) => bytes.as_slice(), + Value::Decimal(decimal) => { + // apache-avro crate's Decimal: use TryFrom to get big-endian bytes + let bytes: Vec = Vec::::try_from(decimal).ok()?; + return Some(decimal_bytes_to_i128(&bytes)); + } + Value::Null => return None, + _ => return None, + }; + Some(decimal_bytes_to_i128(bytes)) +} + +/// Convert big-endian two's complement bytes to i128. +fn decimal_bytes_to_i128(bytes: &[u8]) -> i128 { + if bytes.is_empty() { + return 0; + } + // Sign-extend to 16 bytes + let is_negative = bytes[0] & 0x80 != 0; + let pad = if is_negative { 0xFF } else { 0x00 }; + let mut buf = [pad; 16]; + let start = 16 - bytes.len(); + buf[start..].copy_from_slice(bytes); + i128::from_be_bytes(buf) +} + fn resolve_boolean(value: &Value) -> Option { let v = if let Value::Union(_, b) = value { b @@ -930,3 +1216,158 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use apache_avro::schema::Schema as AvroSchema; + use arrow::array::{AsArray, BooleanArray, Int32Array, ListArray}; + + /// Helper: build an AvroArrowArrayReader from an Avro schema JSON string + /// and a vec of Avro record Values, then read all into a RecordBatch. + fn read_records(schema_json: &str, records: Vec) -> RecordBatch { + let avro_schema = AvroSchema::parse_str(schema_json).unwrap(); + let values = records.into_iter().map(Ok); + let mut reader = AvroArrowArrayReader::try_new(values, &avro_schema).unwrap(); + reader.next_batch(1024).unwrap().expect("expected a batch") + } + + /// Regression test: ARRAY elements wrapped in Avro unions + /// must be resolved before matching. Without the maybe_resolve_union + /// call, every boolean element is treated as null and reads back as + /// false (the zero-initialized default). + /// + /// Uses the same nullable-array schema that Hudi generates: + /// {"type": ["null", {"type": "array", "items": ["null", "boolean"]}]} + #[test] + fn test_boolean_array_with_union_elements() { + let schema_json = r#"{ + "type": "record", + "name": "test", + "fields": [ + {"name": "key", "type": "string"}, + {"name": "arr_bool", "type": ["null", { + "type": "array", + "items": ["null", "boolean"] + }]} + ] + }"#; + + let records = vec![ + Value::Record(vec![ + ("key".into(), Value::String("k1".into())), + ( + "arr_bool".into(), + Value::Union( + 1, + Box::new(Value::Array(vec![ + Value::Union(1, Box::new(Value::Boolean(false))), + Value::Union(1, Box::new(Value::Boolean(true))), + Value::Union(1, Box::new(Value::Boolean(false))), + ])), + ), + ), + ]), + Value::Record(vec![ + ("key".into(), Value::String("k2".into())), + ( + "arr_bool".into(), + Value::Union( + 1, + Box::new(Value::Array(vec![ + Value::Union(1, Box::new(Value::Boolean(true))), + Value::Union(1, Box::new(Value::Boolean(true))), + Value::Union(1, Box::new(Value::Boolean(true))), + ])), + ), + ), + ]), + Value::Record(vec![ + ("key".into(), Value::String("k3".into())), + ( + "arr_bool".into(), + Value::Union( + 1, + Box::new(Value::Array(vec![ + Value::Union(1, Box::new(Value::Boolean(true))), + Value::Union(1, Box::new(Value::Boolean(false))), + Value::Union(1, Box::new(Value::Boolean(true))), + ])), + ), + ), + ]), + ]; + + let batch = read_records(schema_json, records); + assert_eq!(batch.num_rows(), 3); + + let arr_col = batch.column_by_name("arr_bool").unwrap(); + let list_arr = arr_col.as_any().downcast_ref::().unwrap(); + + // k1: [false, true, false] + let k1_vals = list_arr.value(0); + let k1_bools = k1_vals.as_any().downcast_ref::().unwrap(); + assert_eq!(k1_bools.len(), 3); + assert_eq!(k1_bools.value(0), false); + assert_eq!(k1_bools.value(1), true); // THIS was the bug: returned false + assert_eq!(k1_bools.value(2), false); + + // k2: [true, true, true] + let k2_vals = list_arr.value(1); + let k2_bools = k2_vals.as_any().downcast_ref::().unwrap(); + assert_eq!(k2_bools.len(), 3); + assert_eq!(k2_bools.value(0), true); + assert_eq!(k2_bools.value(1), true); + assert_eq!(k2_bools.value(2), true); + + // k3: [true, false, true] + let k3_vals = list_arr.value(2); + let k3_bools = k3_vals.as_any().downcast_ref::().unwrap(); + assert_eq!(k3_bools.len(), 3); + assert_eq!(k3_bools.value(0), true); + assert_eq!(k3_bools.value(1), false); + assert_eq!(k3_bools.value(2), true); + } + + /// Sanity check: ARRAY with union elements works correctly + /// (the primitive path already calls maybe_resolve_union via resolve_item). + #[test] + fn test_int_array_with_union_elements() { + let schema_json = r#"{ + "type": "record", + "name": "test", + "fields": [ + {"name": "key", "type": "string"}, + {"name": "arr_int", "type": ["null", { + "type": "array", + "items": ["null", "int"] + }]} + ] + }"#; + + let records = vec![Value::Record(vec![ + ("key".into(), Value::String("k1".into())), + ( + "arr_int".into(), + Value::Union( + 1, + Box::new(Value::Array(vec![ + Value::Union(1, Box::new(Value::Int(10))), + Value::Union(1, Box::new(Value::Int(20))), + Value::Union(1, Box::new(Value::Int(30))), + ])), + ), + ), + ])]; + + let batch = read_records(schema_json, records); + let arr_col = batch.column_by_name("arr_int").unwrap(); + let list_arr = arr_col.as_any().downcast_ref::().unwrap(); + let k1_vals = list_arr.value(0); + let k1_ints = k1_vals.as_any().downcast_ref::().unwrap(); + assert_eq!(k1_ints.len(), 3); + assert_eq!(k1_ints.value(0), 10); + assert_eq!(k1_ints.value(1), 20); + assert_eq!(k1_ints.value(2), 30); + } +} diff --git a/crates/core/src/avro_to_arrow/schema.rs b/crates/core/src/avro_to_arrow/schema.rs index 5b618b50..d25112d1 100644 --- a/crates/core/src/avro_to_arrow/schema.rs +++ b/crates/core/src/avro_to_arrow/schema.rs @@ -19,7 +19,7 @@ use crate::error::Result; use apache_avro::Schema as AvroSchema; use apache_avro::schema::{Alias, DecimalSchema, EnumSchema, FixedSchema, Name, RecordSchema}; use apache_avro::types::Value; -use arrow::datatypes::{DataType, IntervalUnit, Schema, TimeUnit, UnionMode}; +use arrow::datatypes::{DataType, Fields, IntervalUnit, Schema, TimeUnit, UnionMode}; use arrow::datatypes::{Field, UnionFields}; use std::collections::HashMap; use std::sync::Arc; @@ -70,19 +70,27 @@ fn schema_to_field_with_props( AvroSchema::Double => DataType::Float64, AvroSchema::Bytes => DataType::Binary, AvroSchema::String => DataType::Utf8, - AvroSchema::Array(item_schema) => DataType::List(Arc::new(schema_to_field_with_props( - &item_schema.items, - Some("element"), - false, - None, - )?)), + AvroSchema::Array(item_schema) => { + // Use "array" as the child field name to match Spark/Parquet convention + // (Parquet uses "array" while Avro convention uses "element"). + let mut item_field = + schema_to_field_with_props(&item_schema.items, Some("array"), false, None)?; + // Spark's Parquet writer marks list elements as non-null + item_field = item_field.with_nullable(false); + DataType::List(Arc::new(item_field)) + } AvroSchema::Map(value_schema) => { let value_field = schema_to_field_with_props(&value_schema.types, Some("value"), false, None)?; - DataType::Dictionary( - Box::new(DataType::Utf8), - Box::new(value_field.data_type().clone()), - ) + let entries_field = Field::new( + "key_value", + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + value_field, + ])), + false, + ); + DataType::Map(Arc::new(entries_field), false) } AvroSchema::Union(us) => { // If there are only two variants and one of them is null, set the other type as the field data type @@ -214,7 +222,7 @@ fn default_field_name(dt: &DataType) -> &str { DataType::Struct(_) => "struct", DataType::Union(_, _) => "union", DataType::Dictionary(_, _) => "map", - DataType::Map(_, _) => unimplemented!("Map support not implemented"), + DataType::Map(_, _) => "map", DataType::RunEndEncoded(_, _) => { unimplemented!("RunEndEncoded support not implemented") } diff --git a/crates/core/src/config/table.rs b/crates/core/src/config/table.rs index 6ff9169e..d035966c 100644 --- a/crates/core/src/config/table.rs +++ b/crates/core/src/config/table.rs @@ -217,6 +217,19 @@ impl ConfigParser for HudiTableConfig { .map(|v| v.as_str()) .ok_or(NotFound(self.key())); + // For PrecombineField, also check hoodie.table.ordering.fields (Hudi 1.x + // renamed precombine.field to ordering.fields; the old key is an alternative). + let get_result = if matches!(self, Self::PrecombineField) { + get_result.or_else(|_| { + configs + .get("hoodie.table.ordering.fields") + .map(|v| v.as_str()) + .ok_or(NotFound(self.key())) + }) + } else { + get_result + }; + match self { Self::BaseFileFormat => get_result .and_then(BaseFileFormatValue::from_str) @@ -303,8 +316,10 @@ impl ConfigParser for HudiTableConfig { ); } - if !configs.contains_key(HudiTableConfig::PrecombineField.as_ref()) { - // When precombine field is not available, we treat the table as append-only + if !configs.contains_key(HudiTableConfig::PrecombineField.as_ref()) + && !configs.contains_key("hoodie.table.ordering.fields") + { + // When precombine/ordering field is not available, we treat the table as append-only return HudiConfigValue::String( RecordMergeStrategyValue::AppendOnly.as_ref().to_string(), ); diff --git a/crates/core/src/config/util.rs b/crates/core/src/config/util.rs index fd857a34..ec938b7c 100644 --- a/crates/core/src/config/util.rs +++ b/crates/core/src/config/util.rs @@ -35,6 +35,17 @@ pub fn empty_filters<'a>() -> std::iter::Empty<(&'a str, &'a str, &'a str)> { } /// Splits the given options into two maps: one for Hudi options, and the other for others, which could be storage options for example. +// Full Flow: Props/Config from Scala → Rust + +// Scala (buildProps) → Java (proto builder) → Proto (wire) +// → C++ gluten (unpack) → C++ velox (copy) → C++ (build FfiReaderContext) +// → Rust (new_file_group_reader_with_context) + +// The root cause: In Scala buildProps(), ALL options (hoodie.* + cloud creds like fs.s3a.*) go into props. Same in +// buildReaderContext() → tableConfig. Rust then has to split them apart because Storage::new() needs cloud creds +// and HudiConfigs needs hoodie.* keys. + +// To eliminate split_hudi_options_from_others, we need storage_opts as a separate field pub fn split_hudi_options_from_others( all_options: I, ) -> (HashMap, HashMap) diff --git a/crates/core/src/expression/array_data.rs b/crates/core/src/expression/array_data.rs new file mode 100644 index 00000000..cd02591e --- /dev/null +++ b/crates/core/src/expression/array_data.rs @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.ArrayData`. + +use crate::expression::struct_like::StructLike; +use std::any::Any; + +/// Concrete `StructLike` implementation backed by a vector of boxed `Any` +/// values. Mirrors Java `ArrayData(List data)`. +pub struct ArrayData { + data: Vec>, +} + +impl ArrayData { + pub fn new(data: Vec>) -> Self { + Self { data } + } +} + +impl StructLike for ArrayData { + fn num_fields(&self) -> usize { + self.data.len() + } + + fn get(&self, pos: usize) -> Option<&dyn Any> { + self.data.get(pos).map(|b| b.as_ref() as &dyn Any) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn array_data_implements_struct_like() { + let arr = ArrayData::new(vec![ + Box::new(42i32) as Box, + Box::new("hello".to_string()), + ]); + assert_eq!(arr.num_fields(), 2); + assert_eq!( + arr.get(0).and_then(|v| v.downcast_ref::()).copied(), + Some(42) + ); + assert!(arr.get(2).is_none()); + } +} diff --git a/crates/core/src/expression/binary_expression.rs b/crates/core/src/expression/binary_expression.rs new file mode 100644 index 00000000..6fd6e918 --- /dev/null +++ b/crates/core/src/expression/binary_expression.rs @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.BinaryExpression`. + +use crate::expression::{Expression, Operator}; + +/// Mirrors Java `BinaryExpression`. Fields: left, operator, right. +/// +/// Java has `protected final` fields and getters. Rust struct mirrors +/// that with public accessors. Note: this is a helper struct used by +/// concrete predicates (And, Or, BinaryComparison) via composition, +/// not an abstract class. +#[derive(Debug)] +pub struct BinaryExpression { + pub left: Box, + pub operator: Operator, + pub right: Box, +} + +impl BinaryExpression { + pub fn new(left: Box, operator: Operator, right: Box) -> Self { + Self { + left, + operator, + right, + } + } + + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right(&self) -> &dyn Expression { + self.right.as_ref() + } + pub fn operator(&self) -> Operator { + self.operator + } +} diff --git a/crates/core/src/expression/bind_visitor.rs b/crates/core/src/expression/bind_visitor.rs new file mode 100644 index 00000000..9a4725d8 --- /dev/null +++ b/crates/core/src/expression/bind_visitor.rs @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.BindVisitor`. +//! +//! BindVisitor walks an expression tree, replacing `NameReference`s with +//! `BoundReference`s by looking up names in a `RecordType` schema. +//! +//! Ported as a placeholder — not wired into the keyFilterOpt reader path. +//! See keyFilterOpt design spec §6 deviation #6. + +use crate::expression::expression_visitor::ExpressionVisitor; +use crate::internal_schema::types::RecordType; + +pub struct BindVisitor { + pub schema: RecordType, + pub case_sensitive: bool, +} + +impl BindVisitor { + pub fn new(schema: RecordType, case_sensitive: bool) -> Self { + Self { + schema, + case_sensitive, + } + } +} + +impl ExpressionVisitor for BindVisitor {} diff --git a/crates/core/src/expression/bound_reference.rs b/crates/core/src/expression/bound_reference.rs new file mode 100644 index 00000000..6dfd85b7 --- /dev/null +++ b/crates/core/src/expression/bound_reference.rs @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.BoundReference`. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::struct_like::StructLike; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::Type; +use std::any::Any; + +/// Reference to a column by ordinal position. Mirrors Java `BoundReference`. +#[derive(Debug)] +pub struct BoundReference { + ordinal: usize, + data_type: Box, +} + +impl BoundReference { + pub fn new(ordinal: usize, data_type: Box) -> Self { + Self { ordinal, data_type } + } + + pub fn ordinal(&self) -> usize { + self.ordinal + } +} + +impl Expression for BoundReference { + fn data_type(&self) -> &dyn Type { + self.data_type.as_ref() + } + + fn operator(&self) -> Operator { + Operator::True + } + + /// Mirrors Java `BoundReference.eval(data)` — returns the cell value + /// at `ordinal`. Returns None on the keyFilterOpt path; not exercised. + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let s = data?; + let _ = s.get(self.ordinal)?; + None + } + + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::BoundReference(self) + } +} + +impl LeafExpression for BoundReference {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, ExpressionKind}; + use crate::internal_schema::types::IntType; + + #[test] + fn bound_reference_carries_ordinal_and_type() { + let br = BoundReference::new(3, Box::new(IntType::get())); + assert_eq!(br.ordinal(), 3); + assert_eq!( + br.data_type().type_id(), + crate::internal_schema::TypeID::Int + ); + } + + #[test] + fn bound_reference_kind_pattern_match() { + let br = BoundReference::new(0, Box::new(IntType::get())); + match br.kind() { + ExpressionKind::BoundReference(_) => {} + _ => panic!("expected ExpressionKind::BoundReference"), + } + } +} diff --git a/crates/core/src/expression/comparators.rs b/crates/core/src/expression/comparators.rs new file mode 100644 index 00000000..de6f6516 --- /dev/null +++ b/crates/core/src/expression/comparators.rs @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.Comparators`. + +use crate::expression::LiteralValue; +use crate::internal_schema::{Type, TypeID}; +use std::cmp::Ordering; + +pub struct Comparators; + +impl Comparators { + /// Mirrors Java `Comparators.forType(Type.PrimitiveType)`. Returns a + /// comparator function over `LiteralValue`. For unsupported types the + /// comparator returns `Ordering::Equal` (equivalent to Java throwing, + /// but we don't panic since the keyFilterOpt path doesn't exercise + /// non-string comparisons). + pub fn for_type(ty: &dyn Type) -> fn(&LiteralValue, &LiteralValue) -> Ordering { + match ty.type_id() { + TypeID::Boolean => |a, b| match (a, b) { + (LiteralValue::Bool(x), LiteralValue::Bool(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::Int | TypeID::Date | TypeID::TimeMillis => |a, b| match (a, b) { + (LiteralValue::Int(x), LiteralValue::Int(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::Long + | TypeID::Time + | TypeID::Timestamp + | TypeID::TimestampMillis + | TypeID::LocalTimestampMillis + | TypeID::LocalTimestampMicros => |a, b| match (a, b) { + (LiteralValue::Long(x), LiteralValue::Long(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::Float => |a, b| match (a, b) { + (LiteralValue::Float(x), LiteralValue::Float(y)) => { + x.partial_cmp(y).unwrap_or(Ordering::Equal) + } + _ => Ordering::Equal, + }, + TypeID::Double => |a, b| match (a, b) { + (LiteralValue::Double(x), LiteralValue::Double(y)) => { + x.partial_cmp(y).unwrap_or(Ordering::Equal) + } + _ => Ordering::Equal, + }, + TypeID::String => |a, b| match (a, b) { + (LiteralValue::String(x), LiteralValue::String(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::UUID => |a, b| match (a, b) { + (LiteralValue::UUID(x), LiteralValue::UUID(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + _ => |_, _| Ordering::Equal, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::LiteralValue; + use crate::internal_schema::types::*; + use std::cmp::Ordering; + + #[test] + fn int_comparator_orders_correctly() { + let cmp = Comparators::for_type(&IntType::get()); + assert_eq!( + cmp(&LiteralValue::Int(1), &LiteralValue::Int(2)), + Ordering::Less + ); + assert_eq!( + cmp(&LiteralValue::Int(5), &LiteralValue::Int(5)), + Ordering::Equal + ); + assert_eq!( + cmp(&LiteralValue::Int(9), &LiteralValue::Int(2)), + Ordering::Greater + ); + } + + #[test] + fn string_comparator_lexicographic() { + let cmp = Comparators::for_type(&StringType::get()); + assert_eq!( + cmp( + &LiteralValue::String("a".into()), + &LiteralValue::String("b".into()) + ), + Ordering::Less + ); + } + + #[test] + fn boolean_comparator() { + let cmp = Comparators::for_type(&BooleanType::get()); + assert_eq!( + cmp(&LiteralValue::Bool(false), &LiteralValue::Bool(true)), + Ordering::Less + ); + } +} diff --git a/crates/core/src/expression/expression.rs b/crates/core/src/expression/expression.rs new file mode 100644 index 00000000..4182da37 --- /dev/null +++ b/crates/core/src/expression/expression.rs @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.Expression`. +//! +//! Defines the `Expression` trait and the `Operator` enum. Adds an +//! `ExpressionKind<'_>` accessor for idiomatic Rust pattern matching +//! (see keyFilterOpt design spec §4.1, deviation #2). + +use crate::expression::struct_like::StructLike; +use crate::internal_schema::Type; +use std::any::Any; +use std::fmt::Debug; + +/// Mirrors Java `Expression.Operator`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Operator { + And, + Or, + Not, + Eq, + Gt, + Lt, + GtEq, + LtEq, + In, + IsNull, + IsNotNull, + StartsWith, + Contains, + True, + False, +} + +impl Operator { + /// Mirrors Java `Operator.symbol`. + pub fn symbol(&self) -> &'static str { + match self { + Operator::And => "AND", + Operator::Or => "OR", + Operator::Not => "NOT", + Operator::Eq => "=", + Operator::Gt => ">", + Operator::Lt => "<", + Operator::GtEq => ">=", + Operator::LtEq => "<=", + Operator::In => "IN", + Operator::IsNull => "IS NULL", + Operator::IsNotNull => "IS NOT NULL", + Operator::StartsWith => "starts_with", + Operator::Contains => "contains", + Operator::True => "TRUE", + Operator::False => "FALSE", + } + } +} + +/// Mirrors Java `org.apache.hudi.expression.Expression`. +/// +/// Trait must be `Send + Sync + Debug` so trait objects can be `Arc`'d +/// across the reader pipeline. +pub trait Expression: Debug + Send + Sync + Any { + /// Mirrors Java `Expression.getDataType()`. + fn data_type(&self) -> &dyn Type; + + /// Mirrors Java `Expression.getChildren()`. + fn children(&self) -> Vec<&dyn Expression> { + Vec::new() + } + + /// Mirrors Java `Expression.getOperator()`. + fn operator(&self) -> Operator; + + /// Mirrors Java `Expression.eval(StructLike data)`. Default is `None` + /// — concrete types override. + /// + /// Returns `Box` so callers can downcast to the expected type. + /// Not used on the keyFilterOpt reader path; ported for unit-test parity. + fn eval(&self, _data: Option<&dyn StructLike>) -> Option> { + None + } + + /// Borrowed enum view for idiomatic Rust pattern matching. + /// + /// Variants are added incrementally as each concrete Expression type is + /// introduced in subsequent tasks (Literal — Task 1.8, NameReference — + /// Task 1.9, BoundReference — Task 1.10, Predicate — Task 1.11). + fn kind(&self) -> ExpressionKind<'_>; +} + +/// Borrowed enum view of `Expression` for pattern matching. Variants are +/// added as concrete types come online — see comment on `Expression::kind`. +pub enum ExpressionKind<'a> { + Literal(&'a crate::expression::literal::Literal), + NameReference(&'a crate::expression::name_reference::NameReference), + BoundReference(&'a crate::expression::bound_reference::BoundReference), + Predicate(&'a dyn crate::expression::predicate::Predicate), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn operator_symbol_matches_java() { + assert_eq!(Operator::And.symbol(), "AND"); + assert_eq!(Operator::Or.symbol(), "OR"); + assert_eq!(Operator::Not.symbol(), "NOT"); + assert_eq!(Operator::Eq.symbol(), "="); + assert_eq!(Operator::Gt.symbol(), ">"); + assert_eq!(Operator::Lt.symbol(), "<"); + assert_eq!(Operator::GtEq.symbol(), ">="); + assert_eq!(Operator::LtEq.symbol(), "<="); + assert_eq!(Operator::In.symbol(), "IN"); + assert_eq!(Operator::IsNull.symbol(), "IS NULL"); + assert_eq!(Operator::IsNotNull.symbol(), "IS NOT NULL"); + assert_eq!(Operator::StartsWith.symbol(), "starts_with"); + assert_eq!(Operator::Contains.symbol(), "contains"); + assert_eq!(Operator::True.symbol(), "TRUE"); + assert_eq!(Operator::False.symbol(), "FALSE"); + } +} diff --git a/crates/core/src/expression/expression_visitor.rs b/crates/core/src/expression/expression_visitor.rs new file mode 100644 index 00000000..742a43f6 --- /dev/null +++ b/crates/core/src/expression/expression_visitor.rs @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.ExpressionVisitor`. + +use crate::expression::bound_reference::BoundReference; +use crate::expression::predicates::{ + And, BinaryComparison, FalseExpression, In, IsNotNull, IsNull, Not, Or, StringContains, + StringStartsWith, StringStartsWithAny, TrueExpression, +}; +use crate::expression::{Expression, Literal, NameReference}; + +/// Mirrors Java `ExpressionVisitor`. Each visit method has a default +/// returning `T::default()` (Rust constraint: `T: Default`). +pub trait ExpressionVisitor { + fn always_true(&mut self) -> T + where + T: Default, + { + T::default() + } + fn always_false(&mut self) -> T + where + T: Default, + { + T::default() + } + fn visit_and(&mut self, _and: &And) -> T + where + T: Default, + { + T::default() + } + fn visit_or(&mut self, _or: &Or) -> T + where + T: Default, + { + T::default() + } + fn visit_not(&mut self, _not: &Not) -> T + where + T: Default, + { + T::default() + } + fn visit_predicate(&mut self, _expr: &dyn Expression) -> T + where + T: Default, + { + T::default() + } + fn visit_literal(&mut self, _lit: &Literal) -> T + where + T: Default, + { + T::default() + } + fn visit_name_reference(&mut self, _nr: &NameReference) -> T + where + T: Default, + { + T::default() + } + fn visit_bound_reference(&mut self, _br: &BoundReference) -> T + where + T: Default, + { + T::default() + } + fn visit_binary_comparison(&mut self, _bc: &BinaryComparison) -> T + where + T: Default, + { + T::default() + } + fn visit_in(&mut self, _in_: &In) -> T + where + T: Default, + { + T::default() + } + fn visit_is_null(&mut self, _p: &IsNull) -> T + where + T: Default, + { + T::default() + } + fn visit_is_not_null(&mut self, _p: &IsNotNull) -> T + where + T: Default, + { + T::default() + } + fn visit_string_starts_with(&mut self, _p: &StringStartsWith) -> T + where + T: Default, + { + T::default() + } + fn visit_string_starts_with_any(&mut self, _p: &StringStartsWithAny) -> T + where + T: Default, + { + T::default() + } + fn visit_string_contains(&mut self, _p: &StringContains) -> T + where + T: Default, + { + T::default() + } + fn visit_true(&mut self, _t: &TrueExpression) -> T + where + T: Default, + { + T::default() + } + fn visit_false(&mut self, _f: &FalseExpression) -> T + where + T: Default, + { + T::default() + } +} diff --git a/crates/core/src/expression/leaf_expression.rs b/crates/core/src/expression/leaf_expression.rs new file mode 100644 index 00000000..7f5ada1c --- /dev/null +++ b/crates/core/src/expression/leaf_expression.rs @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.LeafExpression`. + +use crate::expression::Expression; + +/// Marker trait for expressions with no children. Mirrors Java's +/// `LeafExpression` abstract class. +pub trait LeafExpression: Expression {} diff --git a/crates/core/src/expression/literal.rs b/crates/core/src/expression/literal.rs new file mode 100644 index 00000000..2314a0cf --- /dev/null +++ b/crates/core/src/expression/literal.rs @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.Literal`. +//! +//! Diverges from Java in one place: Java `Literal` is generic over T; +//! Rust uses a closed `LiteralValue` enum so callers extracting values +//! via `kind()` can pattern-match without a second downcast. See the +//! keyFilterOpt design spec §6 deviation #1. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::Type; +use crate::internal_schema::types::*; + +/// Closed enum of literal-value variants. Mirrors the runtime values +/// Java's `Literal` can carry. Variants are 1:1 with `Types.java` primitives +/// (we store the unscaled+precision+scale tuple for Decimal rather than a +/// `BigDecimal`). +#[derive(Debug, Clone, PartialEq)] +pub enum LiteralValue { + Bool(bool), + Int(i32), + Long(i64), + Float(f32), + Double(f64), + Date(i32), // days since epoch + Time(i64), // micros since midnight + TimeMillis(i32), // millis since midnight + Timestamp(i64), // micros since epoch + TimestampMillis(i64), // millis since epoch + LocalTimestampMillis(i64), + LocalTimestampMicros(i64), + Decimal { + unscaled: i128, + precision: u8, + scale: u8, + }, + String(String), + Binary(Vec), + Fixed(Vec), + UUID([u8; 16]), + Null, +} + +/// Mirrors Java `Literal(T value, Type type)` — non-generic in Rust. +#[derive(Debug)] +pub struct Literal { + value: LiteralValue, + data_type: Box, +} + +impl Clone for Literal { + fn clone(&self) -> Self { + // Reconstruct data_type from TypeID since Box is not Clone. + let data_type = clone_type_by_id(self.data_type.as_ref()); + Self { + value: self.value.clone(), + data_type, + } + } +} + +/// Clones a `Box` by rebuilding the concrete type from its TypeID. +/// Parameterised types (FixedType, DecimalType) are not yet reachable via the +/// convenience constructors, so they fall through to StringType as a safe +/// default (they can be added here when needed). +fn clone_type_by_id(t: &dyn Type) -> Box { + use crate::internal_schema::TypeID; + match t.type_id() { + TypeID::Boolean => Box::new(BooleanType::get()), + TypeID::Int => Box::new(IntType::get()), + TypeID::Long => Box::new(LongType::get()), + TypeID::Float => Box::new(FloatType::get()), + TypeID::Double => Box::new(DoubleType::get()), + TypeID::Date => Box::new(DateType::get()), + TypeID::Time => Box::new(TimeType::get()), + TypeID::TimeMillis => Box::new(TimeMillisType::get()), + TypeID::Timestamp => Box::new(TimestampType::get()), + TypeID::TimestampMillis => Box::new(TimestampMillisType::get()), + TypeID::LocalTimestampMillis => Box::new(LocalTimestampMillisType::get()), + TypeID::LocalTimestampMicros => Box::new(LocalTimestampMicrosType::get()), + TypeID::String => Box::new(StringType::get()), + TypeID::Binary => Box::new(BinaryType::get()), + TypeID::UUID => Box::new(UUIDType::get()), + // FixedType and DecimalType carry parameters not recoverable from TypeID alone; + // they are not yet used by convenience constructors. Fall back to StringType. + _ => Box::new(StringType::get()), + } +} + +impl Literal { + pub fn new(value: LiteralValue, data_type: Box) -> Self { + Self { value, data_type } + } + + /// Mirrors Java `Literal.getValue()` (returns the carried value enum). + pub fn value(&self) -> &LiteralValue { + &self.value + } + + // Convenience constructors — mirror Java's `Literal.from(value)` static. + pub fn string(s: impl Into) -> Self { + Self::new(LiteralValue::String(s.into()), Box::new(StringType::get())) + } + pub fn int(n: i32) -> Self { + Self::new(LiteralValue::Int(n), Box::new(IntType::get())) + } + pub fn long(n: i64) -> Self { + Self::new(LiteralValue::Long(n), Box::new(LongType::get())) + } + pub fn boolean(b: bool) -> Self { + Self::new(LiteralValue::Bool(b), Box::new(BooleanType::get())) + } + pub fn double(d: f64) -> Self { + Self::new(LiteralValue::Double(d), Box::new(DoubleType::get())) + } + pub fn float(d: f32) -> Self { + Self::new(LiteralValue::Float(d), Box::new(FloatType::get())) + } + pub fn null() -> Self { + Self::new(LiteralValue::Null, Box::new(StringType::get())) + // ^ Java's Literal.from(null) defaults to STRING type. Rust mirrors. + } +} + +impl Expression for Literal { + fn data_type(&self) -> &dyn Type { + self.data_type.as_ref() + } + + /// Java `Literal.getOperator()` is inherited from LeafExpression which + /// in turn returns `Operator.TRUE` (Java has no specific literal op). + fn operator(&self) -> Operator { + Operator::True + } + + fn eval( + &self, + _data: Option<&dyn crate::expression::struct_like::StructLike>, + ) -> Option> { + // Mirrors Java: `Literal.eval(data)` returns the carried value + // ignoring `data`. We box-clone the LiteralValue. + Some(Box::new(self.value.clone())) + } + + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Literal(self) + } +} + +impl LeafExpression for Literal {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, ExpressionKind, Operator}; + use crate::internal_schema::TypeID; + + #[test] + fn literal_string_value_extraction() { + let lit = Literal::string("hello"); + assert_eq!(lit.data_type().type_id(), TypeID::String); + match lit.value() { + LiteralValue::String(s) => assert_eq!(s, "hello"), + _ => panic!("expected String literal"), + } + } + + #[test] + fn literal_int_value_extraction() { + let lit = Literal::int(42); + assert_eq!(lit.data_type().type_id(), TypeID::Int); + match lit.value() { + LiteralValue::Int(n) => assert_eq!(*n, 42), + _ => panic!("expected Int literal"), + } + } + + #[test] + fn literal_kind_pattern_match() { + let lit = Literal::string("k"); + match lit.kind() { + ExpressionKind::Literal(_) => {} + _ => panic!("expected ExpressionKind::Literal"), + } + } + + #[test] + fn literal_operator_is_true() { + let lit = Literal::int(1); + assert_eq!(lit.operator(), Operator::True); + } +} diff --git a/crates/core/src/expression/mod.rs b/crates/core/src/expression/mod.rs new file mode 100644 index 00000000..9aaef65f --- /dev/null +++ b/crates/core/src/expression/mod.rs @@ -0,0 +1,41 @@ +// crates/core/src/expression/mod.rs +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors `org.apache.hudi.expression`. +//! +//! Full port of the Predicate / Expression hierarchy. Includes a +//! `kind() -> {Predicate,Expression}Kind<'_>` accessor on each trait +//! to give downstream Rust code idiomatic pattern-matching instead +//! of `Any::downcast_ref`. See the keyFilterOpt design spec §4.1. + +pub mod array_data; +pub mod binary_expression; +pub mod bind_visitor; +pub mod bound_reference; +pub mod comparators; +pub mod expression; +pub mod expression_visitor; +pub mod leaf_expression; +pub mod literal; +pub mod name_reference; +pub mod partial_bind_visitor; +pub mod predicate; +pub mod predicates; +pub mod struct_like; + +// The following re-exports are commented out until each item is introduced +// in subsequent tasks. Uncomment as each task lands: +pub use expression::{Expression, ExpressionKind, Operator}; +pub use literal::{Literal, LiteralValue}; +pub use name_reference::NameReference; // Task 1.9 +pub use predicate::{Predicate, PredicateKind}; // Task 1.11 diff --git a/crates/core/src/expression/name_reference.rs b/crates/core/src/expression/name_reference.rs new file mode 100644 index 00000000..0663e0ad --- /dev/null +++ b/crates/core/src/expression/name_reference.rs @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.NameReference`. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::Type; +use crate::internal_schema::types::StringType; + +/// Reference to a column by name. Mirrors Java `NameReference(String name)`. +/// +/// Java's `NameReference` has `getDataType()` returning a placeholder +/// `Types.StringType` since the actual type is resolved by `BindVisitor`. +/// Rust mirrors this default. +#[derive(Debug, Clone)] +pub struct NameReference { + name: String, +} + +impl NameReference { + pub fn new(name: impl Into) -> Self { + Self { name: name.into() } + } + + pub fn name(&self) -> &str { + &self.name + } +} + +impl Expression for NameReference { + fn data_type(&self) -> &dyn Type { + // Java returns Types.StringType.get() as a placeholder until + // BindVisitor resolves the real type. Rust mirrors via a + // singleton StringType reference. + static TYPE: StringType = StringType; + &TYPE + } + + fn operator(&self) -> Operator { + Operator::True + } + + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::NameReference(self) + } +} + +impl LeafExpression for NameReference {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, ExpressionKind}; + + #[test] + fn name_reference_carries_name() { + let nr = NameReference::new("_hoodie_record_key"); + assert_eq!(nr.name(), "_hoodie_record_key"); + } + + #[test] + fn name_reference_kind_pattern_match() { + let nr = NameReference::new("foo"); + match nr.kind() { + ExpressionKind::NameReference(_) => {} + _ => panic!("expected ExpressionKind::NameReference"), + } + } +} diff --git a/crates/core/src/expression/partial_bind_visitor.rs b/crates/core/src/expression/partial_bind_visitor.rs new file mode 100644 index 00000000..70060807 --- /dev/null +++ b/crates/core/src/expression/partial_bind_visitor.rs @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.PartialBindVisitor`. +//! +//! Like BindVisitor but tolerates unresolvable NameReferences. Ported as +//! placeholder; not wired. Spec §6 deviation #6. + +use crate::expression::expression_visitor::ExpressionVisitor; +use crate::internal_schema::types::RecordType; + +pub struct PartialBindVisitor { + pub schema: RecordType, + pub case_sensitive: bool, +} + +impl PartialBindVisitor { + pub fn new(schema: RecordType, case_sensitive: bool) -> Self { + Self { + schema, + case_sensitive, + } + } +} + +impl ExpressionVisitor for PartialBindVisitor {} diff --git a/crates/core/src/expression/predicate.rs b/crates/core/src/expression/predicate.rs new file mode 100644 index 00000000..e6496d4c --- /dev/null +++ b/crates/core/src/expression/predicate.rs @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.Predicate`. + +use crate::expression::Expression; + +/// Mirrors Java `interface Predicate extends Expression`. +/// +/// Adds a `pred_kind() -> PredicateKind<'_>` accessor for idiomatic Rust +/// pattern matching. Note this is named `pred_kind` (not `kind`) to avoid +/// collision with the `Expression::kind()` supertrait method when called +/// through a `dyn Predicate` trait object. See keyFilterOpt design spec +/// §4.1, deviation #2. +pub trait Predicate: Expression { + /// Borrowed enum view for pattern matching. + fn pred_kind(&self) -> PredicateKind<'_>; +} + +/// Borrowed enum view of `Predicate` for pattern matching. +pub enum PredicateKind<'a> { + True, + False, + And(&'a crate::expression::predicates::And), + Or(&'a crate::expression::predicates::Or), + Not(&'a crate::expression::predicates::Not), + BinaryComparison(&'a crate::expression::predicates::BinaryComparison), + In(&'a crate::expression::predicates::In), + IsNull(&'a crate::expression::predicates::IsNull), + IsNotNull(&'a crate::expression::predicates::IsNotNull), + StringStartsWith(&'a crate::expression::predicates::StringStartsWith), + StringStartsWithAny(&'a crate::expression::predicates::StringStartsWithAny), + StringContains(&'a crate::expression::predicates::StringContains), +} diff --git a/crates/core/src/expression/predicates.rs b/crates/core/src/expression/predicates.rs new file mode 100644 index 00000000..fa7d9a32 --- /dev/null +++ b/crates/core/src/expression/predicates.rs @@ -0,0 +1,733 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.Predicates`. +//! +//! All 12 inner classes from Java's `Predicates` live in this single file +//! to mirror Java's structure. Factory functions at the bottom mirror the +//! static methods on Java's `Predicates`. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::predicate::{Predicate, PredicateKind}; +use crate::expression::struct_like::StructLike; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::Type; +use crate::internal_schema::types::BooleanType; +use std::any::Any; + +// ========================================================================= +// Singleton predicates +// ========================================================================= + +/// Mirrors Java `Predicates.TrueExpression`. +#[derive(Debug, Clone, Copy)] +pub struct TrueExpression; + +impl Expression for TrueExpression { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::True + } + fn eval(&self, _data: Option<&dyn StructLike>) -> Option> { + Some(Box::new(true)) + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl LeafExpression for TrueExpression {} + +impl Predicate for TrueExpression { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::True + } +} + +/// Mirrors Java `Predicates.FalseExpression`. +#[derive(Debug, Clone, Copy)] +pub struct FalseExpression; + +impl Expression for FalseExpression { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::False + } + fn eval(&self, _data: Option<&dyn StructLike>) -> Option> { + Some(Box::new(false)) + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl LeafExpression for FalseExpression {} + +impl Predicate for FalseExpression { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::False + } +} + +// ========================================================================= +// Boolean combinators +// ========================================================================= + +/// Mirrors Java `Predicates.And`. +#[derive(Debug)] +pub struct And { + pub left: Box, + pub right: Box, +} + +impl And { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right(&self) -> &dyn Expression { + self.right.as_ref() + } +} + +impl Expression for And { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::And + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.left.as_ref(), self.right.as_ref()] + } + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let left = self.left.eval(data)?; + let lb = left.downcast_ref::().copied().unwrap_or(false); + if !lb { + return Some(Box::new(false)); + } + let right = self.right.eval(data)?; + let rb = right.downcast_ref::().copied().unwrap_or(false); + Some(Box::new(lb && rb)) + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for And { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::And(self) + } +} + +/// Mirrors Java `Predicates.Or`. +#[derive(Debug)] +pub struct Or { + pub left: Box, + pub right: Box, +} + +impl Or { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right(&self) -> &dyn Expression { + self.right.as_ref() + } +} + +impl Expression for Or { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::Or + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.left.as_ref(), self.right.as_ref()] + } + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let left = self.left.eval(data)?; + let lb = left.downcast_ref::().copied().unwrap_or(false); + if lb { + return Some(Box::new(true)); + } + let right = self.right.eval(data)?; + let rb = right.downcast_ref::().copied().unwrap_or(false); + Some(Box::new(lb || rb)) + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for Or { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::Or(self) + } +} + +/// Mirrors Java `Predicates.Not`. +#[derive(Debug)] +pub struct Not { + pub child: Box, +} + +impl Not { + pub fn new(child: Box) -> Self { + Self { child } + } + pub fn child(&self) -> &dyn Expression { + self.child.as_ref() + } +} + +impl Expression for Not { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::Not + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.child.as_ref()] + } + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let v = self.child.eval(data)?; + let b = v.downcast_ref::().copied().unwrap_or(false); + Some(Box::new(!b)) + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for Not { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::Not(self) + } +} + +// ========================================================================= +// Comparison + null + string predicates +// ========================================================================= + +/// Mirrors Java `Predicates.BinaryComparison`. +#[derive(Debug)] +pub struct BinaryComparison { + pub left: Box, + pub op: Operator, + pub right: Box, +} + +impl BinaryComparison { + pub fn new(left: Box, op: Operator, right: Box) -> Self { + Self { left, op, right } + } + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right(&self) -> &dyn Expression { + self.right.as_ref() + } +} + +impl Expression for BinaryComparison { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + self.op + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.left.as_ref(), self.right.as_ref()] + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for BinaryComparison { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::BinaryComparison(self) + } +} + +/// Mirrors Java `Predicates.In`. +#[derive(Debug)] +pub struct In { + pub value: Box, + pub valid_values: Vec>, +} + +impl In { + pub fn new(value: Box, valid_values: Vec>) -> Self { + Self { + value, + valid_values, + } + } + pub fn value(&self) -> &dyn Expression { + self.value.as_ref() + } + pub fn right_children(&self) -> &[Box] { + &self.valid_values + } +} + +impl Expression for In { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::In + } + fn children(&self) -> Vec<&dyn Expression> { + let mut all = vec![self.value.as_ref()]; + all.extend(self.valid_values.iter().map(|e| e.as_ref())); + all + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for In { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::In(self) + } +} + +/// Mirrors Java `Predicates.IsNull`. +#[derive(Debug)] +pub struct IsNull { + pub child: Box, +} + +impl IsNull { + pub fn new(child: Box) -> Self { + Self { child } + } + pub fn child(&self) -> &dyn Expression { + self.child.as_ref() + } +} + +impl Expression for IsNull { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::IsNull + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.child.as_ref()] + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for IsNull { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::IsNull(self) + } +} + +/// Mirrors Java `Predicates.IsNotNull`. +#[derive(Debug)] +pub struct IsNotNull { + pub child: Box, +} + +impl IsNotNull { + pub fn new(child: Box) -> Self { + Self { child } + } + pub fn child(&self) -> &dyn Expression { + self.child.as_ref() + } +} + +impl Expression for IsNotNull { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::IsNotNull + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.child.as_ref()] + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for IsNotNull { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::IsNotNull(self) + } +} + +/// Mirrors Java `Predicates.StringStartsWith`. +#[derive(Debug)] +pub struct StringStartsWith { + pub left: Box, + pub right: Box, +} + +impl StringStartsWith { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right(&self) -> &dyn Expression { + self.right.as_ref() + } +} + +impl Expression for StringStartsWith { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::StartsWith + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.left.as_ref(), self.right.as_ref()] + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for StringStartsWith { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::StringStartsWith(self) + } +} + +/// Mirrors Java `Predicates.StringStartsWithAny`. +#[derive(Debug)] +pub struct StringStartsWithAny { + pub left: Box, + pub right: Vec>, +} + +impl StringStartsWithAny { + pub fn new(left: Box, right: Vec>) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right_children(&self) -> &[Box] { + &self.right + } +} + +impl Expression for StringStartsWithAny { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::StartsWith + } + fn children(&self) -> Vec<&dyn Expression> { + let mut all = vec![self.left.as_ref()]; + all.extend(self.right.iter().map(|e| e.as_ref())); + all + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for StringStartsWithAny { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::StringStartsWithAny(self) + } +} + +/// Mirrors Java `Predicates.StringContains`. +#[derive(Debug)] +pub struct StringContains { + pub left: Box, + pub right: Box, +} + +impl StringContains { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { + self.left.as_ref() + } + pub fn right(&self) -> &dyn Expression { + self.right.as_ref() + } +} + +impl Expression for StringContains { + fn data_type(&self) -> &dyn Type { + static T: BooleanType = BooleanType; + &T + } + fn operator(&self) -> Operator { + Operator::Contains + } + fn children(&self) -> Vec<&dyn Expression> { + vec![self.left.as_ref(), self.right.as_ref()] + } + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Predicate(self) + } +} + +impl Predicate for StringContains { + fn pred_kind(&self) -> PredicateKind<'_> { + PredicateKind::StringContains(self) + } +} + +// ========================================================================= +// Factory functions — mirror Java `Predicates` static methods. +// +// `in` is a Rust keyword so the function is named `in_`. +// ========================================================================= + +pub mod predicates_factory { + use super::*; + + pub fn always_true() -> TrueExpression { + TrueExpression + } + pub fn always_false() -> FalseExpression { + FalseExpression + } + pub fn and(left: Box, right: Box) -> And { + And::new(left, right) + } + pub fn or(left: Box, right: Box) -> Or { + Or::new(left, right) + } + pub fn not(child: Box) -> Not { + Not::new(child) + } + pub fn gt(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::Gt, right) + } + pub fn lt(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::Lt, right) + } + pub fn eq(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::Eq, right) + } + pub fn gteq(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::GtEq, right) + } + pub fn lteq(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::LtEq, right) + } + pub fn starts_with(left: Box, right: Box) -> StringStartsWith { + StringStartsWith::new(left, right) + } + pub fn contains(left: Box, right: Box) -> StringContains { + StringContains::new(left, right) + } + pub fn in_(value: Box, valid_values: Vec>) -> In { + In::new(value, valid_values) + } + pub fn is_null(child: Box) -> IsNull { + IsNull::new(child) + } + pub fn is_not_null(child: Box) -> IsNotNull { + IsNotNull::new(child) + } + pub fn starts_with_any( + left: Box, + right: Vec>, + ) -> StringStartsWithAny { + StringStartsWithAny::new(left, right) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::Literal; + use crate::expression::{Expression, Operator, Predicate}; + + #[test] + fn true_expression_eval_returns_true() { + let t = TrueExpression; + let result = t.eval(None).unwrap(); + let bool_val = result.downcast_ref::().expect("expected bool"); + assert!(bool_val); + } + + #[test] + fn true_expression_operator_is_true() { + assert_eq!(TrueExpression.operator(), Operator::True); + } + + #[test] + fn false_expression_eval_returns_false() { + let f = FalseExpression; + let result = f.eval(None).unwrap(); + let bool_val = result.downcast_ref::().expect("expected bool"); + assert!(!bool_val); + } + + #[test] + fn false_expression_operator_is_false() { + assert_eq!(FalseExpression.operator(), Operator::False); + } + + #[test] + fn and_constructs_and_carries_children() { + let a = And::new(Box::new(TrueExpression), Box::new(FalseExpression)); + assert_eq!(a.operator(), Operator::And); + assert_eq!(a.children().len(), 2); + } + + #[test] + fn or_constructs_and_carries_children() { + let o = Or::new(Box::new(TrueExpression), Box::new(FalseExpression)); + assert_eq!(o.operator(), Operator::Or); + assert_eq!(o.children().len(), 2); + } + + #[test] + fn not_carries_child() { + let n = Not::new(Box::new(TrueExpression)); + assert_eq!(n.operator(), Operator::Not); + assert_eq!(n.children().len(), 1); + } + + #[test] + fn and_eval_short_circuits_false() { + let a = And::new(Box::new(FalseExpression), Box::new(TrueExpression)); + let result = a.eval(None).unwrap(); + let b = result.downcast_ref::().unwrap(); + assert!(!b); + } + + #[test] + fn binary_comparison_eq() { + let bc = BinaryComparison::new( + Box::new(Literal::int(5)), + Operator::Eq, + Box::new(Literal::int(5)), + ); + assert_eq!(bc.operator(), Operator::Eq); + } + + #[test] + fn in_predicate_carries_children() { + let val = Box::new(Literal::string("col")); + let validvalues = vec![ + Box::new(Literal::string("a")) as Box, + Box::new(Literal::string("b")) as Box, + ]; + let p = In::new(val, validvalues); + assert_eq!(p.operator(), Operator::In); + assert_eq!(p.right_children().len(), 2); + assert_eq!(p.children().len(), 3); + } + + #[test] + fn is_null_carries_child() { + let p = IsNull::new(Box::new(Literal::int(1))); + assert_eq!(p.operator(), Operator::IsNull); + } + + #[test] + fn is_not_null_carries_child() { + let p = IsNotNull::new(Box::new(Literal::int(1))); + assert_eq!(p.operator(), Operator::IsNotNull); + } + + #[test] + fn string_starts_with_carries_left_right() { + let p = StringStartsWith::new( + Box::new(Literal::string("name")), + Box::new(Literal::string("Al")), + ); + assert_eq!(p.operator(), Operator::StartsWith); + } + + #[test] + fn string_starts_with_any_carries_children() { + let p = StringStartsWithAny::new( + Box::new(Literal::string("name")), + vec![ + Box::new(Literal::string("Al")) as Box, + Box::new(Literal::string("Bo")) as Box, + ], + ); + assert_eq!(p.operator(), Operator::StartsWith); + assert_eq!(p.right_children().len(), 2); + } + + #[test] + fn string_contains_constructs() { + let p = StringContains::new( + Box::new(Literal::string("desc")), + Box::new(Literal::string("foo")), + ); + assert_eq!(p.operator(), Operator::Contains); + } + + #[test] + fn factory_in_constructs_in() { + let p = predicates_factory::in_( + Box::new(Literal::string("col")), + vec![Box::new(Literal::string("a"))], + ); + assert_eq!(p.operator(), Operator::In); + } + + #[test] + fn factory_starts_with_any_constructs_string_starts_with_any() { + let p = predicates_factory::starts_with_any( + Box::new(Literal::string("col")), + vec![Box::new(Literal::string("a"))], + ); + assert_eq!(p.operator(), Operator::StartsWith); + } +} diff --git a/crates/core/src/expression/struct_like.rs b/crates/core/src/expression/struct_like.rs new file mode 100644 index 00000000..58aca7b5 --- /dev/null +++ b/crates/core/src/expression/struct_like.rs @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.expression.StructLike`. + +use std::any::Any; + +/// Read-only view over a record-like value. Mirrors Java's `StructLike`. +/// +/// Java has `int numFields()` + ` T get(int pos, Class javaClass)`. +/// Rust uses `&dyn Any` so callers can downcast. +pub trait StructLike: Send + Sync { + fn num_fields(&self) -> usize; + fn get(&self, pos: usize) -> Option<&dyn Any>; +} + +#[cfg(test)] +mod tests { + use super::*; + + struct MockStruct(Vec>>); + + impl StructLike for MockStruct { + fn num_fields(&self) -> usize { + self.0.len() + } + fn get(&self, pos: usize) -> Option<&dyn std::any::Any> { + self.0[pos].as_deref().map(|v| v as &dyn std::any::Any) + } + } + + #[test] + fn struct_like_basic() { + let s = MockStruct(vec![ + Some(Box::new(42i32)), + Some(Box::new("hello".to_string())), + None, + ]); + assert_eq!(s.num_fields(), 3); + assert!(s.get(2).is_none()); + } +} diff --git a/crates/core/src/file_group/builder.rs b/crates/core/src/file_group/builder.rs index 21fa65b0..402420d0 100644 --- a/crates/core/src/file_group/builder.rs +++ b/crates/core/src/file_group/builder.rs @@ -214,8 +214,6 @@ pub fn file_groups_from_files_partition_records( } // Build FileGroups from parsed files - // Note: Currently only supports file groups with base files. - // TODO: Support file groups with only log files (P1 task) let mut file_groups = Vec::new(); for (file_id, base_files) in file_id_to_base_files { let mut fg = FileGroup::new(file_id.clone(), partition_path.clone()); @@ -229,6 +227,14 @@ pub fn file_groups_from_files_partition_records( file_groups.push(fg); } + // Build FileGroups for log-only file groups (no base files). + // Remaining entries in file_id_to_log_files are file_ids NOT in file_id_to_base_files. + for (file_id, log_files) in file_id_to_log_files { + let mut fg = FileGroup::new(file_id.clone(), partition_path.clone()); + fg.add_log_files(log_files)?; + file_groups.push(fg); + } + if !file_groups.is_empty() { file_groups_map.insert(partition_path.clone(), file_groups); } @@ -502,7 +508,7 @@ mod tests { assert_eq!(file_group.file_slices.len(), 1); let (_, file_slice) = file_group.file_slices.iter().next().unwrap(); assert_eq!( - file_slice.base_file.file_name(), + file_slice.base_file.as_ref().unwrap().file_name(), "file-id-0_0-7-24_20240418173200000.parquet" ); assert_eq!(file_slice.log_files.len(), 2); @@ -586,7 +592,7 @@ mod tests { let file_group = file_groups.iter().next().unwrap(); let file_slice = file_group.file_slices.values().next().unwrap(); assert_eq!( - file_slice.base_file.completion_timestamp, + file_slice.base_file.as_ref().unwrap().completion_timestamp, Some("20240418173210000".to_string()) ); } @@ -976,7 +982,7 @@ mod tests { // Verify completion timestamp was set let file_slice = file_groups[0].file_slices.values().next().unwrap(); assert_eq!( - file_slice.base_file.completion_timestamp, + file_slice.base_file.as_ref().unwrap().completion_timestamp, Some("20240418173210000".to_string()) ); } @@ -1115,7 +1121,7 @@ mod tests { } #[test] - fn test_log_files_without_base_file_not_included() { + fn test_log_files_without_base_file_creates_log_only_group() { let mut records = HashMap::new(); // Only log files, no base file let (key, record) = create_files_record( @@ -1135,8 +1141,17 @@ mod tests { assert!(result.is_ok()); let file_groups_map = result.unwrap(); - // Log-only file groups are not yet supported (see P1 task) - assert!(file_groups_map.is_empty()); + // Log-only file groups are now supported + assert_eq!(file_groups_map.len(), 1); + let file_groups = file_groups_map.get("partition1").unwrap(); + assert_eq!(file_groups.len(), 1); + let fg = &file_groups[0]; + assert_eq!(fg.file_id, "file-id-0"); + // The file group should have a single log-only file slice + assert_eq!(fg.file_slices.len(), 1); + let slice = fg.file_slices.values().next().unwrap(); + assert!(slice.base_file.is_none()); + assert_eq!(slice.log_files.len(), 2); } #[test] diff --git a/crates/core/src/file_group/file_slice.rs b/crates/core/src/file_group/file_slice.rs index 42f6c0da..5984b978 100644 --- a/crates/core/src/file_group/file_slice.rs +++ b/crates/core/src/file_group/file_slice.rs @@ -27,27 +27,49 @@ use std::fmt::Display; use std::path::PathBuf; /// Within a [crate::file_group::FileGroup], -/// a [FileSlice] is a logical group of [BaseFile] and [LogFile]s. +/// a [FileSlice] is a logical group of an optional [BaseFile] and [LogFile]s. +/// +/// Mirrors Java's `FileSlice` which has separate `fileGroupId`, `baseInstantTime`, +/// and nullable `baseFile` fields. +/// +/// A log-only file slice (no base file) is created when a file group has only log files. #[derive(Clone, Debug)] pub struct FileSlice { - pub base_file: BaseFile, + /// The file group id this slice belongs to. + /// Mirrors Java's `FileSlice.fileGroupId`. + pub file_id: String, + /// The instant time that marks this slice's creation. + /// Mirrors Java's `FileSlice.baseInstantTime`. + pub base_instant_time: String, + /// The base file, if present. None for log-only file slices. + /// Mirrors Java's nullable `FileSlice.baseFile`. + pub base_file: Option, pub log_files: BTreeSet, pub partition_path: String, } impl Display for FileSlice { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "FileSlice {{ base_file: {}, log_files: {:?}, partition_path: {} }}", - self.base_file, self.log_files, self.partition_path - ) + match &self.base_file { + Some(bf) => write!( + f, + "FileSlice {{ base_file: {}, log_files: {:?}, partition_path: {} }}", + bf, self.log_files, self.partition_path + ), + None => write!( + f, + "FileSlice {{ , file_id: {}, base_instant_time: {}, log_files: {:?}, partition_path: {} }}", + self.file_id, self.base_instant_time, self.log_files, self.partition_path + ), + } } } impl PartialEq for FileSlice { fn eq(&self, other: &Self) -> bool { - self.base_file == other.base_file && self.partition_path == other.partition_path + self.file_id == other.file_id + && self.base_instant_time == other.base_instant_time + && self.partition_path == other.partition_path } } @@ -55,8 +77,30 @@ impl Eq for FileSlice {} impl FileSlice { pub fn new(base_file: BaseFile, partition_path: String) -> Self { + let file_id = base_file.file_id.clone(); + let base_instant_time = base_file.commit_timestamp.clone(); + Self { + file_id, + base_instant_time, + base_file: Some(base_file), + log_files: BTreeSet::new(), + partition_path, + } + } + + /// Create a log-only file slice (no base file). + /// + /// Mirrors Java's `new FileSlice(fileGroupId, baseInstantTime)` where `baseFile` stays null. + /// Used when a file group has only log files and no base file. + pub fn new_log_only( + file_id: String, + base_instant_time: String, + partition_path: String, + ) -> Self { Self { - base_file, + file_id, + base_instant_time, + base_file: None, log_files: BTreeSet::new(), partition_path, } @@ -85,10 +129,14 @@ impl FileSlice { }) } - /// Returns the relative path of the [BaseFile] in the [FileSlice]. - pub fn base_file_relative_path(&self) -> Result { - let file_name = &self.base_file.file_name(); - self.relative_path_for_file(file_name) + /// Returns the relative path of the [BaseFile] in the [FileSlice], or `None` for log-only slices. + /// + /// Mirrors Java's `FileSlice.getBaseFile()` returning `Option.ofNullable(baseFile)`. + pub fn base_file_relative_path(&self) -> Result> { + match &self.base_file { + Some(bf) => self.relative_path_for_file(&bf.file_name()).map(Some), + None => Ok(None), + } } /// Returns the relative path of the given [LogFile] in the [FileSlice]. @@ -100,7 +148,7 @@ impl FileSlice { /// Returns the enclosing [FileGroup]'s id. #[inline] pub fn file_id(&self) -> &str { - &self.base_file.file_id + &self.file_id } /// Returns the instant time that marks the [FileSlice] creation. @@ -108,7 +156,7 @@ impl FileSlice { /// This is also an instant time stored in the [Timeline]. #[inline] pub fn creation_instant_time(&self) -> &str { - &self.base_file.commit_timestamp + &self.base_instant_time } /// Load [FileMetadata] from storage layer for the [BaseFile] if `file_metadata` is [None] @@ -118,20 +166,27 @@ impl FileSlice { /// this is a no-op since Parquet-specific metadata reading would fail. /// TODO: see if mdt read would benefit from loading hfile metadata as well. pub async fn load_metadata_if_needed(&mut self, storage: &Storage) -> Result<()> { + let bf = match &self.base_file { + Some(bf) => bf, + None => return Ok(()), // log-only slice: no base file metadata to load + }; + // Skip non-Parquet files - metadata loading uses Parquet-specific APIs - if self.base_file.extension != BaseFileFormatValue::Parquet.as_ref() { + if bf.extension != BaseFileFormatValue::Parquet.as_ref() { return Ok(()); } - if let Some(metadata) = &self.base_file.file_metadata { + if let Some(metadata) = &bf.file_metadata { if metadata.fully_populated { return Ok(()); } } - let relative_path = self.base_file_relative_path()?; + let file_name = bf.file_name(); + let relative_path = self.relative_path_for_file(&file_name)?; let fetched_metadata = storage.get_file_metadata(&relative_path).await?; - self.base_file.file_metadata = Some(fetched_metadata); + // Re-borrow mutably after immutable borrows are done + self.base_file.as_mut().unwrap().file_metadata = Some(fetched_metadata); Ok(()) } } @@ -167,13 +222,17 @@ mod tests { )?); let mut slice1 = FileSlice { - base_file: base.clone(), + file_id: base.file_id.clone(), + base_instant_time: base.commit_timestamp.clone(), + base_file: Some(base.clone()), log_files: log_set1, partition_path: EMPTY_PARTITION_PATH.to_string(), }; let slice2 = FileSlice { - base_file: base, + file_id: base.file_id.clone(), + base_instant_time: base.commit_timestamp.clone(), + base_file: Some(base), log_files: log_set2, partition_path: EMPTY_PARTITION_PATH.to_string(), }; @@ -202,23 +261,16 @@ mod tests { #[test] fn test_merge_different_base_files() -> Result<()> { - let mut slice1 = FileSlice { - base_file: BaseFile::from_str( - "54e9a5e9-ee5d-4ed2-acee-720b5810d380-0_0-7-24_20250109233025121.parquet", - )?, - log_files: BTreeSet::new(), - partition_path: EMPTY_PARTITION_PATH.to_string(), - }; - - let slice2 = FileSlice { - base_file: BaseFile::from_str( - "54e9a5e9-ee5d-4ed2-acee-720b5810d380-0_1-19-51_20250109233025121.parquet", - )?, - log_files: BTreeSet::new(), - partition_path: EMPTY_PARTITION_PATH.to_string(), - }; + let base1 = BaseFile::from_str( + "54e9a5e9-ee5d-4ed2-acee-720b5810d380-0_0-7-24_20250109233025121.parquet", + )?; + let base2 = BaseFile::from_str( + "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-0_1-19-51_20250109233025121.parquet", + )?; + let mut slice1 = FileSlice::new(base1, EMPTY_PARTITION_PATH.to_string()); + let slice2 = FileSlice::new(base2, EMPTY_PARTITION_PATH.to_string()); - // Should return error for different base files + // Should return error for different file slices (different file_id) assert!(slice1.merge(&slice2).is_err()); Ok(()) @@ -229,21 +281,34 @@ mod tests { let base = BaseFile::from_str( "54e9a5e9-ee5d-4ed2-acee-720b5810d380-0_1-19-51_20250109233025121.parquet", )?; - let mut slice1 = FileSlice { - base_file: base.clone(), - log_files: BTreeSet::new(), - partition_path: "path/to/partition1".to_string(), - }; - - let slice2 = FileSlice { - base_file: base, - log_files: BTreeSet::new(), - partition_path: "path/to/partition2".to_string(), - }; + let mut slice1 = FileSlice::new(base.clone(), "path/to/partition1".to_string()); + let slice2 = FileSlice::new(base, "path/to/partition2".to_string()); // Should return error for different partition paths assert!(slice1.merge(&slice2).is_err()); Ok(()) } + + #[test] + fn test_log_only_file_slice() -> Result<()> { + let mut slice = FileSlice::new_log_only( + "file-id-001".to_string(), + "20250109233025121".to_string(), + EMPTY_PARTITION_PATH.to_string(), + ); + assert_eq!(slice.file_id(), "file-id-001"); + assert_eq!(slice.creation_instant_time(), "20250109233025121"); + assert!(slice.base_file.is_none()); + assert!(!slice.has_log_file()); + assert_eq!(slice.base_file_relative_path()?, None); + + // Add a log file + slice.log_files.insert(LogFile::from_str( + ".file-id-001_20250109233025121.log.1_0-51-115", + )?); + assert!(slice.has_log_file()); + + Ok(()) + } } diff --git a/crates/core/src/file_group/log_file/content.rs b/crates/core/src/file_group/log_file/content.rs index 440d2842..f157eef4 100644 --- a/crates/core/src/file_group/log_file/content.rs +++ b/crates/core/src/file_group/log_file/content.rs @@ -18,13 +18,12 @@ */ use crate::Result; use crate::avro_to_arrow::arrow_array_reader::AvroArrowArrayReader; -use crate::config::HudiConfigs; use crate::error::CoreError; use crate::file_group::log_file::avro::AvroDataBlockContentReader; use crate::file_group::log_file::log_block::{ BlockMetadataKey, BlockType, LogBlockContent, LogBlockVersion, }; -use crate::file_group::log_file::log_format::LogFormatVersion; +use crate::file_group::reader::reader_context::ReaderContext; use crate::file_group::record_batches::RecordBatches; use crate::hfile::{HFileReader, HFileRecord}; use crate::schema::delete::{avro_schema_for_delete_record, avro_schema_for_delete_record_list}; @@ -38,32 +37,27 @@ use std::sync::Arc; #[allow(dead_code)] pub struct Decoder { batch_size: usize, - hudi_configs: Arc, + reader_context: Arc, } impl Decoder { - pub fn new(hudi_configs: Arc) -> Self { + pub fn new(reader_context: Arc) -> Self { Self { batch_size: 1024, - hudi_configs, + reader_context, } } + /// Decode block content from a reader already positioned at the start of the raw content. + /// + /// `content_length` is the true content-only byte count (matching Java's `contentLength`), + /// read by the caller before invoking this method. pub fn decode_content( &self, reader: &mut (impl Read + Seek), - log_format_version: &LogFormatVersion, - fallback_length: u64, + content_length: u64, block_type: &BlockType, header: &HashMap, ) -> Result { - let content_length = if log_format_version.has_content_length() { - let mut content_length_buf = [0u8; 8]; - reader.read_exact(&mut content_length_buf)?; - u64::from_be_bytes(content_length_buf) - } else { - fallback_length - }; - let reader = reader.by_ref().take(content_length); match block_type { BlockType::AvroData => self @@ -87,7 +81,7 @@ impl Decoder { /// Validate the log block version (first 4 bytes of block content). /// - /// This is NOT the same as [`LogFormatVersion`] (read from the file header). + /// This is NOT the same as `LogFormatVersion` (read from the file header). /// Modern Hudi tables use [`LogBlockVersion::V3`]. fn validate_log_block_version(mut reader: impl Read) -> Result<()> { let mut version_buf = [0u8; 4]; @@ -326,8 +320,7 @@ mod tests { write_record(&mut buf, record2)?; // Create decoder and test - let hudi_configs = HudiConfigs::empty(); - let decoder = Decoder::new(Arc::new(hudi_configs)); + let decoder = Decoder::new(Arc::new(ReaderContext::empty())); let reader = Cursor::new(buf); let mut header = HashMap::new(); @@ -383,8 +376,7 @@ mod tests { writer.close()?; } - let hudi_configs = HudiConfigs::empty(); - let decoder = Decoder::new(Arc::new(hudi_configs)); + let decoder = Decoder::new(Arc::new(ReaderContext::empty())); let bytes = Bytes::from(buf); let mut reader = BufReader::with_capacity(bytes.len(), Cursor::new(bytes)); diff --git a/crates/core/src/file_group/log_file/log_block.rs b/crates/core/src/file_group/log_file/log_block.rs index aa0dc794..4c077ae9 100644 --- a/crates/core/src/file_group/log_file/log_block.rs +++ b/crates/core/src/file_group/log_file/log_block.rs @@ -19,11 +19,16 @@ use crate::Result; use crate::error::CoreError; +use crate::file_group::log_file::content::Decoder; use crate::file_group::log_file::log_format::LogFormatVersion; +use crate::file_group::reader::reader_context::ReaderContext; use crate::file_group::record_batches::RecordBatches; use crate::hfile::HFileRecord; +use bytes::Bytes; use std::collections::HashMap; +use std::io::{Cursor, Seek, SeekFrom}; use std::str::FromStr; +use std::sync::Arc; /// Internal block content version. /// @@ -275,6 +280,23 @@ impl LogBlockContent { } } +/// Location of a log block's content on storage. +/// +/// Mirrors Java's `HoodieLogBlock.blockContentLocation`. Used for lazy +/// content loading: Pass 1 records this location without reading the bytes, +/// then `inflate()` in Pass 3 seeks back and decodes on demand. +#[derive(Debug, Clone)] +pub struct LogBlockContentLocation { + /// Relative path to the log file. + pub log_file_path: String, + /// Byte offset where the content starts within the log file. + pub content_position: u64, + /// Length of the content in bytes. + pub content_length: u64, + /// The full block length (used for seeking past footer/total-length). + pub block_length: u64, +} + #[derive(Debug, Clone)] pub struct LogBlock { pub format_version: LogFormatVersion, @@ -283,10 +305,17 @@ pub struct LogBlock { pub content: LogBlockContent, pub footer: HashMap, pub skipped: bool, + /// Content location for lazy inflate. When set, content starts as `Empty` + /// and is loaded on demand via `inflate_from_bytes()`. + pub content_location: Option, + /// Pre-fetched file bytes for sync inflate. Set during Pass 1 when the + /// log file is already in memory. `Bytes` is ref-counted — cloning is O(1). + /// Mirrors Java's `inputStreamSupplier` on `HoodieLogBlock`. + pub source_bytes: Option, } impl LogBlock { - /// Create a new log block with the given content. + /// Create a new log block with the given content (eager / fully decoded). pub fn new( format_version: LogFormatVersion, block_type: BlockType, @@ -301,6 +330,31 @@ impl LogBlock { content, footer, skipped: false, + content_location: None, + source_bytes: None, + } + } + + /// Create a metadata-only log block for lazy content loading. + /// + /// Mirrors Java's block construction during Pass 1 of `scanInternal`: + /// only headers are read, content bytes are skipped. The `content_location` + /// records where to seek for later `inflate()`. + pub fn new_lazy( + format_version: LogFormatVersion, + block_type: BlockType, + header: HashMap, + content_location: LogBlockContentLocation, + ) -> Self { + Self { + format_version, + block_type, + header, + content: LogBlockContent::Empty, + footer: HashMap::new(), + skipped: false, + content_location: Some(content_location), + source_bytes: None, } } @@ -319,9 +373,74 @@ impl LogBlock { content: LogBlockContent::Empty, footer: HashMap::new(), skipped: true, + content_location: None, + source_bytes: None, } } + /// Set pre-fetched file bytes for sync inflate. + /// + /// Called during Pass 1 when the log file is already in memory. + /// `Bytes` is ref-counted — cloning is O(1). + pub fn set_source_bytes(&mut self, bytes: Bytes) { + self.source_bytes = Some(bytes); + } + + /// Decode block content from pre-fetched file bytes (sync). + /// + /// Mirrors Java's `HoodieDataBlock.readRecordsFromBlockPayload()`: + /// `inflate()` → `deserializeRecords()` → `deflate()`. + /// + /// No-op if content is already populated (e.g., eagerly decoded blocks + /// or blocks already inflated). + pub fn inflate_from_bytes(&mut self, reader_context: Arc) -> Result<()> { + if !self.content.is_empty() { + return Ok(()); + } + + let loc = self.content_location.as_ref().ok_or_else(|| { + CoreError::LogBlockError("No content location for inflate".to_string()) + })?; + + let bytes = self + .source_bytes + .as_ref() + .ok_or_else(|| CoreError::LogBlockError("No source bytes for inflate".to_string()))?; + + log::debug!( + "[LogBlock::inflate_from_bytes] type={:?} file={} pos={} len={} block_len={}", + self.block_type, + loc.log_file_path, + loc.content_position, + loc.content_length, + loc.block_length, + ); + + let mut cursor = Cursor::new(bytes.clone()); + cursor.seek(SeekFrom::Start(loc.content_position))?; + + let decoder = Decoder::new(reader_context); + self.content = decoder.decode_content( + &mut cursor, + loc.content_length, + &self.block_type, + &self.header, + )?; + + // Release raw bytes — mirrors Java's deflate() releasing the byte[] + self.source_bytes = None; + + Ok(()) + } + + /// Release block content (lazy deflate). + /// + /// Mirrors Java's `HoodieLogBlock.deflate()`. After calling this, the content + /// is `Empty` and the memory used by decoded `RecordBatch`es is freed. + pub fn deflate(&mut self) { + self.content = LogBlockContent::Empty; + } + /// Returns the record batches if the content contains Arrow records. /// /// This is a convenience method for backwards compatibility. diff --git a/crates/core/src/file_group/log_file/mod.rs b/crates/core/src/file_group/log_file/mod.rs index 39214358..04f71115 100644 --- a/crates/core/src/file_group/log_file/mod.rs +++ b/crates/core/src/file_group/log_file/mod.rs @@ -25,9 +25,9 @@ use std::fmt::Display; use std::str::FromStr; mod avro; -mod content; -mod log_block; -mod log_format; +pub(crate) mod content; +pub mod log_block; +pub mod log_format; pub mod reader; pub mod scanner; diff --git a/crates/core/src/file_group/log_file/reader.rs b/crates/core/src/file_group/log_file/reader.rs index 8fdb61f1..519fbe6a 100644 --- a/crates/core/src/file_group/log_file/reader.rs +++ b/crates/core/src/file_group/log_file/reader.rs @@ -18,14 +18,13 @@ */ use crate::Result; -use crate::config::HudiConfigs; -use crate::config::table::HudiTableConfig; use crate::error::CoreError; use crate::file_group::log_file::content::Decoder; use crate::file_group::log_file::log_block::{ - BlockMetadataKey, BlockMetadataType, BlockType, LogBlock, + BlockMetadataKey, BlockMetadataType, BlockType, LogBlock, LogBlockContentLocation, }; use crate::file_group::log_file::log_format::{LogFormatVersion, MAGIC}; +use crate::file_group::reader::reader_context::ReaderContext; use crate::storage::Storage; use crate::storage::reader::StorageReader; use crate::timeline::selector::InstantRange; @@ -36,23 +35,21 @@ use std::sync::Arc; #[derive(Debug)] pub struct LogFileReader { - hudi_configs: Arc, + reader_context: Arc, reader: R, timezone: String, } impl LogFileReader { pub async fn new( - hudi_configs: Arc, + reader_context: Arc, storage: Arc, relative_path: &str, ) -> Result { let reader = storage.get_storage_reader(relative_path).await?; - let timezone: String = hudi_configs - .get_or_default(HudiTableConfig::TimelineTimezone) - .into(); + let timezone = reader_context.timezone(); Ok(Self { - hudi_configs, + reader_context, reader, timezone, }) @@ -68,6 +65,47 @@ impl LogFileReader { } Ok(blocks) } + + /// Read all blocks from the log file without instant-range filtering. + /// + /// Unlike [`read_all_blocks`], this method returns ALL blocks including those + /// that would normally be skipped by instant-range filtering. This is needed + /// by [`BaseHoodieLogRecordReader::scan_internal`] which implements its own + /// 4-gate filtering algorithm. + pub fn read_all_blocks_unfiltered(&mut self) -> Result> { + let wide_range = InstantRange::up_to("99991231235959999", &self.timezone); + let mut blocks = Vec::new(); + while let Some(block) = self.read_next_block(&wide_range)? { + // Include all blocks (even skipped ones) since the caller + // handles filtering via the 4-gate algorithm. + blocks.push(block); + } + Ok(blocks) + } + + /// Read all blocks as metadata-only (no content decoding). + /// + /// Mirrors Java's Pass 1 behavior: reads block headers and records + /// `content_location` for later lazy inflate, but seeks past the + /// content bytes without decoding them. Only one block's content is + /// in memory at a time during Pass 3. + /// + /// Each block receives a clone of the underlying file `Bytes` (O(1) ref-count + /// increment), mirroring Java's `inputStreamSupplier` on `HoodieLogBlock`. + /// This enables the buffer to call `inflate_from_bytes()` synchronously + /// during Pass 3 without re-downloading the file. + /// + /// # Arguments + /// * `log_file_path` - The relative path to the log file (stored in `LogBlockContentLocation`) + pub fn read_all_blocks_metadata_only(&mut self, log_file_path: &str) -> Result> { + let file_bytes = self.reader.get_ref_bytes().clone(); + let mut blocks = Vec::new(); + while let Some(mut block) = self.read_next_block_metadata_only(log_file_path)? { + block.set_source_bytes(file_bytes.clone()); + blocks.push(block); + } + Ok(blocks) + } } impl LogFileReader { @@ -200,6 +238,27 @@ impl LogFileReader { Ok(Some(u64::from_be_bytes(size_buf))) } + /// Read 8 bytes for the content length, or fall back to `block_length` for V0. + /// + /// Mirrors Java's `HoodieLogFileReader.readBlock()` step 5: + /// ```java + /// int contentLength = nextBlockVersion.getVersion() != DEFAULT_VERSION + /// ? (int) inputStream.readLong() : blockSize; + /// ``` + fn read_content_length( + &mut self, + format_version: &LogFormatVersion, + block_length: u64, + ) -> Result { + if format_version.has_content_length() { + let mut buf = [0u8; 8]; + self.reader.read_exact(&mut buf)?; + Ok(u64::from_be_bytes(buf)) + } else { + Ok(block_length) + } + } + fn should_skip_block( &self, header: &HashMap, @@ -248,14 +307,12 @@ impl LogFileReader { ))); } - let decoder = Decoder::new(self.hudi_configs.clone()); - let content = decoder.decode_content( - self.reader.by_ref(), - &format_version, - block_length, - &block_type, - &header, - )?; + // Read content length before decoding, matching Java step 5→6 + let content_length = self.read_content_length(&format_version, block_length)?; + + let decoder = Decoder::new(self.reader_context.clone()); + let content = + decoder.decode_content(self.reader.by_ref(), content_length, &block_type, &header)?; let footer = self.read_block_metadata(BlockMetadataType::Footer, &format_version)?; let _ = self.read_total_block_length(&format_version)?; @@ -267,6 +324,65 @@ impl LogFileReader { footer, ))) } + + /// Read the next block as metadata-only (no content decoding). + /// + /// Mirrors Java's `HoodieLogFileReader.readBlock()` with `shouldReadLazily=true`: + /// reads the block header, records the content position in a `LogBlockContentLocation`, + /// then seeks past the content + footer without decoding them. + /// + /// The returned `LogBlock` has `content = Empty` and `content_location = Some(...)`. + /// Call `LogBlock::inflate()` later to load and decode the content on demand. + fn read_next_block_metadata_only(&mut self, log_file_path: &str) -> Result> { + if !self.read_magic()? { + return Ok(None); + } + + let curr_pos = self + .reader + .stream_position() + .map_err(CoreError::ReadLogFileError)?; + + let (block_length, _) = self.read_block_length_or_corrupted_block(curr_pos)?; + let format_version = self.read_log_format_version()?; + let block_type = self.read_block_type(&format_version)?; + let header = self.read_block_metadata(BlockMetadataType::Header, &format_version)?; + + // Read the content length field (8 bytes for V1+), matching Java step 5: + // int contentLength = nextBlockVersion.getVersion() != DEFAULT_VERSION + // ? (int) inputStream.readLong() : blockSize; + let content_length = self.read_content_length(&format_version, block_length)?; + + // Record where the content starts — AFTER the content_length field, + // matching Java step 6: contentPosition = inputStream.getPos() + let content_position = self + .reader + .stream_position() + .map_err(CoreError::ReadLogFileError)?; + + // Seek past the rest of the block (content + footer + total_block_length) + let block_end = curr_pos + .checked_add(8) + .and_then(|v| v.checked_add(block_length)) + .ok_or_else(|| CoreError::LogFormatError("Block length overflow".to_string()))?; + self.reader + .seek(SeekFrom::Start(block_end)) + .map_err(CoreError::ReadLogFileError)?; + + let content_loc = LogBlockContentLocation { + log_file_path: log_file_path.to_string(), + content_position, + content_length, + block_length, + }; + + Ok(Some(LogBlock::new_lazy( + format_version, + block_type, + header, + content_loc, + ))) + } } #[cfg(test)] @@ -316,9 +432,8 @@ mod tests { file_name: &str, ) -> Result> { let dir_url = parse_uri(dir)?; - let hudi_configs = Arc::new(HudiConfigs::new([(HudiTableConfig::PrecombineField, "ts")])); let storage = Storage::new_with_base_url(dir_url)?; - LogFileReader::new(hudi_configs, storage, file_name).await + LogFileReader::new(Arc::new(ReaderContext::empty()), storage, file_name).await } #[tokio::test] diff --git a/crates/core/src/file_group/log_file/scanner.rs b/crates/core/src/file_group/log_file/scanner.rs index b911ffd2..b9df112d 100644 --- a/crates/core/src/file_group/log_file/scanner.rs +++ b/crates/core/src/file_group/log_file/scanner.rs @@ -17,9 +17,9 @@ * under the License. */ use crate::Result; -use crate::config::HudiConfigs; use crate::file_group::log_file::log_block::{BlockType, LogBlock, LogBlockContent}; use crate::file_group::log_file::reader::LogFileReader; +use crate::file_group::reader::reader_context::ReaderContext; use crate::file_group::record_batches::RecordBatches; use crate::hfile::HFileRecord; use crate::storage::Storage; @@ -125,14 +125,14 @@ impl CollectedBlocks { #[derive(Debug)] pub struct LogFileScanner { - hudi_configs: Arc, + reader_context: Arc, storage: Arc, } impl LogFileScanner { - pub fn new(hudi_configs: Arc, storage: Arc) -> Self { + pub fn new(reader_context: Arc, storage: Arc) -> Self { Self { - hudi_configs, + reader_context, storage, } } @@ -148,7 +148,8 @@ impl LogFileScanner { for path in relative_paths { let mut reader = - LogFileReader::new(self.hudi_configs.clone(), self.storage.clone(), &path).await?; + LogFileReader::new(self.reader_context.clone(), self.storage.clone(), &path) + .await?; let blocks = reader.read_all_blocks(instant_range)?; // Collect rollback targets from command blocks @@ -456,8 +457,7 @@ mod tests { let dir_url = parse_uri(metadata_dir.to_str().unwrap())?; let storage = Storage::new_with_base_url(dir_url)?; - let hudi_configs = Arc::new(HudiConfigs::empty()); - let scanner = LogFileScanner::new(hudi_configs, Arc::clone(&storage)); + let scanner = LogFileScanner::new(Arc::new(ReaderContext::empty()), Arc::clone(&storage)); let instant_range = InstantRange::up_to("99991231235959999", "utc"); // Scan each log file individually and validate record counts diff --git a/crates/core/src/file_group/mod.rs b/crates/core/src/file_group/mod.rs index 05293a28..c695e5d2 100644 --- a/crates/core/src/file_group/mod.rs +++ b/crates/core/src/file_group/mod.rs @@ -25,6 +25,7 @@ pub mod builder; pub mod file_slice; pub mod log_file; pub mod reader; +pub mod reader_v1; pub mod record_batches; use crate::Result; @@ -192,7 +193,6 @@ impl FileGroup { /// - **Without completion_timestamp (v6 tables)**: Use exact matching or range lookup /// based on log timestamp. /// - /// TODO: support adding log files to file group without base files. pub fn add_log_file(&mut self, log_file: LogFile) -> Result<&Self> { // Validate file_id matches if log_file.file_id != self.file_id { @@ -217,16 +217,20 @@ impl FileGroup { return Ok(self); } - // No file slice with base instant time <= log's completion time found. - // This means the log file's completion timestamp is earlier than all base files' - // commit timestamps, or the FileGroup has no base files. - // TODO: Support log files without base files in a future priority task. - return Err(CoreError::FileGroup(format!( - "No suitable FileSlice found for log file with completion_timestamp {} in File Group {}. \ - Either the log file's completion timestamp is earlier than all base files' commit timestamps, \ - or the FileGroup has no base files.", - log_completion_time, self.file_id - ))); + // No suitable existing file slice found. + // Create a log-only file slice keyed by the log file's delta commit time. + // Mirrors Java: HoodieFileGroup.getBaseInstantTime() returns + // logFile.getDeltaCommitTime() when fileSlices.isEmpty() or no + // base file has a commit time <= the log's completion time. + let key = log_file.timestamp.clone(); + let mut file_slice = FileSlice::new_log_only( + self.file_id.clone(), + key.clone(), + self.partition_path.clone(), + ); + file_slice.log_files.insert(log_file); + self.file_slices.insert(key, file_slice); + return Ok(self); } // No completion_timestamp: use base instant timestamp-based association (v6 tables) @@ -241,10 +245,16 @@ impl FileGroup { return Ok(self); } - Err(CoreError::FileGroup(format!( - "No suitable FileSlice found for log file with timestamp {} in File Group {}", - log_timestamp, self.file_id - ))) + // No suitable existing file slice found — create a log-only file slice. + let key = log_file.timestamp.clone(); + let mut file_slice = FileSlice::new_log_only( + self.file_id.clone(), + key.clone(), + self.partition_path.clone(), + ); + file_slice.log_files.insert(log_file); + self.file_slices.insert(key, file_slice); + Ok(self) } /// Add multiple [LogFile]s to the corresponding [FileSlice]s in the [FileGroup]. @@ -310,6 +320,8 @@ mod tests { fg.get_file_slice_as_of("20240402123035233") .unwrap() .base_file + .as_ref() + .unwrap() .commit_timestamp, "20240402123035233" ); @@ -419,9 +431,12 @@ mod tests { assert!(fg.file_slices.contains_key("20250113230302428")); // Verify we can get the file slice using request timestamp let slice = fg.get_file_slice_as_of("20250113230302428").unwrap(); - assert_eq!(slice.base_file.commit_timestamp, "20250113230302428"); assert_eq!( - slice.base_file.completion_timestamp, + slice.base_file.as_ref().unwrap().commit_timestamp, + "20250113230302428" + ); + assert_eq!( + slice.base_file.as_ref().unwrap().completion_timestamp, Some("20250113230310000".to_string()) ); } @@ -478,8 +493,9 @@ mod tests { } #[test] - fn test_file_group_log_file_error_cases() { + fn test_file_group_log_file_creates_log_only_slice() { // Test 1: Log file completed before any base file's request time + // → creates a log-only file slice keyed by log's delta commit time let mut fg1 = FileGroup::new("file-id-0".to_string(), EMPTY_PARTITION_PATH.to_string()); let base = create_base_file_with_completion( "file-id-0", @@ -494,16 +510,15 @@ mod tests { Some("20250113230100000"), // completion at t1 < t2 (base request time) 1, ); - let result = fg1.add_log_file(log); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("completion timestamp is earlier than all base files") - ); + fg1.add_log_file(log).unwrap(); + // A log-only slice should be created keyed by the log's delta commit time + assert_eq!(fg1.file_slices.len(), 2); + let log_only_slice = fg1.file_slices.get("20250113230050000").unwrap(); + assert!(log_only_slice.base_file.is_none()); + assert!(log_only_slice.has_log_file()); // Test 2: Log file with completion_timestamp when no file slices exist + // → creates a log-only file slice let mut fg2 = FileGroup::new("file-id-0".to_string(), EMPTY_PARTITION_PATH.to_string()); let log = create_log_file_with_completion( "file-id-0", @@ -511,14 +526,13 @@ mod tests { Some("20250113230000150"), 1, ); - let result = fg2.add_log_file(log); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("No suitable FileSlice found") - ); + fg2.add_log_file(log).unwrap(); + assert_eq!(fg2.file_slices.len(), 1); + let slice = fg2.file_slices.values().next().unwrap(); + assert!(slice.base_file.is_none()); + assert!(slice.has_log_file()); + assert_eq!(slice.file_id(), "file-id-0"); + assert_eq!(slice.creation_instant_time(), "20250113230000010"); } #[test] @@ -702,7 +716,7 @@ mod tests { } #[test] - fn test_file_group_v6_log_file_no_suitable_slice_error() { + fn test_file_group_v6_log_file_creates_log_only_slice() { // V6 table: log file without completion_timestamp let mut fg = FileGroup::new("file-id-0".to_string(), "partition1".to_string()); let base = create_base_file_with_completion("file-id-0", "20240101130000000", None); @@ -710,14 +724,12 @@ mod tests { // Log file timestamp is earlier than base file's commit timestamp // Without completion_timestamp, it uses timestamp-based association + // → creates a log-only file slice keyed by log's delta commit time let log_file = create_log_file_with_completion("file-id-0", "20240101120000000", None, 1); - let result = fg.add_log_file(log_file); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("No suitable FileSlice found for log file with timestamp") - ); + fg.add_log_file(log_file).unwrap(); + assert_eq!(fg.file_slices.len(), 2); + let log_only_slice = fg.file_slices.get("20240101120000000").unwrap(); + assert!(log_only_slice.base_file.is_none()); + assert!(log_only_slice.has_log_file()); } } diff --git a/crates/core/src/file_group/reader/buffer/key_based.rs b/crates/core/src/file_group/reader/buffer/key_based.rs new file mode 100644 index 00000000..a43efd0b --- /dev/null +++ b/crates/core/src/file_group/reader/buffer/key_based.rs @@ -0,0 +1,1313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.buffer.KeyBasedFileGroupRecordBuffer`. +//! +//! The default record buffer. Deduplicates by record key using a HashMap, +//! resolves conflicts via ordering value comparison (delta merge). +//! +//! ## How it's selected (in `DefaultFileGroupRecordBufferLoader`): +//! ```text +//! !is_skip_merge && !sort_outputs && !(use_record_position && base_file_present) +//! → KeyBasedFileGroupRecordBuffer (DEFAULT) +//! ``` +//! +//! ## Method mapping (Java → Rust): +//! - `processDataBlock` → `process_data_block` +//! - `processNextDataRecord` → `process_next_data_record` +//! - `processDeleteBlock` → `process_delete_block` +//! - `processNextDeletedRecord` → `process_next_deleted_record` +//! - `doHasNext` → `do_has_next` +//! - `hasNextBaseRecord` → `has_next_base_record` + +use crate::Result; +use crate::file_group::log_file::log_block::{LogBlock, LogBlockContent}; +use crate::file_group::reader::buffer::record_buffer::FileGroupRecordBuffer; +use crate::file_group::reader::buffer::row_extraction::records_to_batch; +use crate::file_group::reader::buffer::{BufferType, HoodieFileGroupRecordBuffer}; +use crate::file_group::reader::buffered_record::{BufferedRecord, BufferedRecords, DeleteRecord}; +use crate::file_group::reader::read_stats::HoodieReadStats; +use crate::file_group::reader::reader_context::ReaderContext; +use crate::file_group::reader::record_context::RecordContext; +use crate::file_group::reader::record_merger::BufferedRecordMergerFactory; +use crate::file_group::reader::update_processor::create_update_processor; +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; +use std::collections::HashMap; +use std::sync::Arc; + +#[cfg(test)] +use crate::config::table::HudiTableConfig; + +/// Key-based file group record buffer. +/// +/// Mirrors Java's `KeyBasedFileGroupRecordBuffer`. +/// +/// ## Per-record merge during log scanning: +/// ```text +/// processDataBlock(batch): +/// for each row → processNextDataRecord(record, key) +/// → records.get(key) → deltaMerge(new, existing) → records.put(key, merged) +/// +/// processDeleteBlock(batch): +/// for each row → processNextDeletedRecord(deleteRecord, key) +/// → records.get(key) → deltaMergeDelete(delete, existing) → records.put(key, merged) +/// ``` +/// +/// ## Base-vs-log merge at read time: +/// ```text +/// doHasNext(): +/// while baseIterator.hasNext(): +/// if hasNextBaseRecord(base): return true +/// return hasNextLogRecord() +/// +/// hasNextBaseRecord(base): +/// key = extractKey(base) +/// logRecord = records.remove(key) +/// return super.hasNextBaseRecord(base, logRecord) +/// ``` +#[derive(Debug)] +pub struct KeyBasedFileGroupRecordBuffer { + /// Common buffer state. + pub base: FileGroupRecordBuffer, + + /// Reader context (mirrors Java's readerContext). + pub reader_context: Arc, + + /// Record context for record-level operations (key extraction, ordering, etc.). + /// Mirrors Java's `readerContext.getRecordContext()`. + pub record_context: RecordContext, + + /// Key-spec filter set by scan_internal for per-record filtering. + /// Mirrors Java's keySpecOpt threaded through to KeyBasedFileGroupRecordBuffer. + pub key_spec_opt: Option, +} + +impl KeyBasedFileGroupRecordBuffer { + pub fn new( + reader_context: Arc, + merge_mode: String, + read_stats: &HoodieReadStats, + ) -> crate::Result { + let merger = BufferedRecordMergerFactory::create(&merge_mode)?; + let update_processor = create_update_processor(read_stats, false); + // Get the shared RecordContext from ReaderContext (mirrors Java's + // readerContext.getRecordContext() returning the same instance). + let record_context = reader_context.get_record_context().clone(); + + // Extract reader_schema from schema_handler.required_schema at + // construction time. Mirrors Java's FileGroupRecordBuffer constructor: + // this.readerSchema = readerContext.getSchemaHandler().getRequiredSchema() + let reader_schema = reader_context.schema_handler.required_schema.clone(); + + // Get the canonical DeleteContext from the schema handler (single source + // of truth). Mirrors Java's `readerContext.getSchemaHandler().getDeleteContext()`. + // + // The schema handler creates the DeleteContext during + // `prepare_required_schema()` and stores it. HoodieFileGroupReader + // propagates the prepared schema_handler onto reader_context before + // the buffer is created, so `reader_context.schema_handler.delete_context()` + // always returns the canonical instance. + let delete_context = reader_context + .schema_handler + .delete_context() + .cloned() + .expect("DeleteContext must be set on schema_handler by prepare_required_schema()"); + + // Enrich DeleteContext with reader schema at construction time. + // Mirrors Java's: deleteContext = readerContext.getSchemaHandler() + // .getDeleteContext().withReaderSchema(this.readerSchema) + let delete_context = if let Some(ref schema) = reader_schema { + delete_context.with_reader_schema(schema.clone()) + } else { + delete_context + }; + + let mut base = + FileGroupRecordBuffer::new(merge_mode, merger, update_processor, reader_schema); + base.delete_context = Some(delete_context); + + Ok(Self { + base, + reader_context, + record_context, + key_spec_opt: None, + }) + } + + /// Mirrors Java's `KeyBasedFileGroupRecordBuffer.hasNextBaseRecord(T baseRecord)`. + /// + /// Looks up the record key in the records map, removes it if found, + /// then delegates to `FileGroupRecordBuffer.has_next_base_record(base, log)`. + fn has_next_base_record_keyed(&mut self, base_row: &RecordBatch) -> Result { + let keys = self.record_context.get_record_keys(base_row)?; + let key = &keys[0]; // single-row batch + let log_record = self.base.records.remove(key); + + let base_record = BufferedRecord::new_data( + key.clone(), + base_row.clone(), + None, // TODO: extract ordering value + ); + + self.base + .has_next_base_record(&base_record, log_record.as_ref()) + } + + /// Mirrors Java's `KeyBasedFileGroupRecordBuffer.doHasNext()`. + /// + /// First iterates base file records, merging with log records. + /// Then iterates remaining log-only records. + fn do_has_next(&mut self) -> Result { + // Handle merging: iterate base file rows + while let Some(base_row) = self.base.next_base_row() { + if self.has_next_base_record_keyed(&base_row)? { + return Ok(true); + } + } + + // Handle records solely from log files + Ok(self.base.has_next_log_record()) + } +} + +impl HoodieFileGroupRecordBuffer for KeyBasedFileGroupRecordBuffer { + fn get_buffer_type(&self) -> BufferType { + BufferType::KeyBasedMerge + } + + fn set_key_spec(&mut self, key_spec: Option) { + self.key_spec_opt = key_spec; + } + + /// Mirrors Java's `KeyBasedFileGroupRecordBuffer.processDataBlock(HoodieDataBlock, Option)`. + /// + /// Inflates the block on demand (matching Java where inflate/deserialize/deflate + /// happens inside the block triggered by `getRecordsIterator`), then iterates + /// each record, extracts the key, creates a BufferedRecord, and calls + /// `process_next_data_record`. + fn process_data_block(&mut self, block: &mut LogBlock) -> Result<()> { + // Mirrors Java: getRecordsIterator → getEngineRecordIterator + // → readRecordsFromBlockPayload → inflate → deserializeRecords → deflate + block.inflate_from_bytes(self.reader_context.clone())?; + + if let LogBlockContent::Records(record_batches) = std::mem::take(&mut block.content) { + let total_rows: usize = record_batches + .data_batches + .iter() + .map(|b| b.num_rows()) + .sum(); + log::debug!( + "[KeyBasedBuffer] processDataBlock: {} data batches, {} total rows", + record_batches.data_batches.len(), + total_rows, + ); + for batch in record_batches.data_batches { + let records = self + .record_context + .batch_to_buffered_records(&batch, self.base.delete_context.as_ref())?; + for (key, record) in records { + // Mirrors Java BaseHoodieLogRecordReader: skip records + // that don't match the key-spec filter. + if let Some(spec) = &self.key_spec_opt { + if !spec.matches(&key) { + continue; + } + } + self.process_next_data_record(record, &key)?; + } + } + } + block.deflate(); + Ok(()) + } + + /// Mirrors Java's `KeyBasedFileGroupRecordBuffer.processNextDataRecord(BufferedRecord, Serializable)`. + /// + /// ```java + /// BufferedRecord existingRecord = records.get(recordKey); + /// totalLogRecords++; + /// bufferedRecordMerger.deltaMerge(record, existingRecord) + /// .ifPresent(merged -> records.put(recordKey, merged.toBinary(recordContext))); + /// ``` + fn process_next_data_record(&mut self, record: BufferedRecord, key: &str) -> Result<()> { + let has_existing = self.base.records.contains_key(key); + let existing = self.base.records.get(key); + self.base.total_log_records += 1; + + let merged = self + .base + .buffered_record_merger + .delta_merge(&record, existing)?; + + if let Some(mut merged_record) = merged { + log::debug!( + "[KeyBasedBuffer] processNextDataRecord: key={key} has_existing={has_existing} → merged (is_delete={})", + merged_record.is_delete(), + ); + // Mirrors Java: records.put(recordKey, merged.toBinary(readerContext.getRecordContext())) + merged_record.to_binary(&self.record_context); + self.base.records.insert(key.to_string(), merged_record); + } else { + log::debug!( + "[KeyBasedBuffer] processNextDataRecord: key={key} has_existing={has_existing} → dropped by merger", + ); + } + + Ok(()) + } + + /// Mirrors Java's `KeyBasedFileGroupRecordBuffer.processDeleteBlock(HoodieDeleteBlock)`. + /// + /// Inflates the block on demand, then iterates delete records and calls + /// `process_next_deleted_record` for each. + fn process_delete_block(&mut self, block: &mut LogBlock) -> Result<()> { + block.inflate_from_bytes(self.reader_context.clone())?; + + if let LogBlockContent::Records(record_batches) = std::mem::take(&mut block.content) { + let total_deletes: usize = record_batches + .delete_batches + .iter() + .map(|(b, _)| b.num_rows()) + .sum(); + log::debug!( + "[KeyBasedBuffer] processDeleteBlock: {} delete batches, {} total deletes", + record_batches.delete_batches.len(), + total_deletes, + ); + for (batch, _inst) in record_batches.delete_batches { + let delete_records = self + .record_context + .delete_batch_to_buffered_records(&batch)?; + for (key, _record) in delete_records { + // Same key-spec filter as the data block path. + if let Some(spec) = &self.key_spec_opt { + if !spec.matches(&key) { + continue; + } + } + let delete_record = DeleteRecord { + record_key: key.clone(), + partition_path: String::new(), // TODO: extract from batch + ordering_value: None, // TODO: extract from batch + }; + self.process_next_deleted_record(delete_record, &key); + } + } + } + block.deflate(); + Ok(()) + } + + /// Mirrors Java's `KeyBasedFileGroupRecordBuffer.processNextDeletedRecord(DeleteRecord, Serializable)`. + /// + /// ```java + /// BufferedRecord existingRecord = records.get(recordIdentifier); + /// totalLogRecords++; + /// bufferedRecordMerger.deltaMerge(deleteRecord, existingRecord) + /// .ifPresent(deleteRec -> records.put(recordIdentifier, + /// BufferedRecords.fromDeleteRecord(deleteRec, recordContext))); + /// ``` + fn process_next_deleted_record(&mut self, delete_record: DeleteRecord, key: &str) { + let has_existing = self.base.records.contains_key(key); + let existing = self.base.records.get(key); + self.base.total_log_records += 1; + + let surviving = self + .base + .buffered_record_merger + .delta_merge_delete(&delete_record, existing); + + if let Ok(Some(surviving_delete)) = &surviving { + log::debug!( + "[KeyBasedBuffer] processNextDeletedRecord: key={key} has_existing={has_existing} → delete wins", + ); + self.base.records.insert( + key.to_string(), + BufferedRecords::from_delete_record(surviving_delete), + ); + } else { + log::debug!( + "[KeyBasedBuffer] processNextDeletedRecord: key={key} has_existing={has_existing} → existing survives", + ); + } + } + + fn contains_log_record(&self, record_key: &str) -> bool { + self.base.records.contains_key(record_key) + } + + fn size(&self) -> usize { + self.base.records.len() + } + + fn get_total_log_records(&self) -> u64 { + self.base.total_log_records + } + + fn get_log_records(&self) -> &HashMap { + &self.base.records + } + + fn set_reader_schema(&mut self, schema: SchemaRef) { + self.base.reader_schema = Some(schema.clone()); + // Phase 2: Enrich DeleteContext with the reader schema. + // Mirrors Java's `deleteContext.withReaderSchema(this.readerSchema)` + // in FileGroupRecordBuffer constructor. + if let Some(ctx) = self.base.delete_context.take() { + self.base.delete_context = Some(ctx.with_reader_schema(schema)); + } + } + + fn set_base_file_iterator(&mut self, batches: Vec) { + self.base.base_file_batches = batches; + self.base.base_batch_idx = 0; + self.base.base_row_idx = 0; + } + + /// Mirrors Java's `hasNext()` template method: + /// `nextRecord != null || doHasNext()` + fn has_next(&mut self) -> Result { + if self.base.next_record.is_some() { + return Ok(true); + } + self.do_has_next() + } + + /// Mirrors Java's `next()`: + /// Take and return `nextRecord`, set to None. + fn next(&mut self) -> Option { + self.base.next_record.take() + } + + /// Consume the buffer and produce merged output as a RecordBatch. + /// + /// Drives the `has_next()`/`next()` iterator to completion and + /// collects all records into a single batch. + fn merge_and_collect(mut self: Box) -> Result { + let base_rows: usize = self + .base + .base_file_batches + .iter() + .map(|b| b.num_rows()) + .sum(); + let log_records = self.base.records.len(); + log::debug!( + "[KeyBasedBuffer] merge_and_collect: base_rows={base_rows} log_records_in_map={log_records} \ + total_log_records_processed={}", + self.base.total_log_records, + ); + + // Use reader_schema (= required_schema) as the output schema when set, + // since it represents the merge-compatible schema. The base file may be + // projected to fewer columns, while log records have the full writer schema. + // reader_schema is the common ground that both are reconciled to. + let schema = if let Some(schema) = &self.base.reader_schema { + schema.clone() + } else if !self.base.base_file_batches.is_empty() { + self.base.base_file_batches[0].schema() + } else { + // Fallback: find any non-delete record in the buffer. + // HashMap iteration order is non-deterministic, so we must + // search all records — the first entry could be a delete. + let any_data_record = self.base.records.values().find_map(|r| r.get_record()); + match any_data_record.as_ref() { + Some(batch) => batch.schema(), + None => { + return Err(crate::error::CoreError::ReadFileSliceError( + "No schema available for merge output".to_string(), + )); + } + } + }; + + let mut output_records: Vec = Vec::new(); + let mut deletes = 0u64; + while self.has_next()? { + if let Some(record) = self.next() { + if record.is_delete() { + deletes += 1; + } else { + // Simplified: if it came from base+log merge it's an update, + // if from log-only it's an insert. We count all as output. + output_records.push(record); + } + } + } + + log::debug!( + "[KeyBasedBuffer] merge_and_collect output: {} data records, {} deletes skipped", + output_records.len(), + deletes, + ); + + records_to_batch(output_records, schema) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_group::log_file::log_block::{BlockMetadataKey, BlockType, LogBlockContent}; + use crate::file_group::log_file::log_format::LogFormatVersion; + use crate::file_group::reader::buffered_record::OrderingValue; + use crate::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; + use crate::file_group::record_batches::RecordBatches; + use arrow_array::{Int32Array, Int64Array, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use std::sync::Arc; + + // ========================================================================= + // Test helper infrastructure (mirrors Java's BaseTestFileGroupRecordBuffer) + // ========================================================================= + + /// Wrap a RecordBatch in a LogBlock with content already populated. + /// Used by tests so they can call `process_data_block(&mut block)` directly. + fn make_data_block(batch: RecordBatch, instant: &str) -> LogBlock { + let mut header = HashMap::new(); + header.insert(BlockMetadataKey::InstantTime, instant.to_string()); + LogBlock::new( + LogFormatVersion::V1, + BlockType::ParquetData, + header, + LogBlockContent::Records(RecordBatches::new_with_data_batches(vec![batch])), + HashMap::new(), + ) + } + + /// Schema: _hoodie_record_key (Utf8), counter (Int32), ts (Int64) + /// Matches Java test schema: record_key, counter, ts + fn create_test_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("counter", DataType::Int32, false), + Field::new("ts", DataType::Int64, false), + ])) + } + + /// Create a multi-row RecordBatch from a list of (key, counter, ts). + fn create_test_batch(records: &[(&str, i32, i64)]) -> RecordBatch { + let schema = create_test_schema(); + let keys: Vec<&str> = records.iter().map(|(k, _, _)| *k).collect(); + let counters: Vec = records.iter().map(|(_, c, _)| *c).collect(); + let timestamps: Vec = records.iter().map(|(_, _, t)| *t).collect(); + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(keys)), + Arc::new(Int32Array::from(counters)), + Arc::new(Int64Array::from(timestamps)), + ], + ) + .unwrap() + } + + /// Build a KeyBasedFileGroupRecordBuffer with the given merge mode. + fn build_key_based_buffer(merge_mode: &str) -> KeyBasedFileGroupRecordBuffer { + let mut ctx = ReaderContext::empty(); + ctx.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + ctx.rebuild_record_context(String::new()); + // Prepare the schema handler so it creates and stores a DeleteContext + // (required by KeyBasedFileGroupRecordBuffer::new). + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(create_test_schema()) + .with_data_schema(create_test_schema()); + let key_field = ctx.record_key_field().to_string(); + let ordering = ctx.record_context.ordering_field_names.clone(); + handler.prepare_required_schema( + true, + &[key_field], + &ordering, + &ctx.table_config, + false, + merge_mode, + ); + ctx.schema_handler = handler; + let ctx = Arc::new(ctx); + let read_stats = HoodieReadStats::default(); + KeyBasedFileGroupRecordBuffer::new(ctx, merge_mode.to_string(), &read_stats).unwrap() + } + + /// Extract (key, counter, ts) tuples from a RecordBatch, sorted by key. + fn extract_records(batch: &RecordBatch) -> Vec<(String, i32, i64)> { + let keys = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let counters = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let timestamps = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let mut result: Vec<(String, i32, i64)> = (0..batch.num_rows()) + .map(|i| { + ( + keys.value(i).to_string(), + counters.value(i), + timestamps.value(i), + ) + }) + .collect(); + result.sort_by(|a, b| a.0.cmp(&b.0)); + result + } + + // ========================================================================= + // Part B: Tests matching Java's TestKeyBasedFileGroupRecordBuffer + // ========================================================================= + + /// Build a KeyBasedFileGroupRecordBuffer with a custom delete marker. + /// Mirrors Java's `buildKeyBasedFileGroupRecordBuffer(..., deleteMarkerKeyValue)`. + fn build_key_based_buffer_with_delete_marker( + merge_mode: &str, + delete_key: &str, + delete_marker_value: &str, + ) -> KeyBasedFileGroupRecordBuffer { + let mut ctx = ReaderContext::empty(); + ctx.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + ctx.table_config.insert( + "hoodie.datasource.write.payload.delete.field".to_string(), + delete_key.to_string(), + ); + ctx.table_config.insert( + "hoodie.datasource.write.payload.delete.marker".to_string(), + delete_marker_value.to_string(), + ); + ctx.rebuild_record_context(String::new()); + // Prepare the schema handler so it creates and stores a DeleteContext. + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(create_test_schema()) + .with_data_schema(create_test_schema()); + let key_field = ctx.record_key_field().to_string(); + let ordering = ctx.record_context.ordering_field_names.clone(); + handler.prepare_required_schema( + true, + &[key_field], + &ordering, + &ctx.table_config, + false, + merge_mode, + ); + ctx.schema_handler = handler; + let ctx = Arc::new(ctx); + let read_stats = HoodieReadStats::default(); + KeyBasedFileGroupRecordBuffer::new(ctx, merge_mode.to_string(), &read_stats).unwrap() + } + + /// Java: TestKeyBasedFileGroupRecordBuffer.readWithCommitTimeOrdering + /// + /// COMMIT_TIME_ORDERING with custom delete marker (counter=3). + /// Last writer always wins. Record with counter=3 detected as delete. + /// Expected: 2 output rows (key=3 deleted by custom marker). + #[test] + fn test_read_with_commit_time_ordering() { + let mut buffer = + build_key_based_buffer_with_delete_marker("COMMIT_TIME_ORDERING", "counter", "3"); + + // DataBlock1: testRecord1UpdateWithSameTime, testRecord2Update, testRecord2EarlierUpdate + let block1 = create_test_batch(&[ + ("1", 2, 1), // record1UpdateWithSameTime + ("2", 1, 2), // record2Update + ("2", 1, 0), // record2EarlierUpdate (overwrites in commit-time) + ]); + buffer + .process_data_block(&mut make_data_block(block1, "instant1")) + .unwrap(); + + // DataBlock2: testRecord2EarlierUpdate, testRecord3Update, testRecord3DeleteByFieldValue + let block2 = create_test_batch(&[ + ("2", 1, 0), // record2EarlierUpdate (same value, overwrites) + ("3", 1, 2), // record3Update + ("3", 3, 1), // record3DeleteByFieldValue (counter=3 → delete, overwrites) + ]); + buffer + .process_data_block(&mut make_data_block(block2, "instant2")) + .unwrap(); + + let base = create_test_batch(&[("1", 1, 1), ("2", 1, 1), ("3", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + // Java: assertEquals(Arrays.asList(testRecord1UpdateWithSameTime, + // testRecord2EarlierUpdate), actualRecords); + assert_eq!(result.num_rows(), 2); + assert_eq!(records[0], ("1".to_string(), 2, 1)); // record1UpdateWithSameTime + assert_eq!(records[1], ("2".to_string(), 1, 0)); // record2EarlierUpdate (last write) + } + + /// Java: TestKeyBasedFileGroupRecordBuffer.readWithEventTimeOrdering + /// + /// Given: EVENT_TIME_ORDERING, ordering_field="ts" + /// base=[record1(k=1,c=1,ts=1), record2(k=2,c=1,ts=1), record3(k=3,c=1,ts=1)] + /// When: DataBlock=[record1_update(k=1,c=2,ts=1), record2_update(k=2,c=1,ts=2), + /// record2_earlier(k=2,c=1,ts=0), record3_update(k=3,c=1,ts=2), + /// record3_delete(k=3,c=3,ts=1)] + /// Then: Event-time: higher ts wins. + /// k=1: update ts=1 >= existing ts=1 → update wins + /// k=2: update ts=2 > ts=1, earlier ts=0 < ts=2 → update(ts=2) wins + /// k=3: update ts=2 > ts=1, delete ts=1 < ts=2 → update(ts=2) wins + /// Expected: 3 records (all updates survive), 0 deletes + #[test] + fn test_read_with_event_time_ordering() { + let mut buffer = build_key_based_buffer("EVENT_TIME_ORDERING"); + + let block = create_test_batch(&[ + ("1", 2, 1), // record1 update (same time → new wins) + ("2", 1, 2), // record2 update (ts=2) + ("2", 1, 0), // record2 earlier update (ts=0 < ts=2 → ignored) + ("3", 1, 2), // record3 update (ts=2) + ("3", 3, 1), // record3 "delete" (ts=1 < ts=2 → ignored) + ]); + buffer + .process_data_block(&mut make_data_block(block, "instant1")) + .unwrap(); + + let base = create_test_batch(&[("1", 1, 1), ("2", 1, 1), ("3", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(result.num_rows(), 3); + assert_eq!(records[0], ("1".to_string(), 2, 1)); // updated + assert_eq!(records[1], ("2".to_string(), 1, 2)); // update with ts=2 wins + assert_eq!(records[2], ("3".to_string(), 1, 2)); // update with ts=2 wins over "delete" + } + + // ========================================================================= + // Part E: Phase A/B/C tests from FS_logBlockConsumption.md + // ========================================================================= + + /// Phase A: Data record overwrites existing in commit-time mode. + /// + /// Given: Empty buffer, COMMIT_TIME_ORDERING + /// When: process_data_block with 2 records for same key: ("k", v1=1) then ("k", v2=2) + /// Then: records["k"] has v2 (last writer wins) + #[test] + fn test_phase_a_data_record_overwrites_existing() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Two records for same key in one block + let block = create_test_batch(&[("k", 1, 10), ("k", 2, 20)]); + buffer + .process_data_block(&mut make_data_block(block, "instant1")) + .unwrap(); + + assert_eq!(buffer.size(), 1); + let record = buffer.base.records.get("k").unwrap(); + // In commit-time: last writer wins → second record (counter=2, ts=20) + assert!(!record.is_delete()); + } + + /// Phase B: Key exists in both base file and log map. + /// + /// Given: Base=[k1=(c=1,ts=1)], log records=[k1=(c=2,ts=2)] + /// When: merge_and_collect() + /// Then: Output has k1 with log values (log wins in final merge) + #[test] + fn test_phase_b_key_in_both_base_and_log() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + let log_block = create_test_batch(&[("k1", 2, 2)]); + buffer + .process_data_block(&mut make_data_block(log_block, "instant1")) + .unwrap(); + + let base = create_test_batch(&[("k1", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(result.num_rows(), 1); + assert_eq!(records[0], ("k1".to_string(), 2, 2)); // log wins + } + + /// Phase B: Key only in base file (no matching log record). + /// + /// Given: Base=[k2=(c=1,ts=1)], log records={} + /// When: merge_and_collect() + /// Then: Output has k2 with base values (emitted as-is) + #[test] + fn test_phase_b_key_only_in_base() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + // No log records + + let base = create_test_batch(&[("k2", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(result.num_rows(), 1); + assert_eq!(records[0], ("k2".to_string(), 1, 1)); // base as-is + } + + /// Phase B: Delete in log map removes base record. + /// + /// Given: Base=[k3=(c=1,ts=1)], log records=[k3=DELETE] + /// When: merge_and_collect() + /// Then: k3 NOT in output (deleted by log) + #[test] + fn test_phase_b_delete_in_log() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Insert a delete record directly into the map + buffer.base.records.insert( + "k3".to_string(), + BufferedRecord::new_delete("k3".to_string(), None), + ); + + let base = create_test_batch(&[("k3", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + + // Delete record merged with base → delete wins → filtered out + // (BufferedRecord with is_delete=true has data=None, so records_to_batch skips it) + assert_eq!(result.num_rows(), 0); + } + + /// Phase C: Log-only record emitted as INSERT. + /// + /// Given: Base=[], log records=[k4=(c=1,ts=1)] + /// When: merge_and_collect() + /// Then: Output has k4 as INSERT from log only + #[test] + fn test_phase_c_log_only_insert() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + let log_block = create_test_batch(&[("k4", 1, 1)]); + buffer + .process_data_block(&mut make_data_block(log_block, "instant1")) + .unwrap(); + + // No base file + buffer.set_base_file_iterator(vec![]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(result.num_rows(), 1); + assert_eq!(records[0], ("k4".to_string(), 1, 1)); // insert from log + } + + /// Phase C: Empty log map — base records pass through. + /// + /// Given: Base=[k1, k2], log records={} + /// When: merge_and_collect() + /// Then: Output has k1, k2 from base (no log merging) + #[test] + fn test_phase_c_empty_log_map() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + // No log records + + let base = create_test_batch(&[("k1", 1, 1), ("k2", 2, 2)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(result.num_rows(), 2); + assert_eq!(records[0], ("k1".to_string(), 1, 1)); + assert_eq!(records[1], ("k2".to_string(), 2, 2)); + } + + /// Phase C: Empty base — all log records are INSERTs. + /// + /// Given: Base=[], log records=[k1, k2] + /// When: merge_and_collect() + /// Then: Output has k1, k2 as INSERTs + #[test] + fn test_phase_c_empty_base() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + let log_block = create_test_batch(&[("k1", 1, 1), ("k2", 2, 2)]); + buffer + .process_data_block(&mut make_data_block(log_block, "instant1")) + .unwrap(); + + buffer.set_base_file_iterator(vec![]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + + assert_eq!(result.num_rows(), 2); + } + + /// Multiple writes to same key across blocks (commit-time ordering). + /// + /// Given: 3 blocks writing same key "k": v1(i1), v2(i2), then a third value + /// When: Full scan + merge + /// Then: records["k"] = last-written value + #[test] + fn test_multiple_writes_same_key_commit_time() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Block 1: key "k" with counter=1 + let block1 = create_test_batch(&[("k", 1, 10)]); + buffer + .process_data_block(&mut make_data_block(block1, "i1")) + .unwrap(); + + // Block 2: same key with counter=2 + let block2 = create_test_batch(&[("k", 2, 20)]); + buffer + .process_data_block(&mut make_data_block(block2, "i2")) + .unwrap(); + + // Block 3: same key with counter=3 + let block3 = create_test_batch(&[("k", 3, 30)]); + buffer + .process_data_block(&mut make_data_block(block3, "i3")) + .unwrap(); + + assert_eq!(buffer.size(), 1); + + buffer.set_base_file_iterator(vec![]); + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(records[0], ("k".to_string(), 3, 30)); // last write wins + } + + /// Mixed base + log + deletes scenario. + /// + /// Given: Base=[k1, k2, k3], log updates for k1 and k2, log delete for k3, log insert k4 + /// When: merge_and_collect() + /// Then: k1=updated, k2=updated, k3=deleted(absent), k4=inserted + #[test] + fn test_mixed_base_log_delete_insert() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Log: update k1, update k2 + let log_block = create_test_batch(&[("k1", 10, 100), ("k2", 20, 200)]); + buffer + .process_data_block(&mut make_data_block(log_block, "i1")) + .unwrap(); + + // Log: delete k3 (insert as delete record directly) + buffer.base.records.insert( + "k3".to_string(), + BufferedRecord::new_delete("k3".to_string(), None), + ); + + // Log: insert k4 (new key not in base) + let log_insert = create_test_batch(&[("k4", 40, 400)]); + buffer + .process_data_block(&mut make_data_block(log_insert, "i2")) + .unwrap(); + + // Base file + let base = create_test_batch(&[("k1", 1, 1), ("k2", 2, 2), ("k3", 3, 3)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + // k1=updated, k2=updated, k3=deleted(absent), k4=inserted + assert_eq!(result.num_rows(), 3); + assert_eq!(records[0], ("k1".to_string(), 10, 100)); + assert_eq!(records[1], ("k2".to_string(), 20, 200)); + assert_eq!(records[2], ("k4".to_string(), 40, 400)); + } + + /// Process next data record increments total_log_records. + #[test] + fn test_process_next_data_record_increments_counter() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + let block = create_test_batch(&[("k1", 1, 1), ("k2", 2, 2)]); + buffer + .process_data_block(&mut make_data_block(block, "i1")) + .unwrap(); + + assert_eq!(buffer.get_total_log_records(), 2); + } + + /// contains_log_record checks key existence. + #[test] + fn test_contains_log_record() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + let block = create_test_batch(&[("k1", 1, 1)]); + buffer + .process_data_block(&mut make_data_block(block, "i1")) + .unwrap(); + + assert!(buffer.contains_log_record("k1")); + assert!(!buffer.contains_log_record("k2")); + } + + /// Java: TestKeyBasedFileGroupRecordBuffer.readWithCommitTimeOrderingWithRecords + /// + /// COMMIT_TIME_ORDERING with pre-built records including deletes. + /// In commit-time mode, log always wins over base. + /// Expected: 5 output rows (keys 5,6 deleted). + #[test] + fn test_read_with_commit_time_ordering_with_records() { + let mut buffer = + build_key_based_buffer_with_delete_marker("COMMIT_TIME_ORDERING", "counter", "3"); + + // Data records (log updates) + let updates = create_test_batch(&[ + ("1", 2, 1), // record1UpdateWithSameTime + ("2", 1, 2), // record2Update + ("3", 1, 2), // record3Update + ("4", 1, 0), // record4EarlierUpdate + ("7", 1, 5), // record7 (log-only insert) + ]); + buffer + .process_data_block(&mut make_data_block(updates, "instant2")) + .unwrap(); + + // Delete records with default ordering (0) for commit-time mode + // In Java: convertToHoodieRecordsListForDeletes(..., true) → ordering=0 (default) + let delete_5 = DeleteRecord { + record_key: "5".to_string(), + partition_path: String::new(), + ordering_value: Some(OrderingValue::Long(0)), + }; + buffer.process_next_deleted_record(delete_5, "5"); + + let delete_6 = DeleteRecord { + record_key: "6".to_string(), + partition_path: String::new(), + ordering_value: Some(OrderingValue::Long(0)), + }; + buffer.process_next_deleted_record(delete_6, "6"); + + // Base file: testRecord1-6 + let base = create_test_batch(&[ + ("1", 1, 1), + ("2", 1, 1), + ("3", 1, 1), + ("4", 2, 1), // testRecord4 (counter=2) + ("5", 1, 1), + ("6", 1, 5), // testRecord6 (ts=5) + ]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + // Commit-time: log always wins over base + // key=1-4: log data wins → updated values + // key=5,6: log delete wins → DELETED + // key=7: log-only → INSERT + // Java: assertEquals(Stream.of(testRecord1UpdateWithSameTime, testRecord2Update, + // testRecord3Update, testRecord4EarlierUpdate, testRecord7), actualRecords); + assert_eq!(result.num_rows(), 5); + assert_eq!(records[0], ("1".to_string(), 2, 1)); + assert_eq!(records[1], ("2".to_string(), 1, 2)); + assert_eq!(records[2], ("3".to_string(), 1, 2)); + assert_eq!(records[3], ("4".to_string(), 1, 0)); + assert_eq!(records[4], ("7".to_string(), 1, 5)); + } + + // ========================================================================= + // Java: TestKeyBasedFileGroupRecordBuffer.readWithEventTimeOrderingAndDeleteBlock + // ========================================================================= + + /// Java: readWithEventTimeOrderingAndDeleteBlock + /// + /// Given: EVENT_TIME_ORDERING, ordering_field="ts" + /// Base: records 1,2,3 + /// DataBlock1: update key 1(ts=1), update key 2(ts=2), key 3(ts=1) + /// DeleteBlock: delete key 3(no ordering), delete key 2(ts=-1), delete key 1(ts=2) + /// DataBlock2: update key 2(ts=0, earlier), update key 3(ts=2) + /// When: Process all blocks, set base, merge + /// Then: After log scanning: + /// key 1: update(ts=1), then delete(ts=2) wins → DELETE + /// key 2: update(ts=2), then delete(ts=-1) loses, then earlier(ts=0) loses → keeps update(ts=2) + /// key 3: record(ts=1), then delete(no ordering) wins, then update(ts=2) wins → keeps update(ts=2) + /// Final: key1=deleted, key2=update(ts=2), key3=update(ts=2) + /// Output: 2 rows (key2, key3), 1 delete (key1 absent) + #[test] + fn test_read_with_event_time_ordering_and_delete_block() { + let mut buffer = build_key_based_buffer("EVENT_TIME_ORDERING"); + + // DataBlock1 + let block1 = create_test_batch(&[ + ("1", 2, 1), // key 1 update (ts=1) + ("2", 2, 2), // key 2 update (ts=2) + ("3", 1, 1), // key 3 (ts=1) + ]); + buffer + .process_data_block(&mut make_data_block(block1, "instant1")) + .unwrap(); + + // DeleteBlock: delete key "1" with ts=2 (wins over existing ts=1) + // For EVENT_TIME_ORDERING, delete with higher ts wins + let delete_record_1 = DeleteRecord { + record_key: "1".to_string(), + partition_path: String::new(), + ordering_value: Some(OrderingValue::Long(2)), + }; + buffer.process_next_deleted_record(delete_record_1, "1"); + + // Delete key "2" with ts=-1 (loses to existing ts=2) + let delete_record_2 = DeleteRecord { + record_key: "2".to_string(), + partition_path: String::new(), + ordering_value: Some(OrderingValue::Long(-1)), + }; + buffer.process_next_deleted_record(delete_record_2, "2"); + + // Delete key "3" with no ordering (wins by default) + let delete_record_3 = DeleteRecord { + record_key: "3".to_string(), + partition_path: String::new(), + ordering_value: None, + }; + buffer.process_next_deleted_record(delete_record_3, "3"); + + // DataBlock2 + let block2 = create_test_batch(&[ + ("2", 1, 0), // key 2 earlier update (ts=0, loses to ts=2) + ("3", 1, 2), // key 3 update (ts=2, wins over delete with no ordering) + ]); + buffer + .process_data_block(&mut make_data_block(block2, "instant2")) + .unwrap(); + + // Base file + let base = create_test_batch(&[("1", 1, 1), ("2", 1, 1), ("3", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + // key1: deleted (absent), key2: update(ts=2), key3: update(ts=2) + assert_eq!(result.num_rows(), 2, "Expected 2 rows: key1 deleted"); + assert_eq!(records[0], ("2".to_string(), 2, 2)); // key2 with ts=2 update + assert_eq!(records[1], ("3".to_string(), 1, 2)); // key3 with ts=2 update + } + + // ========================================================================= + // Running Map Pattern Tests (from FS_logBlockConsumption.md) + // + // Validates that the records map is a running accumulator: every record + // from every log block immediately mutates it in-place. + // ========================================================================= + + /// Full chain: single record → map update. + /// + /// Given: Empty buffer, COMMIT_TIME_ORDERING + /// When: process_data_block with 1 record (key="k1", counter=1, ts=10) + /// Then: records map contains exactly {"k1": BufferedRecord} + /// + /// Validates: processDataBlock → processNextDataRecord → records.put(key, record) + #[test] + fn test_running_map_single_record_updates_map() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + assert!(buffer.base.records.is_empty(), "Map should start empty"); + + let block = create_test_batch(&[("k1", 1, 10)]); + buffer + .process_data_block(&mut make_data_block(block, "instant1")) + .unwrap(); + + // Map now has exactly 1 entry + assert_eq!(buffer.base.records.len(), 1); + assert!(buffer.base.records.contains_key("k1")); + assert_eq!(buffer.base.total_log_records, 1); + } + + /// Running map: second record for same key overwrites first inline. + /// + /// Given: Buffer with k1=(counter=1,ts=10) already in map + /// When: process_data_block with (key="k1", counter=2, ts=20) + /// Then: Map still has 1 entry, but value is updated + /// + /// Validates: records.get(key) → deltaMerge(new, existing) → records.put(key, merged) + /// For COMMIT_TIME_ORDERING: new always wins regardless of ordering value. + #[test] + fn test_running_map_same_key_overwrites_inline() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // First record + let block1 = create_test_batch(&[("k1", 1, 10)]); + buffer + .process_data_block(&mut make_data_block(block1, "i1")) + .unwrap(); + assert_eq!(buffer.base.records.len(), 1); + + // Second record for same key — overwrites + let block2 = create_test_batch(&[("k1", 2, 20)]); + buffer + .process_data_block(&mut make_data_block(block2, "i2")) + .unwrap(); + + // Still 1 entry (same key), but value updated + assert_eq!(buffer.base.records.len(), 1); + assert_eq!(buffer.base.total_log_records, 2); + } + + /// Running map: accumulates across multiple blocks from different instants. + /// + /// Given: 3 blocks from instants i1, i2, i3 (processed oldest→newest) + /// Block i1: key=K with value v1 + /// Block i2: key=K with value v2 (overwrites v1) + /// Block i3: key=K with value v3 (overwrites v2) + /// When: Process blocks in order + /// Then: Final map: {"K": v3} (last writer wins) + /// total_log_records = 3 (all processed, even if overwritten) + /// + /// This validates the "running accumulator" pattern from the reference: + /// deltaMerge(t1_record, null) → records["K"] = t1_record + /// deltaMerge(t2_record, t1_record) → records["K"] = t2_record + /// deltaMerge(t3_record, t2_record) → records["K"] = t3_record + #[test] + fn test_running_map_across_three_instants() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Simulate oldest→newest processing order (as pollLast produces) + let block_i1 = create_test_batch(&[("K", 1, 10)]); // oldest + buffer + .process_data_block(&mut make_data_block(block_i1, "i1")) + .unwrap(); + assert_eq!(buffer.base.records.len(), 1); + + let block_i2 = create_test_batch(&[("K", 2, 20)]); + buffer + .process_data_block(&mut make_data_block(block_i2, "i2")) + .unwrap(); + assert_eq!(buffer.base.records.len(), 1); // still 1 key + + let block_i3 = create_test_batch(&[("K", 3, 30)]); // newest + buffer + .process_data_block(&mut make_data_block(block_i3, "i3")) + .unwrap(); + assert_eq!(buffer.base.records.len(), 1); + + // total_log_records counts ALL records processed, not just unique keys + assert_eq!(buffer.base.total_log_records, 3); + + // Verify final map state: newest wins + buffer.set_base_file_iterator(vec![]); + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + assert_eq!(records[0], ("K".to_string(), 3, 30)); // i3 value + } + + /// Running map: delete overwrites existing data record. + /// + /// Given: Buffer with k1=(counter=1,ts=10) in map + /// When: process_next_deleted_record for key "k1" + /// Then: Map entry for k1 is now a delete (is_delete=true) + /// When merged with base, k1 is absent from output. + #[test] + fn test_running_map_delete_overwrites_data() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Data record first + let block = create_test_batch(&[("k1", 1, 10)]); + buffer + .process_data_block(&mut make_data_block(block, "i1")) + .unwrap(); + assert!(!buffer.base.records["k1"].is_delete()); + + // Delete for same key + let delete = DeleteRecord { + record_key: "k1".to_string(), + partition_path: String::new(), + ordering_value: None, + }; + buffer.process_next_deleted_record(delete, "k1"); + + // Map entry is now a delete + assert!(buffer.base.records["k1"].is_delete()); + assert_eq!(buffer.base.total_log_records, 2); + + // When merged with base, k1 should be absent + let base = create_test_batch(&[("k1", 1, 1)]); + buffer.set_base_file_iterator(vec![base]); + let result = Box::new(buffer).merge_and_collect().unwrap(); + assert_eq!(result.num_rows(), 0, "Deleted key should be absent"); + } + + /// Full Phase A→B→C chain: 3 blocks, 2 keys, 1 delete. + /// + /// Given: COMMIT_TIME_ORDERING + /// Block i1 (oldest): key=A value=v1, key=B value=v1 + /// Block i2: key=A value=v2 (overwrites) + /// Block i3 (newest): delete key=B + /// Base file: key=A base_val, key=B base_val, key=C base_val + /// When: Full scan + merge + /// Then: Phase A map: {A: v2, B: DELETE} + /// Phase B: A → merge(base, v2) → emit v2 + /// B → merge(base, DELETE) → absent + /// C → no log entry → emit base + /// Phase C: nothing remaining + /// Output: A=v2, C=base_val (2 rows) + #[test] + fn test_full_chain_phase_a_b_c() { + let mut buffer = build_key_based_buffer("COMMIT_TIME_ORDERING"); + + // Phase A: process log blocks oldest→newest + let block_i1 = create_test_batch(&[("A", 1, 10), ("B", 1, 10)]); + buffer + .process_data_block(&mut make_data_block(block_i1, "i1")) + .unwrap(); + + let block_i2 = create_test_batch(&[("A", 2, 20)]); + buffer + .process_data_block(&mut make_data_block(block_i2, "i2")) + .unwrap(); + + // Delete B + let delete_b = DeleteRecord { + record_key: "B".to_string(), + partition_path: String::new(), + ordering_value: None, + }; + buffer.process_next_deleted_record(delete_b, "B"); + + // Verify Phase A map state + assert_eq!(buffer.base.records.len(), 2); // A and B + assert!(!buffer.base.records["A"].is_delete()); + assert!(buffer.base.records["B"].is_delete()); + + // Phase B + C + let base = create_test_batch(&[("A", 0, 0), ("B", 0, 0), ("C", 9, 99)]); + buffer.set_base_file_iterator(vec![base]); + + let result = Box::new(buffer).merge_and_collect().unwrap(); + let records = extract_records(&result); + + assert_eq!(result.num_rows(), 2, "A=updated, B=deleted, C=base"); + assert_eq!(records[0], ("A".to_string(), 2, 20)); // log wins + assert_eq!(records[1], ("C".to_string(), 9, 99)); // base passthrough + } +} diff --git a/crates/core/src/file_group/reader/buffer/loader.rs b/crates/core/src/file_group/reader/buffer/loader.rs new file mode 100644 index 00000000..0fa30dd1 --- /dev/null +++ b/crates/core/src/file_group/reader/buffer/loader.rs @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors: +//! - `FileGroupRecordBufferLoader` (Java interface) +//! - `LogScanningRecordBufferLoader` (Java abstract class) +//! - `DefaultFileGroupRecordBufferLoader` (Java impl) +//! +//! ## Call stack (matching Java 1:1): +//! ```text +//! DefaultFileGroupRecordBufferLoader.getRecordBuffer() +//! ├─ new KeyBasedFileGroupRecordBuffer(...) +//! └─ scanLogFiles(readerContext, storage, inputSplit, ..., recordBuffer) +//! └─ HoodieMergedLogRecordReader.newBuilder()...build() +//! └─ performScan() +//! └─ BaseHoodieLogRecordReader.scanInternal() +//! ``` + +use crate::Result; +use crate::file_group::reader::buffer::HoodieFileGroupRecordBuffer; +use crate::file_group::reader::buffer::key_based::KeyBasedFileGroupRecordBuffer; +use crate::file_group::reader::input_split::InputSplit; +use crate::file_group::reader::merged_log_record_reader::HoodieMergedLogRecordReader; +use crate::file_group::reader::read_stats::HoodieReadStats; +use crate::file_group::reader::reader_context::ReaderContext; +use crate::file_group::reader::reader_parameters::ReaderParameters; +use crate::storage::Storage; +use std::sync::Arc; + +/// Result of loading a record buffer. +/// +/// Mirrors Java's `Pair, List>`. +pub struct RecordBufferLoadResult { + pub record_buffer: Box, + pub valid_block_instants: Vec, +} + +/// Trait for loading file group record buffers. +/// +/// Mirrors Java's `FileGroupRecordBufferLoader` interface. +pub trait FileGroupRecordBufferLoader: Send + Sync + std::fmt::Debug { + /// Create and populate a record buffer for the given input split. + /// + /// Mirrors Java's `getRecordBuffer(...)`. + fn get_record_buffer( + &self, + reader_context: Arc, + storage: Arc, + input_split: &InputSplit, + reader_parameters: &ReaderParameters, + read_stats: &mut HoodieReadStats, + ) -> impl std::future::Future> + Send; +} + +/// Default file group record buffer loader. +/// +/// Mirrors Java's `DefaultFileGroupRecordBufferLoader` (singleton). +/// Extends `LogScanningRecordBufferLoader` (the `scan_log_files` method). +/// Implements `FileGroupRecordBufferLoader`. +/// +/// ## Buffer strategy selection: +/// ```text +/// is_skip_merge? → UnmergedFileGroupRecordBuffer (not implemented) +/// sort_outputs? → SortedKeyBasedFileGroupRecordBuffer (not implemented) +/// use_record_position && base_file? → PositionBasedFileGroupRecordBuffer (not impl) +/// DEFAULT → KeyBasedFileGroupRecordBuffer ★ +/// ``` +#[derive(Debug)] +pub struct DefaultFileGroupRecordBufferLoader; + +impl DefaultFileGroupRecordBufferLoader { + pub fn new() -> Self { + Self + } +} + +impl FileGroupRecordBufferLoader for DefaultFileGroupRecordBufferLoader { + /// Mirrors Java's `DefaultFileGroupRecordBufferLoader.getRecordBuffer(...)`. + /// + /// Steps: + /// 1. Create UpdateProcessor + /// 2. Instantiate buffer (strategy selection — only KeyBased for now) + /// 3. Call `scanLogFiles()` to populate the buffer + async fn get_record_buffer( + &self, + reader_context: Arc, + storage: Arc, + input_split: &InputSplit, + reader_parameters: &ReaderParameters, + read_stats: &mut HoodieReadStats, + ) -> Result { + // Use merge mode from reader context directly (mirrors Java: readerContext.getMergeMode()). + let merge_mode = if reader_context.merge_mode.is_empty() { + "COMMIT_TIME_ORDERING".to_string() + } else { + reader_context.merge_mode.clone() + }; + + // Guard: only COMMIT_TIME_ORDERING and EVENT_TIME_ORDERING are supported. + // CUSTOM merge mode requires partial-update / custom merger which is not yet implemented. + match merge_mode.as_str() { + "COMMIT_TIME_ORDERING" | "EVENT_TIME_ORDERING" => {} + unsupported => { + return Err(crate::error::CoreError::ReadFileSliceError(format!( + "Unsupported merge mode: '{unsupported}'. \ + Only COMMIT_TIME_ORDERING and EVENT_TIME_ORDERING are supported." + ))); + } + } + + log::debug!( + "[DefaultFileGroupRecordBufferLoader] getRecordBuffer: merge_mode={merge_mode} \ + record_key_field={} ordering_fields={:?} \ + log_files={} latest_commit_time={}", + reader_context.record_key_field(), + reader_context.ordering_field_names(), + input_split.log_file_paths.len(), + reader_context.latest_commit_time, + ); + + // STEP: Instantiate buffer (strategy selection) + // Mirrors Java's DefaultFileGroupRecordBufferLoader.getRecordBuffer() lines 67-80. + let is_skip_merge = reader_context + .hoodie_reader_config + .get("hoodie.datasource.merge.type") + .map(|v| v.eq_ignore_ascii_case("skip_merge")) + .unwrap_or(false); + + let record_buffer: Box = if is_skip_merge { + return Err(crate::error::CoreError::Unsupported( + "UnmergedFileGroupRecordBuffer (skip_merge mode) is not yet implemented" + .to_string(), + )); + } else if reader_parameters.sort_output { + return Err(crate::error::CoreError::Unsupported( + "SortedKeyBasedFileGroupRecordBuffer (sort_output mode) is not yet implemented" + .to_string(), + )); + } else if reader_parameters.use_record_position && input_split.base_file_path.is_some() { + return Err(crate::error::CoreError::Unsupported( + "PositionBasedFileGroupRecordBuffer (position-based merge) is not yet implemented" + .to_string(), + )); + } else { + Box::new(KeyBasedFileGroupRecordBuffer::new( + reader_context.clone(), + merge_mode, + read_stats, + )?) + }; + + // STEP: scanLogFiles — build and run HoodieMergedLogRecordReader + let (populated_buffer, valid_block_instants, stats) = scan_log_files( + reader_context, + storage, + input_split, + record_buffer, + reader_parameters, + ) + .await?; + + // Populate read stats from scan stats + read_stats.total_log_read_time_ms = stats.total_time_taken_to_read_and_merge_blocks_ms; + read_stats.total_log_records = stats.total_log_records; + read_stats.total_log_blocks = stats.total_log_blocks; + read_stats.total_log_files_compacted = stats.total_log_files; + read_stats.total_corrupt_log_blocks = stats.total_corrupt_blocks; + read_stats.total_rollback_blocks = stats.total_rollbacks; + + Ok(RecordBufferLoadResult { + record_buffer: populated_buffer, + valid_block_instants, + }) + } +} + +/// Scan log files and populate the record buffer. +/// +/// Mirrors Java's `LogScanningRecordBufferLoader.scanLogFiles(...)`. +/// +/// Builds a `HoodieMergedLogRecordReader` via builder, which calls +/// `performScan()` → `scanInternal()` during construction. +async fn scan_log_files( + reader_context: Arc, + storage: Arc, + input_split: &InputSplit, + record_buffer: Box, + reader_parameters: &ReaderParameters, +) -> Result<( + Box, + Vec, + crate::file_group::reader::merged_log_record_reader::ScanStats, +)> { + if !input_split.has_log_files() { + let stats = crate::file_group::reader::merged_log_record_reader::ScanStats::default(); + return Ok((record_buffer, Vec::new(), stats)); + } + + // Mirrors Java: + // HoodieMergedLogRecordReader.newBuilder() + // .withHoodieReaderContext(readerContext) + // .withStorage(storage) + // .withLogFiles(inputSplit.getLogFiles()) + // .withInstantRange(readerContext.getInstantRange()) + // .withRecordBuffer(recordBuffer) + // .withAllowInflightInstants(readerParameters.allowInflightInstants()) + // .build() + let latest_instant_time = reader_context.latest_commit_time.clone(); + let reader = HoodieMergedLogRecordReader::new_builder() + .with_reader_context(reader_context) + .with_storage(storage) + .with_log_files(input_split.log_file_paths.clone()) + .with_latest_instant_time(latest_instant_time) + .with_record_buffer(record_buffer) + .with_allow_inflight_instants(reader_parameters.allow_inflight_instants) + .with_force_full_scan(true) + .build() + .await?; + + // Decompose: get populated buffer + stats + Ok(reader.into_parts()) +} diff --git a/crates/core/src/file_group/reader/buffer/mod.rs b/crates/core/src/file_group/reader/buffer/mod.rs new file mode 100644 index 00000000..923f9eb7 --- /dev/null +++ b/crates/core/src/file_group/reader/buffer/mod.rs @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Record buffer hierarchy for the file group reader. +//! +//! Mirrors the Java package `org.apache.hudi.common.table.read.buffer`. +//! +//! ## Interface Hierarchy (matches Java 1:1) +//! +//! ```text +//! «trait» HoodieFileGroupRecordBuffer (Java interface) +//! │ +//! │ implements +//! ▼ +//! FileGroupRecordBuffer (common state struct) (Java abstract class) +//! │ +//! ├── KeyBasedFileGroupRecordBuffer (KEY_BASED_MERGE, default) +//! ├── [PositionBasedFileGroupRecordBuffer] (future) +//! ├── [SortedKeyBasedFileGroupRecordBuffer] (future) +//! └── [UnmergedFileGroupRecordBuffer] (future) +//! ``` + +pub mod key_based; +pub mod loader; +pub mod record_buffer; +pub mod row_extraction; + +pub use key_based::KeyBasedFileGroupRecordBuffer; +pub use loader::{DefaultFileGroupRecordBufferLoader, FileGroupRecordBufferLoader}; +pub use record_buffer::FileGroupRecordBuffer; + +use crate::Result; +use crate::file_group::log_file::log_block::LogBlock; +use crate::file_group::reader::buffered_record::{BufferedRecord, DeleteRecord}; +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; +use std::collections::HashMap; + +/// The type of merge buffer in use. +/// +/// Mirrors Java's `HoodieFileGroupRecordBuffer.BufferType`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BufferType { + /// Key-based merge: deduplicates by record key, keeps latest by ordering value. + KeyBasedMerge, + /// Position-based merge (not yet implemented). + PositionBasedMerge, + /// Unmerged: skips merge (not yet implemented). + Unmerged, +} + +/// Trait for file group record buffers. +/// +/// Mirrors Java's `HoodieFileGroupRecordBuffer` interface. +/// +/// Method names match Java 1:1: +/// - `processDataBlock` → `process_data_block` +/// - `processNextDataRecord` → `process_next_data_record` +/// - `processDeleteBlock` → `process_delete_block` +/// - `processNextDeletedRecord` → `process_next_deleted_record` +/// - `containsLogRecord` → `contains_log_record` +/// - `setBaseFileIterator` → `set_base_file_iterator` +/// - `hasNext` / `next` → `has_next` / `next` +pub trait HoodieFileGroupRecordBuffer: Send + Sync + std::fmt::Debug { + /// Returns the buffer type. + fn get_buffer_type(&self) -> BufferType; + + /// Process a data block from log scanning. + /// + /// Mirrors Java's `processDataBlock(HoodieDataBlock, Option)`. + /// The buffer is responsible for inflating the block and extracting records, + /// matching Java where inflate/deserialize/deflate happens inside the block + /// triggered by the buffer's `getRecordsIterator`. + fn process_data_block(&mut self, block: &mut LogBlock) -> Result<()>; + + /// Process a single data record within a data block. + /// + /// Mirrors Java's `processNextDataRecord(BufferedRecord, Serializable)`. + fn process_next_data_record(&mut self, record: BufferedRecord, key: &str) -> Result<()>; + + /// Process a delete block from log scanning. + /// + /// Mirrors Java's `processDeleteBlock(HoodieDeleteBlock)`. + /// The buffer inflates the block and extracts delete records internally. + fn process_delete_block(&mut self, block: &mut LogBlock) -> Result<()>; + + /// Process a single deleted record within a delete block. + /// + /// Mirrors Java's `processNextDeletedRecord(DeleteRecord, Serializable)`. + fn process_next_deleted_record(&mut self, delete_record: DeleteRecord, key: &str); + + /// Check if a record exists in the buffered records. + /// + /// Mirrors Java's `containsLogRecord(String)`. + fn contains_log_record(&self, record_key: &str) -> bool; + + /// Returns the number of records in the buffer. + fn size(&self) -> usize; + + /// Returns the total number of log records processed. + fn get_total_log_records(&self) -> u64; + + /// Returns the underlying records map. + fn get_log_records(&self) -> &HashMap; + + /// Set the reader schema for merge output. + /// + /// This schema is used as a fallback when no base file is present + /// and the first record in the buffer is a delete (which has no schema). + /// Mirrors Java's readerSchema from FileGroupReaderSchemaHandler. + fn set_reader_schema(&mut self, schema: SchemaRef); + + /// Set the base file batches for merge iteration. + /// + /// Mirrors Java's `setBaseFileIterator(ClosableIterator)`. + fn set_base_file_iterator(&mut self, batches: Vec); + + /// Check if next merged record exists. + /// + /// Mirrors Java's `hasNext()`. + fn has_next(&mut self) -> Result; + + /// Return the next merged buffered record. + /// + /// Mirrors Java's `next()`. + fn next(&mut self) -> Option; + + /// Set the key-spec filter. Records whose key doesn't match are skipped + /// during `process_data_block` / `process_delete_block`. Mirrors how + /// Java's BaseHoodieLogRecordReader threads keySpecOpt through to + /// per-record processing. Default: no-op for buffers that don't support filtering. + fn set_key_spec(&mut self, _key_spec: Option) {} + + /// Consume the buffer and produce the merged output as a `RecordBatch`. + /// + /// This is Rust-specific — Java uses the `hasNext()`/`next()` iterator instead. + fn merge_and_collect(self: Box) -> Result; +} diff --git a/crates/core/src/file_group/reader/buffer/record_buffer.rs b/crates/core/src/file_group/reader/buffer/record_buffer.rs new file mode 100644 index 00000000..9105f6de --- /dev/null +++ b/crates/core/src/file_group/reader/buffer/record_buffer.rs @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.buffer.FileGroupRecordBuffer`. +//! +//! Common state for all record buffer variants. In Java this is an abstract class +//! with fields for the records map (ExternalSpillableMap), merger, delete context, +//! update processor, base file iterator, and total log records. +//! +//! In Rust, this is a struct used via composition by concrete implementations +//! (e.g. `KeyBasedFileGroupRecordBuffer`). + +use crate::Result; +use crate::file_group::reader::buffered_record::BufferedRecord; +use crate::file_group::reader::delete_context::DeleteContext; +use crate::file_group::reader::record_merger::BufferedRecordMerger; +use crate::file_group::reader::update_processor::UpdateProcessor; +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; +use std::collections::HashMap; +use std::vec::IntoIter; + +/// Common state for all file group record buffer implementations. +/// +/// Mirrors Java's `FileGroupRecordBuffer` abstract class. +/// +/// ## Key fields (matching Java): +/// - `records` — Java's `ExternalSpillableMap>` +/// (in Rust: `HashMap` — no disk spilling for Phase 1) +/// - `buffered_record_merger` — for delta merge (log-vs-log) and final merge (base-vs-log) +/// - `update_processor` — strategy for processing updates during merge iteration +/// - `base_file_batches` / iteration state — Java's `baseFileIterator` +/// - `next_record` — Java's `nextRecord` (the lookahead for has_next/next pattern) +#[derive(Debug)] +pub struct FileGroupRecordBuffer { + /// The per-key records map. Mirrors Java's `ExternalSpillableMap`. + pub records: HashMap, + + /// The reader schema. + pub reader_schema: Option, + + /// The merge mode string (e.g. "COMMIT_TIME_ORDERING"). + pub record_merge_mode: String, + + /// The record merger for resolving conflicts. + pub buffered_record_merger: Box, + + /// Context for detecting delete records. + pub delete_context: Option, + + /// Processor for update records during merge iteration. + pub update_processor: Box, + + /// Base file record batches (set after log scanning, before merge iteration). + pub base_file_batches: Vec, + + /// Total number of log records processed. + pub total_log_records: u64, + + /// Lookahead record for the has_next/next iteration pattern. + /// Mirrors Java's `nextRecord` field. + pub next_record: Option, + + // ── Base file iteration state ────────────────────────────────────── + /// Current batch index in base_file_batches. + pub base_batch_idx: usize, + /// Current row index within the current base file batch. + pub base_row_idx: usize, + + // ── Log record iteration state ───────────────────────────────────── + /// Drain iterator for remaining log records (created lazily on first call to has_next_log_record). + pub log_record_iter: Option>, +} + +impl FileGroupRecordBuffer { + pub fn new( + record_merge_mode: String, + buffered_record_merger: Box, + update_processor: Box, + reader_schema: Option, + ) -> Self { + Self { + records: HashMap::new(), + reader_schema, + record_merge_mode, + buffered_record_merger, + delete_context: None, + update_processor, + base_file_batches: Vec::new(), + total_log_records: 0, + next_record: None, + base_batch_idx: 0, + base_row_idx: 0, + log_record_iter: None, + } + } + + /// Template method: check if next merged record exists. + /// + /// Mirrors Java's `hasNext()`: `nextRecord != null || doHasNext()` + pub fn has_next(&mut self, do_has_next: F) -> Result + where + F: FnOnce(&mut Self) -> Result, + { + if self.next_record.is_some() { + return Ok(true); + } + do_has_next(self) + } + + /// Return the next merged buffered record. + /// + /// Mirrors Java's `next()`: take and return nextRecord. + pub fn next(&mut self) -> Option { + self.next_record.take() + } + + /// Merge a base record with its corresponding log record (if any). + /// + /// Mirrors Java's `hasNextBaseRecord(T baseRecord, BufferedRecord logRecordInfo)`. + /// + /// If `log_record_info` is Some, does a final merge (base vs log). + /// If None, the base record is an insert (no log record for this key). + pub fn has_next_base_record( + &mut self, + base_record: &BufferedRecord, + log_record_info: Option<&BufferedRecord>, + ) -> Result { + if let Some(log_record) = log_record_info { + // Merge base + log + let merge_result = self + .buffered_record_merger + .final_merge(base_record, log_record)?; + let processed = self.update_processor.process_update( + &log_record.record_key, + Some(base_record), + &merge_result, + merge_result.is_delete(), + )?; + self.next_record = processed; + return Ok(self.next_record.is_some()); + } + + // Insert: base record only, no log record + self.next_record = Some(base_record.clone()); + Ok(true) + } + + /// Iterate remaining log records not consumed by base file iteration. + /// + /// Mirrors Java's `hasNextLogRecord()`. + pub fn has_next_log_record(&mut self) -> bool { + if self.log_record_iter.is_none() { + // Drain remaining records from the map + let remaining: Vec = self.records.drain().map(|(_, v)| v).collect(); + self.log_record_iter = Some(remaining.into_iter()); + } + + let iter = self.log_record_iter.as_mut().unwrap(); + while let Some(record) = iter.next() { + let processed = self.update_processor.process_update( + &record.record_key, + None, + &record, + record.is_delete(), + ); + match processed { + Ok(Some(r)) => { + self.next_record = Some(r); + return true; + } + Ok(None) => continue, + Err(_) => return false, + } + } + false + } + + /// Check if there are more rows to iterate in the base file batches. + pub fn has_more_base_rows(&self) -> bool { + if self.base_batch_idx >= self.base_file_batches.len() { + return false; + } + if self.base_batch_idx == self.base_file_batches.len() - 1 { + return self.base_row_idx < self.base_file_batches[self.base_batch_idx].num_rows(); + } + true + } + + /// Get the next base file row as a single-row RecordBatch and advance the position. + pub fn next_base_row(&mut self) -> Option { + while self.base_batch_idx < self.base_file_batches.len() { + let batch = &self.base_file_batches[self.base_batch_idx]; + if self.base_row_idx < batch.num_rows() { + let row = batch.slice(self.base_row_idx, 1); + self.base_row_idx += 1; + return Some(row); + } + self.base_batch_idx += 1; + self.base_row_idx = 0; + } + None + } +} diff --git a/crates/core/src/file_group/reader/buffer/row_extraction.rs b/crates/core/src/file_group/reader/buffer/row_extraction.rs new file mode 100644 index 00000000..70ed701b --- /dev/null +++ b/crates/core/src/file_group/reader/buffer/row_extraction.rs @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Utility functions to bridge Arrow columnar data with per-record operations. +//! +//! Record-level operations (key extraction, ordering value extraction, batch-to-record +//! conversion) have moved to [`RecordContext`](crate::file_group::reader::record_context::RecordContext). +//! +//! This module retains Arrow batch-level utilities that don't need RecordContext: +//! - `slice_row` — zero-copy single-row extraction +//! - `records_to_batch` — reassemble BufferedRecords into a RecordBatch +//! - `reconcile_batch_to_schema` — handle Avro/Parquet schema differences + +use crate::Result; +use crate::error::CoreError; +use crate::file_group::reader::buffered_record::BufferedRecord; +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_schema::SchemaRef; + +/// Slice a single row from a RecordBatch as a new single-row RecordBatch. +/// +/// This is zero-copy (just adjusts array offsets in Arrow). +pub fn slice_row(batch: &RecordBatch, row_index: usize) -> RecordBatch { + batch.slice(row_index, 1) +} + +/// Reassemble a collection of BufferedRecords back into a single RecordBatch. +/// +/// Concatenates all single-row batches from the records into one batch. +/// Records without data (deletes) are skipped. +pub fn records_to_batch(records: Vec, schema: SchemaRef) -> Result { + // Mirrors Java: BufferedRecord.getRecord() unwraps from binary if needed + let batches: Vec = records.iter().filter_map(|r| r.get_record()).collect(); + + if batches.is_empty() { + return Ok(RecordBatch::new_empty(schema)); + } + + // Reconcile schemas: if a batch's schema differs from the target (e.g., Avro + // log records vs Parquet base records have different field names for Map/List + // children), rebuild each batch under the target schema. + let reconciled: Vec = batches + .into_iter() + .map(|b| { + if b.schema() == schema { + b + } else { + reconcile_batch_to_schema(&b, &schema) + } + }) + .collect(); + + let batch_refs: Vec<&RecordBatch> = reconciled.iter().collect(); + arrow::compute::concat_batches(&schema, batch_refs.into_iter()) + .map_err(|e| CoreError::ReadFileSliceError(format!("Failed to concat record batches: {e}"))) +} + +/// Reconcile a RecordBatch to a target schema. +/// +/// Handles field name differences between Avro-derived schemas (from log files) and +/// Parquet-derived schemas (from base files). For example: +/// - List child field: Avro uses "element", Parquet uses "array" +/// - Map entries field: Avro uses "key_value", Parquet uses column name +/// +/// Rebuilds each column's ArrayData with the target schema's field metadata. +fn reconcile_batch_to_schema(batch: &RecordBatch, target_schema: &SchemaRef) -> RecordBatch { + let columns: Vec = target_schema + .fields() + .iter() + .map(|target_field| { + let idx = batch + .schema() + .index_of(target_field.name()) + .expect("column name must exist in source batch"); + let source_col = batch.column(idx); + // If types match exactly, use as-is + if source_col.data_type() == target_field.data_type() { + source_col.clone() + } else { + // Try arrow_cast first; if it fails, rebuild the ArrayData + // with the target DataType (changes field names but not actual data) + arrow_cast::cast(source_col, target_field.data_type()).unwrap_or_else(|_| { + let source_data = source_col.to_data(); + let rebuilt = source_data + .into_builder() + .data_type(target_field.data_type().clone()) + .build(); + match rebuilt { + Ok(data) => arrow_array::make_array(data), + Err(_) => source_col.clone(), + } + }) + } + }) + .collect(); + RecordBatch::try_new(target_schema.clone(), columns).unwrap_or_else(|_| batch.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_group::reader::record_context::RecordContext; + use arrow_array::{Float64Array, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use std::sync::Arc; + + fn make_test_batch(num_rows: usize) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("name", DataType::Utf8, true), + Field::new("value", DataType::Float64, false), + ])); + + let keys: Vec = (0..num_rows).map(|i| format!("key_{i}")).collect(); + let names: Vec> = (0..num_rows).map(|i| Some(format!("name_{i}"))).collect(); + let values: Vec = (0..num_rows).map(|i| i as f64).collect(); + + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(keys)), + Arc::new(StringArray::from(names)), + Arc::new(Float64Array::from(values)), + ], + ) + .unwrap() + } + + #[test] + fn test_slice_row() { + let batch = make_test_batch(3); + let row = slice_row(&batch, 1); + assert_eq!(row.num_rows(), 1); + let ctx = RecordContext::default(); + let keys = ctx.get_record_keys(&row).unwrap(); + assert_eq!(keys, vec!["key_1"]); + } + + #[test] + fn test_records_to_batch_roundtrip() { + let batch = make_test_batch(3); + let schema = batch.schema(); + let ctx = RecordContext::default(); + let records = ctx.batch_to_buffered_records(&batch, None).unwrap(); + let buffered: Vec = records.into_iter().map(|(_, r)| r).collect(); + + let result = records_to_batch(buffered, schema.clone()).unwrap(); + assert_eq!(result.num_rows(), 3); + assert_eq!(result.schema(), schema); + } + + #[test] + fn test_records_to_batch_empty() { + let schema = make_test_batch(0).schema(); + let result = records_to_batch(vec![], schema.clone()).unwrap(); + assert_eq!(result.num_rows(), 0); + } +} diff --git a/crates/core/src/file_group/reader/buffered_record.rs b/crates/core/src/file_group/reader/buffered_record.rs new file mode 100644 index 00000000..ce4c3973 --- /dev/null +++ b/crates/core/src/file_group/reader/buffered_record.rs @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.BufferedRecord` and +//! `org.apache.hudi.common.model.DeleteRecord`. +//! +//! In Java Hudi, `BufferedRecord` wraps a single engine-native record +//! with its key, ordering value, and operation type. In hudi-rs, the +//! "engine record" is an Arrow `RecordBatch`. +//! +//! During log scanning, records are accumulated into the record buffer. +//! At read time, base file records are merged with log records to produce +//! the final output. + +use arrow_array::RecordBatch; + +use super::record_context::RecordContext; + +/// The universal record envelope flowing through the merge pipeline. +/// +/// Mirrors Java's `BufferedRecord`. In Rust/Arrow, a single +/// "record" is represented as a single-row `RecordBatch` when needed +/// for per-record operations, or as a multi-row batch for bulk operations. +/// +/// During the log scanning phase, `BufferedRecord`s are stored in the +/// record buffer's map keyed by record key. +#[derive(Debug, Clone)] +pub struct BufferedRecord { + /// The record key extracted from the record. + pub record_key: String, + + /// The engine-native record data (Arrow RecordBatch). + /// `None` for delete records, or after `to_binary()` has been called. + pub data: Option, + + /// The binary form of the record data (Arrow IPC bytes). + /// Populated by `to_binary()`, consumed by `get_record()`. + /// Mirrors Java's binary representation after `toBinary(RecordContext)`. + pub binary_data: Option>, + + /// The ordering value used for merge conflict resolution. + /// Higher ordering value wins during delta merge. + pub ordering_value: Option, + + /// Whether this record represents a deletion. + pub is_delete: bool, +} + +impl BufferedRecord { + /// Create a new data record. + pub fn new_data( + record_key: String, + data: RecordBatch, + ordering_value: Option, + ) -> Self { + Self { + record_key, + data: Some(data), + binary_data: None, + ordering_value, + is_delete: false, + } + } + + /// Create a new delete record. + pub fn new_delete(record_key: String, ordering_value: Option) -> Self { + Self { + record_key, + data: None, + binary_data: None, + ordering_value, + is_delete: true, + } + } + + /// Returns true if this record represents a deletion. + pub fn is_delete(&self) -> bool { + self.is_delete + } + + /// Returns true if this record has no data payload (neither batch nor binary). + pub fn is_empty(&self) -> bool { + self.data.is_none() && self.binary_data.is_none() + } + + /// Convert the record data to binary format for compact storage. + /// + /// Mirrors Java's `BufferedRecord.toBinary(RecordContext recordContext)`. + /// + /// ```java + /// public BufferedRecord toBinary(RecordContext recordContext) { + /// if (record != null) { + /// HoodieSchema schema = recordContext.getSchemaFromBufferRecord(this); + /// if (schema != null) { + /// record = recordContext.seal(recordContext.toBinaryRow(schema, record)); + /// } + /// } + /// return this; + /// } + /// ``` + /// + /// For Arrow: serializes the `RecordBatch` to IPC bytes via + /// `RecordContext::to_binary_row()` + `RecordContext::seal()`, + /// stores in `binary_data`, clears `data`. + pub fn to_binary(&mut self, _record_context: &RecordContext) -> &mut Self { + if self.data.is_some() { + let schema = RecordContext::get_schema_from_buffer_record(self); + if let Some(ref schema) = schema { + let batch = self.data.take().unwrap(); + let bytes = RecordContext::to_binary_row(schema, &batch); + self.binary_data = Some(RecordContext::seal(bytes)); + } + } + self + } + + /// Return the record data, deserializing from binary if needed. + /// + /// Mirrors Java's `BufferedRecord.getRecord()`. + /// + /// In Java, `getRecord()` returns the record as-is (UnsafeRow IS InternalRow). + /// In Rust/Arrow, after `to_binary()` we must explicitly deserialize IPC bytes + /// back to `RecordBatch`. + pub fn get_record(&self) -> Option { + if let Some(ref batch) = self.data { + return Some(batch.clone()); + } + if let Some(ref bytes) = self.binary_data { + return RecordContext::from_binary(bytes).ok(); + } + None + } +} + +/// Comparable ordering value for merge conflict resolution. +/// +/// In Java Hudi, this is `Comparable`. In Rust, we support +/// the common types used as precombine fields. +#[derive(Debug, Clone, PartialEq)] +pub enum OrderingValue { + Long(i64), + String(String), +} + +impl PartialOrd for OrderingValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Eq for OrderingValue {} + +impl Ord for OrderingValue { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + match (self, other) { + (OrderingValue::Long(a), OrderingValue::Long(b)) => a.cmp(b), + (OrderingValue::String(a), OrderingValue::String(b)) => a.cmp(b), + // Cross-type comparison: treat Long < String (arbitrary but consistent) + (OrderingValue::Long(_), OrderingValue::String(_)) => std::cmp::Ordering::Less, + (OrderingValue::String(_), OrderingValue::Long(_)) => std::cmp::Ordering::Greater, + } + } +} + +/// Immutable value object representing a record to be deleted. +/// +/// Mirrors Java's `org.apache.hudi.common.model.DeleteRecord`. +/// Created from `HoodieDeleteBlock` during log scanning. +#[derive(Debug, Clone)] +pub struct DeleteRecord { + /// The record key to delete. + pub record_key: String, + + /// The partition path. + pub partition_path: String, + + /// Ordering value for merge conflict resolution with existing records. + pub ordering_value: Option, +} + +/// Factory methods for creating `BufferedRecord` instances. +/// +/// Mirrors Java's `org.apache.hudi.common.table.read.BufferedRecords` (static factory). +pub struct BufferedRecords; + +impl BufferedRecords { + /// Create a `BufferedRecord` from an engine record (Arrow RecordBatch). + /// + /// Extracts the record key and ordering value from the batch using the + /// provided schema and field names. + pub fn from_engine_record( + record_key: String, + data: RecordBatch, + ordering_value: Option, + is_delete: bool, + ) -> BufferedRecord { + if is_delete { + BufferedRecord::new_delete(record_key, ordering_value) + } else { + BufferedRecord::new_data(record_key, data, ordering_value) + } + } + + /// Create a `BufferedRecord` from a `DeleteRecord`. + /// + /// Produces a `BufferedRecord` with `data=None` and `is_delete=true`. + pub fn from_delete_record(delete_record: &DeleteRecord) -> BufferedRecord { + BufferedRecord::new_delete( + delete_record.record_key.clone(), + delete_record.ordering_value.clone(), + ) + } +} diff --git a/crates/core/src/file_group/reader/buffered_record_converter.rs b/crates/core/src/file_group/reader/buffered_record_converter.rs new file mode 100644 index 00000000..ea62d94e --- /dev/null +++ b/crates/core/src/file_group/reader/buffered_record_converter.rs @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.BufferedRecordConverter`. +//! +//! Converts engine-specific records (`RecordBatch` in Rust / `T` in Java) +//! into [`BufferedRecord`] according to different [`IteratorMode`]s. + +use crate::Result; +use crate::file_group::reader::buffered_record::BufferedRecord; +use arrow_array::RecordBatch; + +/// Converter from engine records to [`BufferedRecord`]. +/// +/// Mirrors Java's `BufferedRecordConverter` functional interface. +/// +/// Implementations differ by [`super::iterator_mode::IteratorMode`]: +/// - `EngineRecord`: wraps the record directly (reuses a `BufferedRecord` envelope). +/// - `RecordKey`: extracts only the record key. +/// - `HoodieRecord`: full extraction with ordering value. +pub trait BufferedRecordConverter: Send + Sync + std::fmt::Debug { + /// Convert an engine record into a [`BufferedRecord`]. + /// + /// Mirrors Java's `BufferedRecord convert(T record)`. + fn convert(&self, record: RecordBatch) -> Result; +} diff --git a/crates/core/src/file_group/reader/delete_context.rs b/crates/core/src/file_group/reader/delete_context.rs new file mode 100644 index 00000000..1fe213dc --- /dev/null +++ b/crates/core/src/file_group/reader/delete_context.rs @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.DeleteContext`. +//! +//! Knows how to detect delete operations from schema metadata and +//! table properties. Constructed in two phases: +//! 1. `new(properties, table_schema)` — in the schema handler +//! 2. `with_reader_schema(reader_schema)` — in the record buffer + +use arrow_schema::SchemaRef; +use std::collections::HashMap; + +/// Context for detecting delete records in the merge pipeline. +/// +/// Built from table properties and schema. Tells the buffer +/// "how do I detect a delete in a record?" +/// +/// ## Java counterpart +/// `org.apache.hudi.common.table.read.DeleteContext` +/// +/// ## Two-phase construction (mirrors Java) +/// - Phase 1: `new(props, table_schema)` — created in `FileGroupReaderSchemaHandler` +/// Sets `custom_delete_marker` and `has_built_in_delete_field` from props + tableSchema. +/// - Phase 2: `with_reader_schema(schema)` — called in `FileGroupRecordBuffer` +/// Sets `reader_schema` and computes `hoodie_operation_pos` from the reader schema. +#[derive(Debug, Clone)] +pub struct DeleteContext { + /// Custom delete marker key-value pair, if configured. + /// When set, a record is considered deleted if it contains this key-value. + /// Sourced from `DELETE_KEY` and `DELETE_MARKER` table properties. + pub custom_delete_marker: Option<(String, String)>, + + /// Whether the schema has a built-in delete field (`_hoodie_is_deleted`). + pub has_built_in_delete_field: bool, + + /// Position of the `_hoodie_operation` field in the reader schema, if present. + pub hoodie_operation_pos: Option, + + /// The reader schema. + pub reader_schema: SchemaRef, +} + +/// Property key for the custom delete marker field name. +/// Mirrors Java's `DefaultHoodieRecordPayload.DELETE_KEY`. +const DELETE_KEY_PROP: &str = "hoodie.datasource.write.payload.delete.field"; + +/// Property key for the custom delete marker value. +/// Mirrors Java's `DefaultHoodieRecordPayload.DELETE_MARKER`. +const DELETE_MARKER_PROP: &str = "hoodie.datasource.write.payload.delete.marker"; + +impl DeleteContext { + /// Phase 1: Create a `DeleteContext` from table properties and table schema. + /// + /// Mirrors Java's `new DeleteContext(Properties props, Schema tableSchema)`. + /// - Extracts custom delete marker from properties (DELETE_KEY, DELETE_MARKER). + /// - Checks if `_hoodie_is_deleted` field exists in the table schema. + /// - `hoodie_operation_pos` is NOT set here (set in `with_reader_schema`). + pub fn new(props: &HashMap, table_schema: &SchemaRef) -> Self { + let custom_delete_marker = Self::get_custom_delete_marker(props); + let has_built_in_delete_field = table_schema + .column_with_name("_hoodie_is_deleted") + .is_some(); + + Self { + custom_delete_marker, + has_built_in_delete_field, + hoodie_operation_pos: None, + reader_schema: table_schema.clone(), + } + } + + /// Create a `DeleteContext` from table properties only (no schema yet). + /// + /// Used at buffer construction time when the reader schema is not yet + /// available. Uses conservative defaults for schema-dependent fields + /// (`has_built_in_delete_field = true`) so that `is_delete_record` falls + /// through to runtime column lookups rather than skipping checks. + /// + /// Call `with_reader_schema()` later to provide actual schema info. + pub fn from_props(props: &HashMap) -> Self { + let custom_delete_marker = Self::get_custom_delete_marker(props); + Self { + custom_delete_marker, + // Conservative: assume field might exist; is_delete_record will + // do a runtime column lookup and handle absence gracefully. + has_built_in_delete_field: true, + hoodie_operation_pos: None, + reader_schema: arrow_schema::Schema::empty().into(), + } + } + + /// Phase 2: Enrich with the reader schema. + /// + /// Mirrors Java's `DeleteContext.withReaderSchema(Schema readerSchema)`. + /// Sets `reader_schema` and computes `hoodie_operation_pos` and + /// `has_built_in_delete_field` from it. + pub fn with_reader_schema(mut self, schema: SchemaRef) -> Self { + self.has_built_in_delete_field = schema.column_with_name("_hoodie_is_deleted").is_some(); + self.hoodie_operation_pos = schema + .column_with_name("_hoodie_operation") + .map(|(idx, _)| idx); + self.reader_schema = schema; + self + } + + /// Convenience: create from reader schema only (no properties). + /// + /// Used when properties are not available (e.g., in tests or simple paths). + pub fn from_reader_schema(schema: SchemaRef) -> Self { + let has_built_in_delete_field = schema.column_with_name("_hoodie_is_deleted").is_some(); + + let hoodie_operation_pos = schema + .column_with_name("_hoodie_operation") + .map(|(idx, _)| idx); + + Self { + custom_delete_marker: None, + has_built_in_delete_field, + hoodie_operation_pos, + reader_schema: schema, + } + } + + /// Extract custom delete marker key-value from properties. + /// + /// Mirrors Java's `DeleteContext.getCustomDeleteMarkerKeyValue(Properties)`. + fn get_custom_delete_marker(props: &HashMap) -> Option<(String, String)> { + let key = props.get(DELETE_KEY_PROP)?; + let marker = props.get(DELETE_MARKER_PROP)?; + if key.is_empty() || marker.is_empty() { + return None; + } + Some((key.clone(), marker.clone())) + } +} diff --git a/crates/core/src/file_group/reader/input_split.rs b/crates/core/src/file_group/reader/input_split.rs new file mode 100644 index 00000000..8cb72d10 --- /dev/null +++ b/crates/core/src/file_group/reader/input_split.rs @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.InputSplit`. +//! +//! Describes the data to be read from a file group: an optional base file, +//! a list of log files, and the partition path context. + +use crate::file_group::log_file::LogFile; +use std::str::FromStr; + +/// Describes the input data for a file group read. +/// +/// Carries the base file (if any), the list of log files to scan, +/// the partition path, and the byte range to read from the base file. +#[derive(Debug, Clone)] +pub struct InputSplit { + /// Path to the base file (relative to table root), if present. + pub base_file_path: Option, + + /// Commit time of the base file, if present. + pub base_file_commit_time: Option, + + /// Relative paths to log files to scan. + pub log_file_paths: Vec, + + /// Partition path for this file group (e.g. "year=2024/month=01"). + pub partition_path: String, + + /// Byte offset to start reading from in the base file. + pub start: i64, + + /// Number of bytes to read from the base file. + pub length: i64, +} + +impl InputSplit { + pub fn new( + base_file_path: Option, + base_file_commit_time: Option, + log_file_paths: Vec, + partition_path: String, + ) -> Self { + // Sort log files ascending by deltaCommitTime → logVersion → writeToken, + // matching Java's InputSplit which applies HoodieLogFile.getLogFileComparator(). + // The C++ side may send them in descending order (from FileSlice's reverse TreeSet). + let log_file_paths = Self::sort_log_file_paths(log_file_paths); + Self { + base_file_path, + base_file_commit_time, + log_file_paths, + partition_path, + start: 0, + length: -1, + } + } + + /// Sort log file paths ascending by deltaCommitTime → logVersion → writeToken. + /// + /// Mirrors Java's `InputSplit` constructor which sorts via + /// `HoodieLogFile.getLogFileComparator()`. + fn sort_log_file_paths(mut paths: Vec) -> Vec { + if paths.len() <= 1 { + return paths; + } + paths.sort_by(|a, b| { + let name_a = a.rsplit('/').next().unwrap_or(a); + let name_b = b.rsplit('/').next().unwrap_or(b); + match (LogFile::from_str(name_a), LogFile::from_str(name_b)) { + (Ok(lf_a), Ok(lf_b)) => lf_a.cmp(&lf_b), + _ => a.cmp(b), // fallback to lexicographic if parsing fails + } + }); + paths + } + + /// Returns true if there are log files that need to be merged with the base file. + pub fn has_log_files(&self) -> bool { + !self.log_file_paths.is_empty() + } + + /// Returns true if there is no base file and no log files. + pub fn has_no_records_to_merge(&self) -> bool { + !self.has_log_files() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Java's InputSplit sorts log files ascending by deltaCommitTime → logVersion → writeToken + /// (via HoodieLogFile.getLogFileComparator()). The C++ side sends them in descending order + /// (from FileSlice's reverse TreeSet). InputSplit must re-sort to ascending. + #[test] + fn test_log_file_paths_sorted_ascending() { + // Log files in DESCENDING deltaCommitTime order (as C++ sends them) + let log_files = vec![ + ".72d44234-0_20260408194651210.log.1_0-272-505".to_string(), // NEWER + ".72d44234-0_20260408194649548.log.1_0-254-473".to_string(), // OLDER + ]; + + let split = InputSplit::new( + Some("base.parquet".to_string()), + None, + log_files, + String::new(), + ); + + // Must be re-sorted to ASCENDING deltaCommitTime + assert_eq!(split.log_file_paths.len(), 2); + assert!( + split.log_file_paths[0].contains("20260408194649548"), + "First log file should be the OLDER commit, got: {}", + split.log_file_paths[0] + ); + assert!( + split.log_file_paths[1].contains("20260408194651210"), + "Second log file should be the NEWER commit, got: {}", + split.log_file_paths[1] + ); + } + + /// Same deltaCommitTime, different logVersions — should sort by version ascending. + #[test] + fn test_log_file_paths_sorted_by_version() { + let log_files = vec![ + ".fileId-0_20260408194649548.log.3_0-100-200".to_string(), // version 3 + ".fileId-0_20260408194649548.log.1_0-100-200".to_string(), // version 1 + ".fileId-0_20260408194649548.log.2_0-100-200".to_string(), // version 2 + ]; + + let split = InputSplit::new(None, None, log_files, String::new()); + + assert!(split.log_file_paths[0].contains(".log.1_")); + assert!(split.log_file_paths[1].contains(".log.2_")); + assert!(split.log_file_paths[2].contains(".log.3_")); + } + + /// With partition path prefix — sort should work on the file name portion. + #[test] + fn test_log_file_paths_sorted_with_partition_prefix() { + let log_files = vec![ + "year=2024/month=01/.fileId-0_20260408194651210.log.1_0-272-505".to_string(), + "year=2024/month=01/.fileId-0_20260408194649548.log.1_0-254-473".to_string(), + ]; + + let split = InputSplit::new(None, None, log_files, "year=2024/month=01".to_string()); + + assert!(split.log_file_paths[0].contains("20260408194649548")); + assert!(split.log_file_paths[1].contains("20260408194651210")); + } + + /// Empty or single log file — no sorting needed, should not crash. + #[test] + fn test_log_file_paths_empty_and_single() { + let split_empty = InputSplit::new(None, None, vec![], String::new()); + assert!(split_empty.log_file_paths.is_empty()); + + let split_single = InputSplit::new( + None, + None, + vec![".fileId-0_20260408194649548.log.1_0-254-473".to_string()], + String::new(), + ); + assert_eq!(split_single.log_file_paths.len(), 1); + } +} diff --git a/crates/core/src/file_group/reader/iterator_mode.rs b/crates/core/src/file_group/reader/iterator_mode.rs new file mode 100644 index 00000000..c45343f8 --- /dev/null +++ b/crates/core/src/file_group/reader/iterator_mode.rs @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.IteratorMode`. +//! +//! Controls what form the output records take when iterating. + +/// The mode in which the file group reader yields records. +/// +/// In Java Hudi, this controls whether the iterator produces engine-native +/// records, HoodieRecord wrappers, or just record keys. +/// +/// In hudi-rs, we always work with Arrow RecordBatch, so `EngineRecord` +/// is the primary mode. The other modes are kept for API symmetry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum IteratorMode { + /// Yield engine-native records (Arrow RecordBatch in Rust). + #[default] + EngineRecord, + + /// Yield HoodieRecord wrappers (not yet implemented in Rust). + HoodieRecord, + + /// Yield only record keys (not yet implemented in Rust). + RecordKey, +} + +impl IteratorMode { + pub fn from_str_opt(s: &str) -> Option { + match s { + "ENGINE_RECORD" => Some(Self::EngineRecord), + "HOODIE_RECORD" => Some(Self::HoodieRecord), + "RECORD_KEY" => Some(Self::RecordKey), + _ => None, + } + } +} diff --git a/crates/core/src/file_group/reader/key_spec.rs b/crates/core/src/file_group/reader/key_spec.rs new file mode 100644 index 00000000..975ed307 --- /dev/null +++ b/crates/core/src/file_group/reader/key_spec.rs @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors Java `org.apache.hudi.common.table.log.KeySpec` hierarchy +//! (`KeySpec`, `FullKeySpec`, `PrefixKeySpec`) plus +//! `HoodieMergedLogRecordReader.createKeySpec(Option)`. + +use crate::expression::predicate::{Predicate, PredicateKind}; +use crate::expression::{Expression, ExpressionKind, LiteralValue}; +use std::sync::Arc; + +/// Mirrors Java's `KeySpec` interface, with `FullKeySpec` and `PrefixKeySpec` +/// flattened into enum variants. +#[derive(Debug, Clone)] +pub enum KeySpec { + /// Full keys list (from `Predicates::In`). Mirrors Java `FullKeySpec`. + FullKeys(Vec), + /// Prefix list (from `Predicates::StringStartsWithAny`). Mirrors Java `PrefixKeySpec`. + PrefixKeys(Vec), +} + +impl KeySpec { + /// Test the given record-key string against this key-spec. + pub fn matches(&self, key: &str) -> bool { + match self { + KeySpec::FullKeys(keys) => keys.iter().any(|k| k == key), + KeySpec::PrefixKeys(prefixes) => prefixes.iter().any(|p| key.starts_with(p.as_str())), + } + } +} + +/// Mirrors Java `HoodieMergedLogRecordReader.createKeySpec(Option)`, +/// implemented in idiomatic Rust via `PredicateKind` pattern matching instead +/// of Java's `instanceof`/cast. +pub fn create_key_spec(filter: Option<&dyn Predicate>) -> Option { + match filter?.pred_kind() { + PredicateKind::In(p) => Some(KeySpec::FullKeys(string_literals(p.right_children())?)), + PredicateKind::StringStartsWithAny(p) => { + Some(KeySpec::PrefixKeys(string_literals(p.right_children())?)) + } + _ => None, + } +} + +/// Extract `String` values from a list of `Literal(LiteralValue::String(_))` +/// expressions. Returns `None` if any child is not a String literal. +pub(crate) fn string_literals(exprs: &[Box]) -> Option> { + exprs + .iter() + .map(|e| match e.kind() { + ExpressionKind::Literal(lit) => match lit.value() { + LiteralValue::String(s) => Some(s.clone()), + _ => None, + }, + _ => None, + }) + .collect() +} + +/// Convenience: accept an `Option>` directly (the type +/// stored on `ReaderContext`). +pub fn create_key_spec_from_arc(filter: &Option>) -> Option { + create_key_spec(filter.as_deref()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::predicates::predicates_factory; + use crate::expression::{Literal, NameReference}; + + #[test] + fn create_key_spec_none_when_filter_is_none() { + assert!(create_key_spec(None).is_none()); + } + + #[test] + fn create_key_spec_in_predicate_yields_full_keys() { + let p = predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![ + Box::new(Literal::string("k1")) as Box, + Box::new(Literal::string("k2")) as Box, + ], + ); + let spec = create_key_spec(Some(&p)).expect("expected Some(KeySpec)"); + match spec { + KeySpec::FullKeys(keys) => assert_eq!(keys, vec!["k1".to_string(), "k2".to_string()]), + _ => panic!("expected FullKeys"), + } + } + + #[test] + fn create_key_spec_starts_with_any_yields_prefix_keys() { + let p = predicates_factory::starts_with_any( + Box::new(NameReference::new("_hoodie_record_key")), + vec![ + Box::new(Literal::string("alpha")) as Box, + Box::new(Literal::string("beta")) as Box, + ], + ); + let spec = create_key_spec(Some(&p)).expect("expected Some(KeySpec)"); + match spec { + KeySpec::PrefixKeys(prefixes) => { + assert_eq!(prefixes, vec!["alpha".to_string(), "beta".to_string()]) + } + _ => panic!("expected PrefixKeys"), + } + } + + #[test] + fn create_key_spec_returns_none_for_non_in_non_starts_with() { + let p = predicates_factory::eq( + Box::new(Literal::string("a")), + Box::new(Literal::string("b")), + ); + assert!(create_key_spec(Some(&p)).is_none()); + } + + #[test] + fn full_keys_matches() { + let spec = KeySpec::FullKeys(vec!["a".into(), "b".into()]); + assert!(spec.matches("a")); + assert!(spec.matches("b")); + assert!(!spec.matches("c")); + } + + #[test] + fn prefix_keys_matches() { + let spec = KeySpec::PrefixKeys(vec!["pre".into(), "alpha".into()]); + assert!(spec.matches("prefix-1")); + assert!(spec.matches("alphabet")); + assert!(!spec.matches("beta")); + } +} diff --git a/crates/core/src/file_group/reader/log_record_reader.rs b/crates/core/src/file_group/reader/log_record_reader.rs new file mode 100644 index 00000000..9b29a4cd --- /dev/null +++ b/crates/core/src/file_group/reader/log_record_reader.rs @@ -0,0 +1,1327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.log.BaseHoodieLogRecordReader`. +//! +//! The 2-pass log scanning engine that reads log blocks, filters them through +//! 4 gates, resolves compaction, and dispatches to the record buffer. +//! +//! ## 3-pass algorithm (matching Java's `scanInternal`): +//! +//! **Pass 1 — Forward scan with 4 gates:** +//! - Gate 1: Corrupt → skip +//! - Gate 2: Future (instant > latestInstantTime) → skip +//! - Gate 3: Inflight (skipped in Phase 1) +//! - Gate 4: Instant range filter → skip +//! - Rollback command → remove target instant +//! - Data/Delete → add to `instant_to_blocks_map` +//! +//! **Pass 2 — Reverse iteration with compaction resolution:** +//! Iterate instants newest→oldest, `Collections.reverse(logBlocks)` then `addLast`. +//! Result: deque has latest-first at head, earliest-last at tail. +//! +//! **Pass 3 — `processQueuedBlocksForInstant`:** +//! Drain deque via `pollLast` (tail-first) = oldest instant processed first. + +use crate::Result; +use crate::file_group::log_file::log_block::{BlockMetadataKey, BlockType, LogBlock}; +use crate::file_group::log_file::reader::LogFileReader; +use crate::file_group::reader::buffer::HoodieFileGroupRecordBuffer; +use crate::file_group::reader::reader_context::ReaderContext; +use crate::storage::Storage; +use crate::timeline::selector::InstantRange; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; + +// ========================================================================= +// Pass 1 result +// ========================================================================= + +/// Output of Pass 1 (forward scan with 4 gates). +#[derive(Debug)] +pub struct Pass1Result { + /// Instants in first-seen order (no duplicates). + pub ordered_instants_list: Vec, + /// instant time → blocks for that instant (in file-read order). + pub instant_to_blocks_map: HashMap>, + /// Stats + pub total_log_blocks: u64, + pub total_corrupt_blocks: u64, + pub total_rollbacks: u64, +} + +/// Run Pass 1: forward scan with 4 gates. +/// +/// Processes a flat list of blocks (from all log files, in file-read order) +/// and classifies them into `instant_to_blocks_map` + `ordered_instants_list`. +/// +/// ## 4 Gates (per-block): +/// 1. Corrupt → skip +/// 2. Future (instant > latest_instant_time) → skip +/// 3. Inflight → skipped in Phase 1 +/// 4. Instant range → skip if not in range +/// 5. Rollback command → remove target instant from maps +pub fn forward_scan_pass1( + all_blocks: Vec, + latest_instant_time: &str, + instant_range: &Option, + timezone: &str, +) -> Result { + let mut target_rollback_instants: HashSet = HashSet::new(); + let mut instant_to_blocks_map: HashMap> = HashMap::new(); + let mut ordered_instants_list: Vec = Vec::new(); + let mut total_log_blocks: u64 = 0; + let mut total_corrupt_blocks: u64 = 0; + let mut total_rollbacks: u64 = 0; + + log::debug!( + "[Pass1] forward_scan: {} total blocks, latest_instant_time={}, has_instant_range={}", + all_blocks.len(), + latest_instant_time, + instant_range.is_some(), + ); + + for block in all_blocks { + total_log_blocks += 1; + + // Gate 1: Corrupt blocks → skip + if block.block_type == BlockType::Corrupted { + log::debug!("[Pass1] Gate1: corrupt block #{total_log_blocks} skipped"); + total_corrupt_blocks += 1; + continue; + } + + let instant_time = match block.instant_time() { + Ok(t) => t.to_string(), + Err(_) => { + log::debug!("[Pass1] block #{total_log_blocks} has no instant time, skipping"); + continue; + } + }; + + // Gate 2: Future blocks → skip (instant > latestInstantTime) + if block.block_type != BlockType::Command && instant_time.as_str() > latest_instant_time { + log::debug!( + "[Pass1] Gate2: future block #{total_log_blocks} instant={instant_time} > {latest_instant_time}, skipped" + ); + continue; + } + + // Gate 3: Inflight check — SKIPPED (no metaclient) + + // Gate 4: Instant range filter (not for command blocks) + if block.block_type != BlockType::Command { + if let Some(range) = instant_range { + if range.not_in_range(&instant_time, timezone)? { + log::debug!( + "[Pass1] Gate4: block #{total_log_blocks} instant={instant_time} out of range, skipped" + ); + continue; + } + } + } + + log::debug!( + "[Pass1] block #{total_log_blocks} passed all gates: type={:?} instant={instant_time}", + block.block_type, + ); + + // Classify the block + match block.block_type { + BlockType::AvroData + | BlockType::HfileData + | BlockType::ParquetData + | BlockType::Delete => { + let blocks_list = instant_to_blocks_map + .entry(instant_time.clone()) + .or_default(); + if blocks_list.is_empty() { + ordered_instants_list.push(instant_time.clone()); + } + blocks_list.push(block); + } + BlockType::Command => { + if block.is_rollback_block() { + total_rollbacks += 1; + if let Ok(target) = block.target_instant_time() { + let target = target.to_string(); + log::debug!("[Pass1] ROLLBACK: removing instant={target}"); + target_rollback_instants.insert(target.clone()); + ordered_instants_list.retain(|t| t != &target); + instant_to_blocks_map.remove(&target); + } + } + } + _ => {} + } + } + + log::debug!( + "[Pass1] complete: ordered_instants={:?} total_blocks={} corrupt={} rollbacks={}", + ordered_instants_list, + total_log_blocks, + total_corrupt_blocks, + total_rollbacks, + ); + for (instant, blocks) in &instant_to_blocks_map { + log::debug!( + "[Pass1] instant={instant}: {} block(s) [{:?}]", + blocks.len(), + blocks + .iter() + .map(|b| format!("{:?}", b.block_type)) + .collect::>(), + ); + } + + Ok(Pass1Result { + ordered_instants_list, + instant_to_blocks_map, + total_log_blocks, + total_corrupt_blocks, + total_rollbacks, + }) +} + +// ========================================================================= +// Pass 2 result +// ========================================================================= + +/// Output of Pass 2 (reverse iteration with compaction resolution). +#[derive(Debug)] +pub struct Pass2Result { + /// The final ordered deque of blocks to be processed. + /// Latest instant at head (front), earliest at tail (back). + /// Drain via `pop_back` for oldest-first processing. + pub current_instant_log_blocks: VecDeque, + /// Dedup guard — instants whose blocks are enqueued. + pub instant_times_included: HashSet, + /// Which instant times actually contributed blocks (ordered output). + pub valid_block_instants: Vec, +} + +/// Run Pass 2: reverse iteration with compaction resolution. +/// +/// Iterates `ordered_instants_list` from newest→oldest, applying: +/// - `Collections.reverse(logBlocks)` then `addLast` (Rust: `reverse()` + `push_back`) +/// - Compaction: if block has `COMPACTED_BLOCK_TIMES` header, register mapping +/// - Deduplication: `instant_times_included` prevents double-enqueue +/// +/// ## Invariants (from FS_logFileReadInputs_fileOrderRequestTime.md): +/// +/// **Invariant 1**: `instant_times_included` and `valid_block_instants` contain +/// exactly the same instant strings (Set vs List). +/// +/// **Invariant 2**: Every instant in `instant_times_included`/`valid_block_instants` +/// has its blocks in `current_instant_log_blocks`, and vice versa. +/// +/// **Invariant 3**: Each instant appears in `instant_times_included` at most once (dedup). +/// +/// **Invariant 4**: `current_instant_log_blocks` is ordered latest-first (reverse chronological). +/// Drain via `pop_back` produces oldest-first processing order. +pub fn reverse_scan_pass2(pass1: &mut Pass1Result) -> Pass2Result { + log::debug!( + "[Pass2] reverse_scan: {} instants to process (newest→oldest)", + pass1.ordered_instants_list.len(), + ); + + let mut current_instant_log_blocks: VecDeque = VecDeque::new(); + let mut instant_times_included: HashSet = HashSet::new(); + let mut valid_block_instants: Vec = Vec::new(); + let mut block_time_to_compaction_block_time_map: HashMap = HashMap::new(); + + for i in (0..pass1.ordered_instants_list.len()).rev() { + let instant_time = &pass1.ordered_instants_list[i]; + let instants_blocks = match pass1.instant_to_blocks_map.get(instant_time) { + Some(blocks) if !blocks.is_empty() => blocks, + _ => continue, + }; + + let first_block = &instants_blocks[0]; + + // Check for compacted blocks (has COMPACTED_BLOCK_TIMES header) + if let Some(compacted_times) = first_block + .header + .get(&BlockMetadataKey::CompactedBlockTimes) + { + for original_instant in compacted_times.split(',') { + let original_instant = original_instant.trim().to_string(); + let final_instant = block_time_to_compaction_block_time_map + .get(instant_time) + .cloned() + .unwrap_or_else(|| instant_time.clone()); + block_time_to_compaction_block_time_map.insert(original_instant, final_instant); + } + } else { + // Normal data block — check if it's been compacted + let compacted_final = block_time_to_compaction_block_time_map.get(instant_time); + + if let Some(final_instant) = compacted_final { + let final_instant = final_instant.clone(); + if instant_times_included.contains(&final_instant) { + continue; + } + if let Some(blocks) = pass1.instant_to_blocks_map.get(&final_instant) { + let mut reversed = blocks.clone(); + reversed.reverse(); + for block in reversed { + current_instant_log_blocks.push_back(block); + } + } + instant_times_included.insert(final_instant.clone()); + valid_block_instants.push(final_instant); + } else { + // Not compacted — add blocks directly + // Java: Collections.reverse(logBlocks) then forEach(addLast) + let blocks = pass1 + .instant_to_blocks_map + .get(instant_time) + .cloned() + .unwrap_or_default(); + let mut reversed = blocks; + reversed.reverse(); + for block in reversed { + current_instant_log_blocks.push_back(block); + } + instant_times_included.insert(instant_time.clone()); + valid_block_instants.push(instant_time.clone()); + } + } + } + + log::debug!( + "[Pass2] complete: {} blocks in deque, valid_instants={:?}", + current_instant_log_blocks.len(), + valid_block_instants, + ); + + Pass2Result { + current_instant_log_blocks, + instant_times_included, + valid_block_instants, + } +} + +// ========================================================================= +// BaseHoodieLogRecordReader +// ========================================================================= + +/// The base log record reader implementing the 3-pass scanning algorithm. +/// +/// Mirrors Java's `BaseHoodieLogRecordReader` (abstract base class). +/// +/// ## Java hierarchy: +/// ```text +/// BaseHoodieLogRecordReader (abstract — this struct) +/// └─ HoodieMergedLogRecordReader (concrete — wraps this via composition) +/// ``` +/// +/// ## Fields matching Java: +/// - `reader_context` ↔ `readerContext` +/// - `storage` ↔ `storage` +/// - `log_file_paths` ↔ `logFiles` +/// - `latest_instant_time` ↔ `latestInstantTime` +/// - `instant_range` ↔ `instantRange` +/// - `force_full_scan` ↔ `forceFullScan` +/// - `record_buffer` ↔ `recordBuffer` +/// - `allow_inflight_instants` ↔ `allowInflightInstants` +pub struct BaseHoodieLogRecordReader { + pub reader_context: Arc, + pub storage: Arc, + pub log_file_paths: Vec, + pub latest_instant_time: String, + /// Mirrors Java's `private final Option instantRange`. + pub instant_range: Option, + /// Mirrors Java's `protected final boolean forceFullScan`. + /// When true, scanning happens eagerly in the constructor. + pub force_full_scan: bool, + pub record_buffer: Box, + pub allow_inflight_instants: bool, + /// Mirrors Java `BaseHoodieLogRecordReader.keySpecOpt` — set by the + /// scanner caller to narrow which records are processed. + pub key_spec_opt: Option, + + // ── Stats / state (mirrors Java's AtomicLong counters + progress) ── + pub valid_block_instants: Vec, + pub total_log_files: u64, + pub total_log_blocks: u64, + pub total_log_records: u64, + pub total_corrupt_blocks: u64, + pub total_rollbacks: u64, + /// Mirrors Java's `private float progress` (0.0 → 1.0). + pub progress: f32, +} + +impl BaseHoodieLogRecordReader { + /// Mirrors Java's `scanInternal(Option keySpecOpt, boolean skipProcessingBlocks)`. + /// + /// # Arguments + /// - `key_spec_opt` — optional key-spec filter; records whose key doesn't match are skipped. + /// Mirrors Java's `keySpecOpt` parameter. + /// - `skip_processing_blocks` — when `true`, Pass 3 (block processing) is skipped. + /// Java's `HoodieMergedLogRecordReader.scan(true)` uses this to collect block + /// metadata without actually merging records. + pub async fn scan_internal( + &mut self, + key_spec_opt: Option, + skip_processing_blocks: bool, + ) -> Result<()> { + // Store on self so deeper dispatch points can consult it. + self.key_spec_opt = key_spec_opt; + + // Reset state (mirrors Java: currentInstantLogBlocks = new ArrayDeque<>(), progress = 0.0f, ...) + self.valid_block_instants.clear(); + self.progress = 0.0; + self.total_log_files = 0; + self.total_log_blocks = 0; + self.total_log_records = 0; + self.total_corrupt_blocks = 0; + self.total_rollbacks = 0; + + // Propagate to buffer for per-record filtering (mirrors Java's + // BaseHoodieLogRecordReader → buffer key-spec threading). + self.record_buffer.set_key_spec(self.key_spec_opt.clone()); + + let timezone = self.reader_context.timezone(); + + // Read all blocks as metadata-only (no content decoding). + // Mirrors Java's Pass 1 where content bytes are skipped via seek(). + // Content is loaded lazily via inflate() during Pass 3. + let mut all_blocks: Vec = Vec::new(); + for path in &self.log_file_paths.clone() { + let mut reader = + LogFileReader::new(self.reader_context.clone(), self.storage.clone(), path).await?; + let blocks = reader.read_all_blocks_metadata_only(path)?; + self.total_log_files += 1; + all_blocks.extend(blocks); + } + + // Pass 1: Forward scan with 4 gates + let mut pass1 = forward_scan_pass1( + all_blocks, + &self.latest_instant_time, + &self.instant_range, + &timezone, + )?; + + self.total_log_blocks = pass1.total_log_blocks; + self.total_corrupt_blocks = pass1.total_corrupt_blocks; + self.total_rollbacks = pass1.total_rollbacks; + + // Pass 2: Reverse iteration with compaction resolution + let mut pass2 = reverse_scan_pass2(&mut pass1); + + self.valid_block_instants = pass2.valid_block_instants; + + log::debug!( + "Pass 1+2 complete: {} blocks queued, {} valid instants", + pass2.current_instant_log_blocks.len(), + self.valid_block_instants.len() + ); + + // Pass 3: Process queued blocks (oldest → newest via pop_back). + // Mirrors Java: if (!currentInstantLogBlocks.isEmpty() && !skipProcessingBlocks) { ... } + if !pass2.current_instant_log_blocks.is_empty() && !skip_processing_blocks { + log::debug!("Merging the final data blocks"); + self.process_queued_blocks_for_instant(&mut pass2.current_instant_log_blocks)?; + } + + // Done + self.progress = 1.0; + self.total_log_records = self.record_buffer.get_total_log_records(); + + Ok(()) + } + + /// Mirrors Java's `shouldLookupRecords()`. + /// + /// Point-wise record lookups are only enabled when scanner is NOT in + /// full-scan mode. + pub fn should_lookup_records(&self) -> bool { + !self.force_full_scan + } + + // ── Getters (mirrors Java's getter methods on BaseHoodieLogRecordReader) ── + + pub fn get_progress(&self) -> f32 { + self.progress + } + + pub fn get_total_log_files(&self) -> u64 { + self.total_log_files + } + + pub fn get_total_log_records(&self) -> u64 { + self.total_log_records + } + + pub fn get_total_log_blocks(&self) -> u64 { + self.total_log_blocks + } + + pub fn get_total_rollbacks(&self) -> u64 { + self.total_rollbacks + } + + pub fn get_total_corrupt_blocks(&self) -> u64 { + self.total_corrupt_blocks + } + + pub fn get_valid_block_instants(&self) -> &[String] { + &self.valid_block_instants + } + + /// Mirrors Java's `processQueuedBlocksForInstant(Deque, ...)`. + /// + /// Drains the deque from the back (oldest→newest via `pop_back`/`pollLast`). + /// Dispatches each block to the record buffer — the buffer is responsible for + /// inflating, extracting records, and deflating (matching Java's design where + /// `processQueuedBlocksForInstant` does NOT call `inflate()`). + fn process_queued_blocks_for_instant( + &mut self, + log_blocks: &mut VecDeque, + ) -> Result<()> { + log::debug!( + "[Pass3] processQueuedBlocksForInstant: {} blocks to process (pop_back = oldest first)", + log_blocks.len(), + ); + let mut block_num = 0u64; + + while let Some(mut block) = log_blocks.pop_back() { + block_num += 1; + let instant_time = block.instant_time().unwrap_or("unknown").to_string(); + log::debug!( + "[Pass3] block #{block_num}: type={:?} instant={instant_time}", + block.block_type, + ); + + match block.block_type { + BlockType::AvroData | BlockType::HfileData | BlockType::ParquetData => { + self.record_buffer.process_data_block(&mut block)?; + log::debug!( + "[Pass3] after processing data block #{block_num}: buffer size={}", + self.record_buffer.size(), + ); + } + BlockType::Delete => { + self.record_buffer.process_delete_block(&mut block)?; + } + BlockType::Corrupted => { + log::warn!("Found corrupt block not rolled back"); + } + _ => {} + } + } + Ok(()) + } +} + +// ========================================================================= +// Tests +// ========================================================================= + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_group::log_file::log_block::{CommandBlock, LogBlockContent}; + use crate::file_group::log_file::log_format::LogFormatVersion; + + // ── Helpers ────────────────────────────────────────────────────────── + + /// Create a data block with given instant time and a tag for identification. + fn make_data_block(instant_time: &str) -> LogBlock { + let mut header = HashMap::new(); + header.insert(BlockMetadataKey::InstantTime, instant_time.to_string()); + LogBlock::new( + LogFormatVersion::V1, + BlockType::AvroData, + header, + LogBlockContent::Empty, + HashMap::new(), + ) + } + + /// Create a delete block with given instant time. + fn make_delete_block(instant_time: &str) -> LogBlock { + let mut header = HashMap::new(); + header.insert(BlockMetadataKey::InstantTime, instant_time.to_string()); + LogBlock::new( + LogFormatVersion::V1, + BlockType::Delete, + header, + LogBlockContent::Empty, + HashMap::new(), + ) + } + + /// Create a corrupt block. + fn make_corrupt_block() -> LogBlock { + LogBlock::new( + LogFormatVersion::V1, + BlockType::Corrupted, + HashMap::new(), + LogBlockContent::Empty, + HashMap::new(), + ) + } + + /// Create a rollback command block targeting the given instant. + fn make_rollback_block(instant_time: &str, target_instant: &str) -> LogBlock { + let mut header = HashMap::new(); + header.insert(BlockMetadataKey::InstantTime, instant_time.to_string()); + header.insert( + BlockMetadataKey::TargetInstantTime, + target_instant.to_string(), + ); + header.insert( + BlockMetadataKey::CommandBlockType, + (CommandBlock::Rollback as u32).to_string(), + ); + LogBlock::new( + LogFormatVersion::V1, + BlockType::Command, + header, + LogBlockContent::Empty, + HashMap::new(), + ) + } + + /// Create a compacted block with COMPACTED_BLOCK_TIMES header. + fn make_compacted_block(instant_time: &str, compacted_times: &str) -> LogBlock { + let mut header = HashMap::new(); + header.insert(BlockMetadataKey::InstantTime, instant_time.to_string()); + header.insert( + BlockMetadataKey::CompactedBlockTimes, + compacted_times.to_string(), + ); + LogBlock::new( + LogFormatVersion::V1, + BlockType::AvroData, + header, + LogBlockContent::Empty, + HashMap::new(), + ) + } + + fn get_instant(block: &LogBlock) -> String { + block.instant_time().unwrap().to_string() + } + + // ===================================================================== + // Pass 1: 4-Gate Filtering Tests + // ===================================================================== + + /// Gate 1: Corrupt blocks are skipped. + /// + /// Given: 1 corrupt block + 1 data block + /// When: forward_scan_pass1() + /// Then: total_corrupt_blocks=1, only data block in map + #[test] + fn test_pass1_gate1_corrupt_block_skipped() { + let blocks = vec![make_corrupt_block(), make_data_block("20250101")]; + + let result = forward_scan_pass1(blocks, "99991231235959999", &None, "utc").unwrap(); + + assert_eq!(result.total_corrupt_blocks, 1); + assert_eq!(result.ordered_instants_list, vec!["20250101"]); + assert_eq!(result.instant_to_blocks_map.len(), 1); + } + + /// Gate 2: Future blocks (instant > latestInstantTime) are skipped. + /// + /// Given: latest_instant_time="20250103", block at "20250105" + /// When: forward_scan_pass1() + /// Then: future block not in map + #[test] + fn test_pass1_gate2_future_block_skipped() { + let blocks = vec![ + make_data_block("20250101"), + make_data_block("20250105"), // future + ]; + + let result = forward_scan_pass1(blocks, "20250103", &None, "utc").unwrap(); + + assert_eq!(result.ordered_instants_list, vec!["20250101"]); + assert!(!result.instant_to_blocks_map.contains_key("20250105")); + } + + /// Gate 4: Instant range filter skips out-of-range blocks. + /// + /// Given: instant_range up to "20250102000000000", block at "20250103000000000" + /// When: forward_scan_pass1() + /// Then: out-of-range block skipped + #[test] + fn test_pass1_gate4_instant_range_filter() { + let blocks = vec![ + make_data_block("20250101000000000"), + make_data_block("20250103000000000"), // out of range + ]; + let range = Some(InstantRange::up_to("20250102000000000", "utc")); + + let result = forward_scan_pass1(blocks, "99991231235959999", &range, "utc").unwrap(); + + assert_eq!(result.ordered_instants_list, vec!["20250101000000000"]); + } + + /// Rollback command removes target instant from maps atomically. + /// + /// Given: B1(i=20250101), B2(i=20250102), R1(rollback target=20250102) + /// When: forward_scan_pass1() + /// Then: instant 20250102 removed, only 20250101 remains, total_rollbacks=1 + #[test] + fn test_pass1_rollback_removes_instant() { + let blocks = vec![ + make_data_block("20250101"), + make_data_block("20250102"), + make_rollback_block("20250103", "20250102"), + ]; + + let result = forward_scan_pass1(blocks, "99991231235959999", &None, "utc").unwrap(); + + assert_eq!(result.total_rollbacks, 1); + assert_eq!(result.ordered_instants_list, vec!["20250101"]); + assert!(!result.instant_to_blocks_map.contains_key("20250102")); + } + + /// orderedInstantsList has no duplicates even when same instant appears in multiple blocks. + /// + /// Given: Two blocks with same instant time + /// When: forward_scan_pass1() + /// Then: orderedInstantsList has one entry, map has 2 blocks + #[test] + fn test_pass1_ordered_instants_no_duplicates() { + let blocks = vec![ + make_data_block("20250101"), + make_data_block("20250101"), // same instant + ]; + + let result = forward_scan_pass1(blocks, "99991231235959999", &None, "utc").unwrap(); + + assert_eq!(result.ordered_instants_list, vec!["20250101"]); + assert_eq!(result.instant_to_blocks_map["20250101"].len(), 2); + } + + /// orderedInstantsList and instantToBlocksMap key sets match. + /// + /// Given: Blocks from 3 instants + /// When: forward_scan_pass1() + /// Then: Both have exactly the same set of instant strings + #[test] + fn test_pass1_list_and_map_key_parity() { + let blocks = vec![ + make_data_block("20250101"), + make_data_block("20250102"), + make_data_block("20250103"), + ]; + + let result = forward_scan_pass1(blocks, "99991231235959999", &None, "utc").unwrap(); + + let list_set: HashSet<&String> = result.ordered_instants_list.iter().collect(); + let map_set: HashSet<&String> = result.instant_to_blocks_map.keys().collect(); + assert_eq!(list_set, map_set); + } + + // ===================================================================== + // Pass 2: Reverse Iteration & Deque Building + // ===================================================================== + + /// Invariant 1: instant_times_included and valid_block_instants contain + /// exactly the same instant strings. + /// + /// Given: 3 instants [t1, t2, t3], no compaction + /// When: reverse_scan_pass2() + /// Then: instant_times_included == valid_block_instants (as sets) + #[test] + fn test_pass2_invariant1_included_equals_valid() { + let blocks = vec![ + make_data_block("t1"), + make_data_block("t2"), + make_data_block("t3"), + ]; + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + let pass2 = reverse_scan_pass2(&mut pass1); + + let included: HashSet<&String> = pass2.instant_times_included.iter().collect(); + let valid: HashSet<&String> = pass2.valid_block_instants.iter().collect(); + assert_eq!(included, valid, "Invariant 1: included == valid"); + } + + /// Invariant 2: Every instant in included/valid has blocks in deque, and vice versa. + /// + /// Given: 3 instants with data blocks + /// When: reverse_scan_pass2() + /// Then: All blocks in deque belong to instants in valid_block_instants + #[test] + fn test_pass2_invariant2_blocks_match_instants() { + let blocks = vec![ + make_data_block("t1"), + make_data_block("t2"), + make_data_block("t2"), // 2 blocks for t2 + make_data_block("t3"), + ]; + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + let pass2 = reverse_scan_pass2(&mut pass1); + + // All blocks in deque belong to valid instants + for block in &pass2.current_instant_log_blocks { + let instant = get_instant(block); + assert!( + pass2.instant_times_included.contains(&instant), + "Block instant {instant} should be in instant_times_included" + ); + } + // Every valid instant has at least one block in deque + for instant in &pass2.valid_block_instants { + let has_block = pass2 + .current_instant_log_blocks + .iter() + .any(|b| get_instant(b) == *instant); + assert!( + has_block, + "Valid instant {instant} should have blocks in deque" + ); + } + } + + /// Invariant 3: Compaction deduplication prevents double-enqueue. + /// + /// Given: orderedInstants=[i1, i2, M1], M1.CompactedBlockTimes="i1,i2" + /// When: reverse_scan_pass2() + /// Then: M1's blocks used once, i1 and i2 blocks skipped. + /// valid_block_instants contains only M1 (+ potentially i4 etc.) + #[test] + fn test_pass2_invariant3_compaction_dedup() { + let blocks = vec![ + make_data_block("i1"), + make_data_block("i2"), + make_compacted_block("i3", "i1,i2"), + ]; + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + let pass2 = reverse_scan_pass2(&mut pass1); + + // Only i3 (compacted) should be in valid_block_instants + assert_eq!(pass2.valid_block_instants, vec!["i3"]); + assert_eq!(pass2.instant_times_included.len(), 1); + assert!(pass2.instant_times_included.contains("i3")); + + // Deque should only have i3's block + assert_eq!(pass2.current_instant_log_blocks.len(), 1); + let block = &pass2.current_instant_log_blocks[0]; + assert_eq!(get_instant(block), "i3"); + } + + /// Invariant 4: Deque is ordered latest-first. pollLast produces oldest-first. + /// + /// Given: orderedInstantsList = [t1, t2, t3] + /// When: reverse_scan_pass2() + /// Then: Deque front=t3, back=t1. + /// pop_back order: t1 → t2 → t3 (oldest first) + #[test] + fn test_pass2_invariant4_deque_ordering() { + let blocks = vec![ + make_data_block("t1"), + make_data_block("t2"), + make_data_block("t3"), + ]; + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + let mut pass2 = reverse_scan_pass2(&mut pass1); + + // pop_back drains tail-first = oldest first + let first = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!( + get_instant(&first), + "t1", + "First pop_back should be oldest (t1)" + ); + + let second = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&second), "t2", "Second pop_back should be t2"); + + let third = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!( + get_instant(&third), + "t3", + "Third pop_back should be newest (t3)" + ); + + assert!(pass2.current_instant_log_blocks.is_empty()); + } + + /// Multi-block instant: within-instant order preserved after double reversal. + /// + /// Given: t1→[A], t2→[B,C] (B seen before C in forward scan) + /// When: reverse_scan_pass2(), then drain via pop_back + /// Then: pop_back order: A(t1) → B(t2) → C(t2) + /// (original file-read order restored within each instant) + #[test] + fn test_pass2_multi_block_instant_order() { + // t1: 1 block, t2: 2 blocks (B seen first, then C) + let blocks = vec![ + make_data_block("t1"), // A + make_data_block("t2"), // B + make_delete_block("t2"), // C (delete block for same instant) + ]; + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + + // Verify pass1 accumulated correctly + assert_eq!(pass1.instant_to_blocks_map["t2"].len(), 2); + + let mut pass2 = reverse_scan_pass2(&mut pass1); + + // Drain via pop_back: should be t1's blocks, then t2's in original order + let b1 = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&b1), "t1"); + assert_eq!(b1.block_type, BlockType::AvroData); + + let b2 = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&b2), "t2"); + assert_eq!(b2.block_type, BlockType::AvroData); // B was first + + let b3 = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&b3), "t2"); + assert_eq!(b3.block_type, BlockType::Delete); // C was second + } + + /// Concrete trace from howLogBlocksAreFiletered.md: + /// + /// Given: log files with B1(i=20250101), B2(i=20250102), B3(i=20250102), + /// B4(i=20250103), R1(rollback target=20250102) + /// When: Pass 1 then Pass 2 + /// Then: After Pass 1: orderedInstantsList=[20250101, 20250103] + /// After Pass 2: pop_back order = B1(20250101), B4(20250103) + #[test] + fn test_concrete_trace_from_docs() { + let blocks = vec![ + make_data_block("20250101"), // B1 + make_data_block("20250102"), // B2 + make_data_block("20250102"), // B3 + make_data_block("20250103"), // B4 + make_rollback_block("20250104", "20250102"), // R1: rollback 20250102 + ]; + + let mut pass1 = forward_scan_pass1(blocks, "99991231235959999", &None, "utc").unwrap(); + + assert_eq!(pass1.ordered_instants_list, vec!["20250101", "20250103"]); + assert!(!pass1.instant_to_blocks_map.contains_key("20250102")); + + let mut pass2 = reverse_scan_pass2(&mut pass1); + + // pop_back: B1(20250101) first, then B4(20250103) + let first = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&first), "20250101"); + + let second = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&second), "20250103"); + + assert!(pass2.current_instant_log_blocks.is_empty()); + } + + /// From FS_logFileReadInputs_fileOrderRequestTime.md: 5-block example. + /// + /// Given: t1→[A,B], t2→[C], t3→[D,E] + /// When: Pass 2 reverse + pop_back drain + /// Then: pop_back order: A, B, C, D, E (oldest→newest, within-instant order preserved) + #[test] + fn test_five_block_example_from_docs() { + // Simulate: t1→[A,B], t2→[C], t3→[D,E] + let blocks = vec![ + make_data_block("t1"), // A + make_data_block("t1"), // B + make_data_block("t2"), // C + make_data_block("t3"), // D + make_delete_block("t3"), // E (different block type to distinguish) + ]; + + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + assert_eq!(pass1.ordered_instants_list, vec!["t1", "t2", "t3"]); + assert_eq!(pass1.instant_to_blocks_map["t1"].len(), 2); + assert_eq!(pass1.instant_to_blocks_map["t2"].len(), 1); + assert_eq!(pass1.instant_to_blocks_map["t3"].len(), 2); + + let mut pass2 = reverse_scan_pass2(&mut pass1); + + // Deque has 5 blocks. pop_back should produce: A,B,C,D,E + assert_eq!(pass2.current_instant_log_blocks.len(), 5); + + let a = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&a), "t1"); + assert_eq!(a.block_type, BlockType::AvroData); // A + + let b = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&b), "t1"); + assert_eq!(b.block_type, BlockType::AvroData); // B + + let c = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&c), "t2"); // C + + let d = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&d), "t3"); + assert_eq!(d.block_type, BlockType::AvroData); // D + + let e = pass2.current_instant_log_blocks.pop_back().unwrap(); + assert_eq!(get_instant(&e), "t3"); + assert_eq!(e.block_type, BlockType::Delete); // E + } + + // ===================================================================== + // Commit-Time Ordering: Full chain from log file order → deque → + // processNextDataRecord → running map → correct final state + // + // From FS_logFileReadInputs_fileOrderRequestTime.md: + // "The double-reversal (reverse iteration of instants + addLast + // in the deque + pollLast to consume) produces oldest-first + // processing. The last writer to records.put(key, ...) is + // always the newest deltaCommitTime." + // ===================================================================== + + use crate::config::HudiConfigs; + use crate::config::table::HudiTableConfig; + use crate::file_group::reader::buffer::key_based::KeyBasedFileGroupRecordBuffer; + use crate::file_group::reader::read_stats::HoodieReadStats; + use crate::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; + use crate::file_group::record_batches::RecordBatches; + use arrow_array::{Int32Array, Int64Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + + fn test_schema() -> std::sync::Arc { + std::sync::Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("counter", DataType::Int32, false), + Field::new("ts", DataType::Int64, false), + ])) + } + + fn test_batch(records: &[(&str, i32, i64)]) -> RecordBatch { + let schema = test_schema(); + let keys: Vec<&str> = records.iter().map(|(k, _, _)| *k).collect(); + let counters: Vec = records.iter().map(|(_, c, _)| *c).collect(); + let timestamps: Vec = records.iter().map(|(_, _, t)| *t).collect(); + RecordBatch::try_new( + schema, + vec![ + std::sync::Arc::new(StringArray::from(keys)), + std::sync::Arc::new(Int32Array::from(counters)), + std::sync::Arc::new(Int64Array::from(timestamps)), + ], + ) + .unwrap() + } + + /// Create a data block with actual RecordBatch content for a given instant. + fn make_data_block_with_content(instant_time: &str, records: &[(&str, i32, i64)]) -> LogBlock { + let mut header = HashMap::new(); + header.insert(BlockMetadataKey::InstantTime, instant_time.to_string()); + let batch = test_batch(records); + let mut rb = RecordBatches::new(); + rb.push_data_batch(batch); + LogBlock::new( + LogFormatVersion::V1, + BlockType::AvroData, + header, + LogBlockContent::Records(rb), + HashMap::new(), + ) + } + + fn make_test_buffer() -> Box { + let mut ctx = ReaderContext::empty(); + ctx.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + ctx.rebuild_record_context(String::new()); + // Prepare schema handler with DeleteContext so buffer construction succeeds. + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(test_schema()) + .with_data_schema(test_schema()); + let key_field = ctx.record_key_field().to_string(); + let ordering = ctx.record_context.ordering_field_names.clone(); + handler.prepare_required_schema( + true, + &[key_field], + &ordering, + &ctx.table_config, + false, + "COMMIT_TIME_ORDERING", + ); + ctx.schema_handler = handler; + let ctx = std::sync::Arc::new(ctx); + let stats = HoodieReadStats::default(); + Box::new( + KeyBasedFileGroupRecordBuffer::new(ctx, "COMMIT_TIME_ORDERING".to_string(), &stats) + .unwrap(), + ) + } + + /// Commit-time ordering: oldest instant processed first, newest last. + /// + /// Given: 3 instants (t1, t2, t3) each writing key "K" with different values + /// t1 writes K=(counter=1, ts=10) ← oldest + /// t2 writes K=(counter=2, ts=20) + /// t3 writes K=(counter=3, ts=30) ← newest + /// When: forward_scan_pass1 → reverse_scan_pass2 → process via pop_back + /// Then: pop_back order: t1 → t2 → t3 (oldest first) + /// After processing: records["K"] = (counter=3, ts=30) (newest wins) + /// + /// This validates the full chain from FS_logFileReadInputs_fileOrderRequestTime.md: + /// deltaMerge(t1_record, null) → records["K"] = t1_record + /// deltaMerge(t2_record, t1_record) → records["K"] = t2_record + /// deltaMerge(t3_record, t2_record) → records["K"] = t3_record (newest wins) + #[test] + fn test_commit_time_ordering_newest_instant_wins() { + let blocks = vec![ + make_data_block_with_content("t1", &[("K", 1, 10)]), + make_data_block_with_content("t2", &[("K", 2, 20)]), + make_data_block_with_content("t3", &[("K", 3, 30)]), + ]; + + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + assert_eq!(pass1.ordered_instants_list, vec!["t1", "t2", "t3"]); + + let mut pass2 = reverse_scan_pass2(&mut pass1); + + // Verify deque ordering: pop_back gives t1, t2, t3 (oldest first) + let mut buffer = make_test_buffer(); + let mut order = Vec::new(); + while let Some(mut block) = pass2.current_instant_log_blocks.pop_back() { + let instant = block.instant_time().unwrap().to_string(); + order.push(instant.clone()); + // Process through the buffer (same as processQueuedBlocksForInstant) + buffer.process_data_block(&mut block).unwrap(); + } + + // Verify processing order: oldest → newest + assert_eq!(order, vec!["t1", "t2", "t3"]); + + // Verify running map final state: newest instant (t3) wins + assert_eq!(buffer.size(), 1); + let record = buffer.get_log_records().get("K").unwrap(); + // For COMMIT_TIME_ORDERING, the last writer (t3) always wins + assert!(!record.is_delete()); + + // Verify through merge_and_collect + buffer.set_base_file_iterator(vec![]); + let result = buffer.merge_and_collect().unwrap(); + let keys = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let counters = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let timestamps = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(keys.value(0), "K"); + assert_eq!(counters.value(0), 3); // t3's value + assert_eq!(timestamps.value(0), 30); // t3's timestamp + } + + /// Multi-block instant: within-instant blocks processed in file-read order. + /// + /// Given: t1→[blockA(K=v1), blockB(K=v2)], t2→[blockC(K=v3)] + /// blockA written before blockB in the same log file + /// When: Full pass1 → pass2 → process via pop_back + /// Then: Within t1: blockA processed first, then blockB (file-read order) + /// Then t2: blockC processed last + /// Final: records["K"] = v3 (t2, newest instant) + /// + /// From the reference: "Collections.reverse + addLast + pollLast pattern is + /// effectively a double-reversal that restores original file-read order within + /// each instant." + #[test] + fn test_commit_time_within_instant_file_read_order() { + let blocks = vec![ + make_data_block_with_content("t1", &[("K", 1, 10)]), // blockA + make_data_block_with_content("t1", &[("K", 2, 20)]), // blockB (same instant) + make_data_block_with_content("t2", &[("K", 3, 30)]), // blockC + ]; + + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + assert_eq!(pass1.instant_to_blocks_map["t1"].len(), 2); + + let mut pass2 = reverse_scan_pass2(&mut pass1); + + let mut buffer = make_test_buffer(); + let mut order_instants = Vec::new(); + let mut order_values = Vec::new(); + while let Some(mut block) = pass2.current_instant_log_blocks.pop_back() { + let instant = block.instant_time().unwrap().to_string(); + order_instants.push(instant.clone()); + // Peek at content to record ordering before processing + if let LogBlockContent::Records(ref rb) = block.content { + for batch in &rb.data_batches { + let counters = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + order_values.push(counters.value(0)); + } + } + buffer.process_data_block(&mut block).unwrap(); + } + + // Processing order: blockA(t1) → blockB(t1) → blockC(t2) + assert_eq!(order_instants, vec!["t1", "t1", "t2"]); + assert_eq!(order_values, vec![1, 2, 3]); // file-read order within t1 + + // Final map state: t2's blockC wins (newest instant) + buffer.set_base_file_iterator(vec![]); + let result = buffer.merge_and_collect().unwrap(); + let counters = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(counters.value(0), 3); // t2's value wins + } + + /// Delta merge chain: t1_record → t2_record → t3_record for same key. + /// + /// Given: COMMIT_TIME_ORDERING, 3 instants writing key "K" + /// When: Process oldest → newest + /// Then: deltaMerge called 3 times: + /// 1. deltaMerge(t1_record, null) → records["K"] = t1_record + /// 2. deltaMerge(t2_record, t1_record) → records["K"] = t2_record + /// 3. deltaMerge(t3_record, t2_record) → records["K"] = t3_record + /// total_log_records = 3 + #[test] + fn test_delta_merge_chain_three_instants() { + let mut buffer = make_test_buffer(); + + // Simulate oldest→newest processing (as pollLast produces) + let mut block_t1 = make_data_block_with_content("t1", &[("K", 1, 10)]); + buffer.process_data_block(&mut block_t1).unwrap(); + assert_eq!(buffer.size(), 1); + + let mut block_t2 = make_data_block_with_content("t2", &[("K", 2, 20)]); + buffer.process_data_block(&mut block_t2).unwrap(); + assert_eq!(buffer.size(), 1); // same key, overwritten + + let mut block_t3 = make_data_block_with_content("t3", &[("K", 3, 30)]); + buffer.process_data_block(&mut block_t3).unwrap(); + assert_eq!(buffer.size(), 1); + + // All 3 records were processed + assert_eq!(buffer.get_total_log_records(), 3); + + // Final value is from t3 (newest) + buffer.set_base_file_iterator(vec![]); + let result = buffer.merge_and_collect().unwrap(); + let counters = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(result.num_rows(), 1); + assert_eq!(counters.value(0), 3); + } + + /// Multiple keys across instants: each key gets newest-instant value. + /// + /// Given: t1 writes {A=1, B=1}, t2 writes {A=2, C=1}, t3 writes {B=3} + /// When: Process oldest → newest + /// Then: records = {A: t2_value(2), B: t3_value(3), C: t2_value(1)} + #[test] + fn test_commit_time_multiple_keys_across_instants() { + let blocks = vec![ + make_data_block_with_content("t1", &[("A", 1, 10), ("B", 1, 10)]), + make_data_block_with_content("t2", &[("A", 2, 20), ("C", 1, 20)]), + make_data_block_with_content("t3", &[("B", 3, 30)]), + ]; + + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + let mut pass2 = reverse_scan_pass2(&mut pass1); + + let mut buffer = make_test_buffer(); + while let Some(mut block) = pass2.current_instant_log_blocks.pop_back() { + buffer.process_data_block(&mut block).unwrap(); + } + + // Verify map state + assert_eq!(buffer.size(), 3); // A, B, C + + buffer.set_base_file_iterator(vec![]); + let result = buffer.merge_and_collect().unwrap(); + assert_eq!(result.num_rows(), 3); + + let keys = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let counters = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + + // Sort by key for deterministic checking + let mut kv: Vec<(String, i32)> = (0..result.num_rows()) + .map(|i| (keys.value(i).to_string(), counters.value(i))) + .collect(); + kv.sort_by(|a, b| a.0.cmp(&b.0)); + + assert_eq!(kv[0], ("A".to_string(), 2)); // t2 wins for A + assert_eq!(kv[1], ("B".to_string(), 3)); // t3 wins for B + assert_eq!(kv[2], ("C".to_string(), 1)); // only in t2 + } + + /// Full chain with base file merge: log overwrites base for matching keys. + /// + /// Given: Base=[K=base_val(0)], t1 writes K=1, t2 writes K=2 + /// When: Process logs (oldest→newest), then merge with base + /// Then: Output: K=2 (newest log instant wins over both base and t1) + #[test] + fn test_commit_time_full_chain_with_base_merge() { + let blocks = vec![ + make_data_block_with_content("t1", &[("K", 1, 10)]), + make_data_block_with_content("t2", &[("K", 2, 20)]), + ]; + + let mut pass1 = forward_scan_pass1(blocks, "z", &None, "utc").unwrap(); + let mut pass2 = reverse_scan_pass2(&mut pass1); + + let mut buffer = make_test_buffer(); + while let Some(mut block) = pass2.current_instant_log_blocks.pop_back() { + buffer.process_data_block(&mut block).unwrap(); + } + + // Set base file with original value for K + let base = test_batch(&[("K", 0, 0)]); + buffer.set_base_file_iterator(vec![base]); + + let result = buffer.merge_and_collect().unwrap(); + assert_eq!(result.num_rows(), 1); + + let counters = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + // Log record from t2 wins over both base and t1 + assert_eq!(counters.value(0), 2); + } +} diff --git a/crates/core/src/file_group/reader/merged_log_record_reader.rs b/crates/core/src/file_group/reader/merged_log_record_reader.rs new file mode 100644 index 00000000..9d7b719e --- /dev/null +++ b/crates/core/src/file_group/reader/merged_log_record_reader.rs @@ -0,0 +1,532 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.log.HoodieMergedLogRecordReader`. +//! +//! Thin wrapper around [`BaseHoodieLogRecordReader`] that adds: +//! - Builder pattern (matching Java's `HoodieMergedLogRecordReader.Builder`) +//! - Timing of the scan operation +//! - `performScan()` invoked from constructor when `forceFullScan=true` +//! +//! ## Java hierarchy: +//! ```text +//! BaseHoodieLogRecordReader (abstract scan engine) +//! └─ HoodieMergedLogRecordReader (this class — adds timing + builder) +//! ``` + +use crate::Result; +use crate::error::CoreError; +use crate::file_group::reader::buffer::HoodieFileGroupRecordBuffer; +use crate::file_group::reader::log_record_reader::BaseHoodieLogRecordReader; +use crate::file_group::reader::reader_context::ReaderContext; +use crate::storage::Storage; +use crate::timeline::selector::InstantRange; +use std::sync::Arc; + +/// Statistics from the log scanning operation. +#[derive(Debug, Clone, Default)] +pub struct ScanStats { + pub total_time_taken_to_read_and_merge_blocks_ms: u64, + pub num_merged_records_in_log: u64, + pub total_log_files: u64, + pub total_log_blocks: u64, + pub total_log_records: u64, + pub total_corrupt_blocks: u64, + pub total_rollbacks: u64, +} + +/// Merged log record reader with timing and builder pattern. +/// +/// Mirrors Java's `HoodieMergedLogRecordReader`. +/// +/// ## Usage (matching Java): +/// ```ignore +/// let reader = HoodieMergedLogRecordReader::new_builder() +/// .with_reader_context(reader_context) +/// .with_storage(storage) +/// .with_log_files(log_file_paths) +/// .with_latest_instant_time(latest_time) +/// .with_instant_range(Some(range)) +/// .with_record_buffer(buffer) +/// .with_allow_inflight_instants(false) +/// .build() +/// .await?; +/// +/// let (buffer, valid_instants, stats) = reader.into_parts(); +/// ``` +pub struct HoodieMergedLogRecordReader { + pub(crate) base: BaseHoodieLogRecordReader, + num_merged_records_in_log: u64, + total_time_taken_to_read_and_merge_blocks_ms: u64, +} + +impl std::fmt::Debug for HoodieMergedLogRecordReader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HoodieMergedLogRecordReader") + .field("num_merged_records", &self.num_merged_records_in_log) + .field( + "total_time_ms", + &self.total_time_taken_to_read_and_merge_blocks_ms, + ) + .finish() + } +} + +impl HoodieMergedLogRecordReader { + /// Create a new builder. + /// + /// Mirrors Java's `HoodieMergedLogRecordReader.newBuilder()`. + pub fn new_builder() -> Builder { + Builder::default() + } + + /// Scan delta-log files processing blocks. + /// + /// Mirrors Java's `scan()`: + /// ```java + /// public final void scan() { scan(false); } + /// ``` + pub async fn scan(&mut self) -> Result<()> { + self.scan_with_skip(false).await + } + + /// Scan with control over block processing. + /// + /// Mirrors Java's `scan(boolean skipProcessingBlocks)`: + /// ```java + /// public final void scan(boolean skipProcessingBlocks) { + /// if (forceFullScan) { return; } // already scanned in constructor + /// scanInternal(Option.empty(), skipProcessingBlocks); + /// } + /// ``` + pub async fn scan_with_skip(&mut self, skip_processing_blocks: bool) -> Result<()> { + if self.base.force_full_scan { + // When full-scan is enforced, scanning is invoked upfront (during initialization) + return Ok(()); + } + self.base.scan_internal(None, skip_processing_blocks).await + } + + /// Mirrors Java's `performScan()`: + /// ```java + /// private void performScan() { + /// timer.startTimer(); + /// Option keySpecOpt = createKeySpec(readerContext.getKeyFilterOpt()); + /// scanInternal(keySpecOpt, false); + /// this.totalTimeTakenToReadAndMergeBlocks = timer.endTimer(); + /// this.numMergedRecordsInLog = recordBuffer.size(); + /// } + /// ``` + async fn perform_scan(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + // Mirrors Java: Option keySpecOpt = createKeySpec(readerContext.getKeyFilterOpt()); + let key_spec_opt = crate::file_group::reader::key_spec::create_key_spec_from_arc( + &self.base.reader_context.key_filter_opt, + ); + + self.base.scan_internal(key_spec_opt, false).await?; + + self.total_time_taken_to_read_and_merge_blocks_ms = start.elapsed().as_millis() as u64; + self.num_merged_records_in_log = self.base.record_buffer.size() as u64; + + log::info!( + "Number of log files scanned => {}", + self.base.log_file_paths.len() + ); + log::info!( + "Number of entries in Map => {}", + self.base.record_buffer.size() + ); + + Ok(()) + } + + /// Decompose the reader after scanning, returning the populated buffer, + /// valid block instants, and scan statistics. + pub fn into_parts(self) -> (Box, Vec, ScanStats) { + let stats = ScanStats { + total_time_taken_to_read_and_merge_blocks_ms: self + .total_time_taken_to_read_and_merge_blocks_ms, + num_merged_records_in_log: self.num_merged_records_in_log, + total_log_files: self.base.total_log_files, + total_log_blocks: self.base.total_log_blocks, + total_log_records: self.base.total_log_records, + total_corrupt_blocks: self.base.total_corrupt_blocks, + total_rollbacks: self.base.total_rollbacks, + }; + + let valid_instants = self.base.valid_block_instants; + let buffer = self.base.record_buffer; + + (buffer, valid_instants, stats) + } + + // ── Getters (delegate to base, mirrors Java's inherited access) ───── + + pub fn get_num_merged_records_in_log(&self) -> u64 { + self.num_merged_records_in_log + } + + pub fn get_total_time_taken_to_read_and_merge_blocks(&self) -> u64 { + self.total_time_taken_to_read_and_merge_blocks_ms + } + + pub fn get_total_log_files(&self) -> u64 { + self.base.get_total_log_files() + } + + pub fn get_total_log_records(&self) -> u64 { + self.base.get_total_log_records() + } + + pub fn get_total_log_blocks(&self) -> u64 { + self.base.get_total_log_blocks() + } + + pub fn get_total_corrupt_blocks(&self) -> u64 { + self.base.get_total_corrupt_blocks() + } + + pub fn get_total_rollbacks(&self) -> u64 { + self.base.get_total_rollbacks() + } + + pub fn get_valid_block_instants(&self) -> &[String] { + self.base.get_valid_block_instants() + } + + pub fn get_progress(&self) -> f32 { + self.base.get_progress() + } +} + +/// Builder for `HoodieMergedLogRecordReader`. +/// +/// Mirrors Java's `HoodieMergedLogRecordReader.Builder extends BaseHoodieLogRecordReader.Builder`. +/// +/// ## Required fields: +/// - `reader_context` +/// - `storage` +/// - `record_buffer` +/// +/// ## Builder flow: +/// `build()` constructs the reader and calls `perform_scan()` when +/// `force_full_scan=true` (the default, matching Java). +pub struct Builder { + reader_context: Option>, + storage: Option>, + log_file_paths: Vec, + latest_instant_time: Option, + instant_range: Option, + /// Whether instant_range was explicitly set via `with_instant_range()`. + instant_range_explicitly_set: bool, + record_buffer: Option>, + /// By default true, matching Java: `private boolean forceFullScan = true;` + force_full_scan: bool, + allow_inflight_instants: bool, +} + +impl Default for Builder { + fn default() -> Self { + Self { + reader_context: None, + storage: None, + log_file_paths: Vec::new(), + latest_instant_time: None, + instant_range: None, + instant_range_explicitly_set: false, + record_buffer: None, + force_full_scan: true, + allow_inflight_instants: false, + } + } +} + +impl Builder { + /// Mirrors Java's `withHoodieReaderContext(readerContext)`. + pub fn with_reader_context(mut self, ctx: Arc) -> Self { + self.reader_context = Some(ctx); + self + } + + /// Mirrors Java's `withStorage(HoodieStorage)`. + pub fn with_storage(mut self, storage: Arc) -> Self { + self.storage = Some(storage); + self + } + + /// Mirrors Java's `withLogFiles(List)`. + pub fn with_log_files(mut self, paths: Vec) -> Self { + self.log_file_paths = paths; + self + } + + pub fn with_latest_instant_time(mut self, time: String) -> Self { + self.latest_instant_time = Some(time); + self + } + + /// Mirrors Java's `withInstantRange(Option)`. + pub fn with_instant_range(mut self, range: Option) -> Self { + self.instant_range = range; + self.instant_range_explicitly_set = true; + self + } + + /// Mirrors Java's `withRecordBuffer(HoodieFileGroupRecordBuffer)`. + pub fn with_record_buffer(mut self, buffer: Box) -> Self { + self.record_buffer = Some(buffer); + self + } + + /// Mirrors Java's `withForceFullScan(boolean)`. + pub fn with_force_full_scan(mut self, force: bool) -> Self { + self.force_full_scan = force; + self + } + + /// Mirrors Java's `withAllowInflightInstants(boolean)`. + pub fn with_allow_inflight_instants(mut self, allow: bool) -> Self { + self.allow_inflight_instants = allow; + self + } + + /// Build and optionally perform scan. + /// + /// Mirrors Java's `build()` which calls the constructor, and the + /// constructor calls `performScan()` when `forceFullScan=true`. + /// + /// If `instant_range` was not explicitly set, it is derived from + /// `reader_context.instant_range` (matching Java's pattern where + /// the caller passes `readerContext.getInstantRange()` to the builder). + pub async fn build(self) -> Result { + let reader_context = self + .reader_context + .ok_or_else(|| CoreError::ReadFileSliceError("reader_context required".into()))?; + let storage = self + .storage + .ok_or_else(|| CoreError::ReadFileSliceError("storage required".into()))?; + let record_buffer = self + .record_buffer + .ok_or_else(|| CoreError::ReadFileSliceError("record_buffer required".into()))?; + + // Derive instant_range from reader_context if not explicitly set + let instant_range = if self.instant_range_explicitly_set { + self.instant_range + } else { + reader_context.instant_range.clone() + }; + + let base = BaseHoodieLogRecordReader { + reader_context, + storage, + log_file_paths: self.log_file_paths, + latest_instant_time: self + .latest_instant_time + .unwrap_or_else(|| "99991231235959999".to_string()), + instant_range, + force_full_scan: self.force_full_scan, + record_buffer, + allow_inflight_instants: self.allow_inflight_instants, + key_spec_opt: None, + valid_block_instants: Vec::new(), + total_log_files: 0, + total_log_blocks: 0, + total_log_records: 0, + total_corrupt_blocks: 0, + total_rollbacks: 0, + progress: 0.0, + }; + + let mut reader = HoodieMergedLogRecordReader { + base, + num_merged_records_in_log: 0, + total_time_taken_to_read_and_merge_blocks_ms: 0, + }; + + // Mirrors Java constructor: if (forceFullScan) { performScan(); } + if self.force_full_scan { + reader.perform_scan().await?; + } + + Ok(reader) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_group::reader::buffer::key_based::KeyBasedFileGroupRecordBuffer; + use crate::file_group::reader::read_stats::HoodieReadStats; + use crate::storage::util::parse_uri; + use std::collections::HashMap; + + fn make_test_reader_context() -> Arc { + let mut ctx = ReaderContext::empty(); + // Prepare schema handler with DeleteContext for buffer construction. + let schema = + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "_hoodie_record_key", + arrow_schema::DataType::Utf8, + false, + )])); + let mut handler = + crate::file_group::reader::schema_handler::FileGroupReaderSchemaHandler::new() + .with_table_schema(schema.clone()) + .with_data_schema(schema); + handler.prepare_required_schema( + true, + &["_hoodie_record_key".to_string()], + &[], + &ctx.table_config, + false, + "COMMIT_TIME_ORDERING", + ); + ctx.schema_handler = handler; + Arc::new(ctx) + } + + fn make_test_buffer() -> Box { + let ctx = make_test_reader_context(); + let stats = HoodieReadStats::default(); + Box::new( + KeyBasedFileGroupRecordBuffer::new(ctx, "COMMIT_TIME_ORDERING".to_string(), &stats) + .unwrap(), + ) + } + + /// Java: TestHoodieMergedLogRecordReader — builder validation + /// + /// Given: Builder without record_buffer + /// When: build() + /// Then: Error returned + #[tokio::test] + async fn test_builder_requires_record_buffer() { + let ctx = make_test_reader_context(); + let storage = Storage::new_with_base_url(parse_uri("file:///tmp").unwrap()).unwrap(); + + let result = HoodieMergedLogRecordReader::new_builder() + .with_reader_context(ctx) + .with_storage(storage) + .with_force_full_scan(false) + .build() + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("record_buffer"), + "Error should mention record_buffer: {err}" + ); + } + + /// Given: Builder without reader_context + /// When: build() + /// Then: Error returned + #[tokio::test] + async fn test_builder_requires_reader_context() { + let storage = Storage::new_with_base_url(parse_uri("file:///tmp").unwrap()).unwrap(); + let buffer = make_test_buffer(); + + let result = HoodieMergedLogRecordReader::new_builder() + .with_storage(storage.clone()) + .with_record_buffer(buffer) + .with_force_full_scan(false) + .build() + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("reader_context"), + "Error should mention reader_context: {err}" + ); + } + + /// Given: Builder with empty log files (no scan needed) + /// When: build() with force_full_scan=true + /// Then: Reader created successfully, num_merged_records=0 + #[tokio::test] + async fn test_perform_scan_empty_log_files() { + let ctx = make_test_reader_context(); + let storage = Storage::new_with_base_url(parse_uri("file:///tmp").unwrap()).unwrap(); + let buffer = make_test_buffer(); + + let reader = HoodieMergedLogRecordReader::new_builder() + .with_reader_context(ctx) + .with_storage(storage.clone()) + .with_log_files(vec![]) // empty + .with_record_buffer(buffer) + .with_force_full_scan(true) + .build() + .await + .unwrap(); + + assert_eq!(reader.get_num_merged_records_in_log(), 0); + assert_eq!(reader.get_total_log_files(), 0); + assert_eq!(reader.get_total_log_blocks(), 0); + } + + /// Given: Builder with force_full_scan=false + /// When: build() + /// Then: Reader created without scanning (lazy mode) + #[tokio::test] + async fn test_builder_lazy_mode() { + let ctx = make_test_reader_context(); + let storage = Storage::new_with_base_url(parse_uri("file:///tmp").unwrap()).unwrap(); + let buffer = make_test_buffer(); + + let reader = HoodieMergedLogRecordReader::new_builder() + .with_reader_context(ctx) + .with_storage(storage.clone()) + .with_record_buffer(buffer) + .with_force_full_scan(false) + .build() + .await + .unwrap(); + + // In lazy mode, no scan performed + assert_eq!(reader.get_num_merged_records_in_log(), 0); + assert_eq!(reader.get_total_time_taken_to_read_and_merge_blocks(), 0); + } + + /// into_parts() returns buffer, valid instants, and stats. + #[tokio::test] + async fn test_into_parts_returns_components() { + let ctx = make_test_reader_context(); + let storage = Storage::new_with_base_url(parse_uri("file:///tmp").unwrap()).unwrap(); + let buffer = make_test_buffer(); + + let reader = HoodieMergedLogRecordReader::new_builder() + .with_reader_context(ctx) + .with_storage(storage.clone()) + .with_log_files(vec![]) + .with_record_buffer(buffer) + .with_force_full_scan(true) + .build() + .await + .unwrap(); + + let (buffer, valid_instants, stats) = reader.into_parts(); + assert_eq!(buffer.size(), 0); + assert!(valid_instants.is_empty()); + assert_eq!(stats.total_log_files, 0); + } +} diff --git a/crates/core/src/file_group/reader/mod.rs b/crates/core/src/file_group/reader/mod.rs new file mode 100644 index 00000000..d8bd9108 --- /dev/null +++ b/crates/core/src/file_group/reader/mod.rs @@ -0,0 +1,791 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Redesigned file group reader matching the Apache Hudi open-source class hierarchy. +//! +//! ## Class hierarchy mapping (Java → Rust) +//! +//! ```text +//! Java Rust +//! ──── ──── +//! HoodieFileGroupReader → HoodieFileGroupReader +//! HoodieReaderContext → (fields on HoodieFileGroupReader) +//! InputSplit → InputSplit +//! ReaderParameters → ReaderParameters +//! HoodieReadStats → HoodieReadStats +//! IteratorMode → IteratorMode +//! FileGroupReaderSchemaHandler → FileGroupReaderSchemaHandler +//! BufferedRecord → BufferedRecord +//! DeleteRecord → DeleteRecord +//! BufferedRecords (factory) → BufferedRecords +//! BufferedRecordConverter → BufferedRecordConverter +//! DeleteContext → DeleteContext +//! BufferedRecordMerger → BufferedRecordMerger (trait) +//! UpdateProcessor → UpdateProcessor (trait) +//! +//! HoodieFileGroupRecordBuffer (interface) → HoodieFileGroupRecordBuffer (trait) +//! FileGroupRecordBuffer (abstract) → FileGroupRecordBuffer (struct, composition) +//! KeyBasedFileGroupRecordBuffer → KeyBasedFileGroupRecordBuffer +//! +//! FileGroupRecordBufferLoader (interface) → FileGroupRecordBufferLoader (trait) +//! LogScanningRecordBufferLoader (abstract) → scan_log_files() function +//! DefaultFileGroupRecordBufferLoader → DefaultFileGroupRecordBufferLoader +//! ``` +//! +//! ## Call stack tree (the implementation window) +//! +//! ```text +//! HoodieFileGroupReader.get_closable_iterator() +//! └─ get_buffered_record_iterator(IteratorMode) +//! └─ init_record_iterators() +//! ├─── make_base_file_iterator() +//! │ └─ storage.get_parquet_file_data(...) +//! └─── record_buffer_loader.get_record_buffer(...) +//! └─ DefaultFileGroupRecordBufferLoader.get_record_buffer() +//! ├─ new KeyBasedFileGroupRecordBuffer(...) +//! └─ scan_log_files(...) +//! └─ LogFileScanner.scan() +//! ``` + +pub mod buffer; +pub mod buffered_record; +pub mod buffered_record_converter; +pub mod delete_context; +pub mod input_split; +pub mod iterator_mode; +pub mod key_spec; +pub mod log_record_reader; +pub mod merged_log_record_reader; +pub mod output_converter; +pub mod read_stats; +pub mod reader_context; +pub mod reader_parameters; +pub mod record_context; +pub mod record_merger; +pub mod schema_handler; +pub mod update_processor; + +use crate::Result; +use crate::error::CoreError; +use crate::file_group::reader::buffer::HoodieFileGroupRecordBuffer; +use crate::file_group::reader::buffer::loader::{ + DefaultFileGroupRecordBufferLoader, FileGroupRecordBufferLoader, +}; +use crate::file_group::reader::buffered_record_converter::BufferedRecordConverter; +use crate::file_group::reader::input_split::InputSplit; +use crate::file_group::reader::iterator_mode::IteratorMode; +use crate::file_group::reader::output_converter::OutputConverter; +use crate::file_group::reader::read_stats::HoodieReadStats; +use crate::file_group::reader::reader_context::ReaderContext; +use crate::file_group::reader::reader_parameters::ReaderParameters; +use crate::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; +use crate::storage::Storage; +use arrow_array::{Array, RecordBatch, StringArray}; +use arrow_schema::SchemaRef; +use std::sync::Arc; + +/// The top-level file group reader orchestrator. +/// +/// Mirrors Java's `org.apache.hudi.common.table.read.HoodieFileGroupReader`. +/// +/// This is the main entry point for reading a file group. It: +/// 1. Accepts an `InputSplit` describing what to read (base file + log files) +/// 2. Creates the [`FileGroupReaderSchemaHandler`] from `data_schema` + `requested_schema` +/// 3. Creates base file iterators via storage +/// 4. Delegates log scanning + buffer creation to `FileGroupRecordBufferLoader` +/// 5. Merges base file records with log records via the buffer +/// 6. Projects output back to `requested_schema` via `OutputConverter` +/// +/// ## Construction +/// +/// Use [`HoodieFileGroupReader::builder()`] for the builder pattern, or construct +/// directly with [`HoodieFileGroupReader::new()`]. +#[derive(Debug)] +pub struct HoodieFileGroupReader { + // ── Context (mirrors Java's HoodieReaderContext) ──────────────── + /// Reader context carrying merge mode, instant range, and config maps. + reader_context: Arc, + + /// Storage for reading base files and log files. + storage: Arc, + + // ── Input ────────────────────────────────────────────────────────── + /// Describes what to read: base file, log files, partition path. + input_split: InputSplit, + + // ── Configuration ────────────────────────────────────────────────── + /// Reader flags: use_record_position, emit_delete, sort_output, etc. + reader_parameters: ReaderParameters, + + /// The current iterator mode. + #[allow(dead_code)] + iterator_mode: IteratorMode, + + // ── Schema (mirrors Java's readerContext.getSchemaHandler()) ─────── + /// Schema handler created in the constructor from `data_schema` + + /// `requested_schema`, exactly like Java lines 119-121. + /// Owns the `required_schema` used for base file projection and the + /// `output_converter` used for final projection. + schema_handler: FileGroupReaderSchemaHandler, + + // ── Strategy ─────────────────────────────────────────────────────── + /// Buffer loader: selects buffer impl + triggers log scan. + record_buffer_loader: DefaultFileGroupRecordBufferLoader, + + // ── Mutable state (populated during read) ────────────────────────── + /// Core structure to store and process records. + /// Mirrors Java's `HoodieFileGroupRecordBuffer recordBuffer`. + record_buffer: Option>, + + /// Base file record iterator (set during `initRecordIterators`). + /// Mirrors Java's `ClosableIterator baseFileIterator`. + base_file_iterator: Option>, + + /// Optional converter for projecting/transforming output records. + /// Mirrors Java's `Option> outputConverter`. + output_converter: Option>, + + /// Read statistics accumulator. + read_stats: HoodieReadStats, + + /// Valid block instants from log scanning. + valid_block_instants: Vec, + + /// Converter for engine records to [`BufferedRecord`]. + /// Mirrors Java's `BufferedRecordConverter bufferedRecordConverter`. + buffered_record_converter: Option>, +} + +impl HoodieFileGroupReader { + /// Create a new file group reader. + /// + /// Mirrors Java's `HoodieFileGroupReader(readerContext, storage, tablePath, + /// latestCommitTime, dataSchema, requestedSchema, ...)` constructor. + /// + /// The constructor: + /// 1. Creates a [`FileGroupReaderSchemaHandler`] from `data_schema` + + /// `requested_schema` (Java lines 119-121) + /// 2. Calls `prepare_required_schema()` to compute the `required_schema` + /// (Java: automatic in `FileGroupReaderSchemaHandler` constructor, line 105) + /// 3. Obtains the `output_converter` from the schema handler (Java line 122) + /// + /// # Arguments + /// * `reader_context` — Engine context with merge mode, ordering fields, table config. + /// * `storage` — Storage layer for reading base files and log files. + /// * `input_split` — Describes what to read (base file path, log file paths, partition). + /// * `reader_parameters` — Reader flags (use_record_position, emit_delete, etc.). + /// * `data_schema` — Full table schema (what columns exist in the files). + /// Maps to Java's `dataSchema` / `tableSchema` parameter. + /// * `requested_schema` — Column projection requested by the caller. + /// Maps to Java's `requestedSchema` parameter. `None` means all columns. + pub fn new( + reader_context: Arc, + storage: Arc, + input_split: InputSplit, + reader_parameters: ReaderParameters, + data_schema: Option, + requested_schema: Option, + ) -> Self { + log::debug!( + "HoodieFileGroupReader::new partition={} base_file={} log_files={} \ + ordering_fields={:?} latest_commit_time={} record_key_field={}", + input_split.partition_path, + input_split.base_file_path.as_deref().unwrap_or(""), + input_split.log_file_paths.len(), + reader_context.ordering_field_names(), + reader_context.latest_commit_time, + reader_context.record_key_field(), + ); + for (i, lf) in input_split.log_file_paths.iter().enumerate() { + log::debug!(" log_file[{i}]: {lf}"); + } + + // Mirrors Java lines 119-121: + // readerContext.setSchemaHandler( + // new FileGroupReaderSchemaHandler(readerContext, dataSchema, requestedSchema, ...)); + // + // When schemas are explicitly provided (direct construction / tests), create + // a new handler. When they are not provided (FFI path via builder), use the + // handler already on reader_context — which was populated by the FFI bridge + // from the Avro JSON schemas passed through the Substrait proto. + let mut schema_handler = if data_schema.is_some() || requested_schema.is_some() { + let mut handler = FileGroupReaderSchemaHandler::new(); + if let Some(ds) = data_schema { + handler = handler.with_table_schema(ds.clone()).with_data_schema(ds); + } + if let Some(rs) = requested_schema { + handler = handler.with_requested_schema(rs); + } + handler + } else { + reader_context.schema_handler.clone() + }; + + // Mirrors Java FileGroupReaderSchemaHandler constructor line 105: + // this.requiredSchema = prepareRequiredSchema(this.deleteContext); + // + // Uses record_key_fields() (all key fields) instead of record_key_field() + // (single) to support composite record keys in virtual-key mode. + // Mirrors Java's getMandatoryFieldsForMerging() lines 250-258. + let has_instant_range = reader_context.instant_range.is_some(); + schema_handler.prepare_required_schema( + input_split.has_log_files(), + &reader_context.record_key_fields(), + &reader_context.ordering_field_names().to_vec(), + &reader_context.table_config, + has_instant_range, + &reader_context.merge_mode, + ); + + // Bootstrap merge reordering is not yet supported in hudi-rs. + // Java's prepareRequiredSchema() (lines 280-288) partitions fields into + // meta and data columns and reorders them for bootstrap tables. Until + // that is implemented, reject bootstrap merge at construction time. + if reader_context.needs_bootstrap_merge { + panic!( + "Bootstrap merge is not yet supported in hudi-rs. \ + Table at '{}' has bootstrap base files that require \ + meta/data column reordering.", + reader_context.table_path, + ); + } + + // Mirrors Java line 122: + // this.outputConverter = readerContext.getSchemaHandler().getOutputConverter(); + let output_converter = schema_handler.get_output_converter(); + + // Propagate the prepared schema_handler back onto a new reader_context + // so downstream consumers (record buffer, log scanner) see the canonical + // schema_handler with its stored DeleteContext. Mirrors Java's + // `readerContext.setSchemaHandler(...)` — in Java the reader context is + // mutable; in Rust we create a new Arc with the updated handler. + let reader_context = { + let mut updated = (*reader_context).clone(); + updated.schema_handler = schema_handler.clone(); + Arc::new(updated) + }; + + Self { + reader_context, + storage, + input_split, + reader_parameters, + iterator_mode: IteratorMode::EngineRecord, + schema_handler, + record_buffer_loader: DefaultFileGroupRecordBufferLoader::new(), + record_buffer: None, + base_file_iterator: None, + output_converter, + read_stats: HoodieReadStats::default(), + valid_block_instants: Vec::new(), + buffered_record_converter: None, + } + } + + /// Create a builder for configuring the reader. + pub fn builder() -> HoodieFileGroupReaderBuilder { + HoodieFileGroupReaderBuilder::default() + } + + // ========================================================================= + // Main read API (mirrors Java's getClosableIterator / getBufferedRecordIterator) + // ========================================================================= + + /// Read the file group and return the merged output as a `RecordBatch`. + /// + /// This is the main entry point, equivalent to Java's + /// `getClosableIterator()` → `getBufferedRecordIterator()` → `initRecordIterators()`. + /// + /// In Java, this returns a lazy iterator. In Rust/Arrow, we return + /// the fully merged `RecordBatch` since Arrow's columnar format makes + /// batch-level merging more efficient than per-record iteration. + pub async fn read(&mut self) -> Result { + self.init_record_iterators().await + } + + /// Initialize record iterators: read base file + scan/merge log files. + /// + /// Mirrors Java's `HoodieFileGroupReader.initRecordIterators()`. + /// + /// ```text + /// initRecordIterators() + /// ├─ makeBaseFileIterator() + /// └─ recordBufferLoader.getRecordBuffer(...) + /// → recordBuffer.setBaseFileIterator(baseFileIterator) + /// ``` + async fn init_record_iterators(&mut self) -> Result { + log::debug!( + "[HoodieFileGroupReader] initRecordIterators: partition={} base_file={} log_files={}", + self.input_split.partition_path, + self.input_split + .base_file_path + .as_deref() + .unwrap_or(""), + self.input_split.log_file_paths.len(), + ); + + // Step 1: Make base file iterator (read base file data) + // Mirrors Java: this.baseFileIterator = makeBaseFileIterator(); + let mut base_file_batches = self.make_base_file_batches().await?; + let base_rows: usize = base_file_batches.iter().map(|b| b.num_rows()).sum(); + log::debug!( + "[HoodieFileGroupReader] makeBaseFileIterator: {} batches, {} total rows", + base_file_batches.len(), + base_rows, + ); + if let Some(first) = base_file_batches.first() { + let schema = first.schema(); + let field_names: Vec<&str> = + schema.fields().iter().map(|f| f.name().as_str()).collect(); + log::debug!( + "[HoodieFileGroupReader] base file schema: {:?}", + field_names + ); + } + + // Apply instant range filter on base file records. + // Mirrors Java's makeBaseFileIterator() lines 169-171: + // return readerContext.getInstantRange().isPresent() + // ? readerContext.applyInstantRangeFilter(recordIterator) + // : recordIterator; + if self.reader_context.instant_range.is_some() { + base_file_batches = self.apply_instant_range_filter(base_file_batches)?; + let filtered_rows: usize = base_file_batches.iter().map(|b| b.num_rows()).sum(); + log::debug!( + "[HoodieFileGroupReader] applyInstantRangeFilter: {base_rows} → {filtered_rows} rows" + ); + } + + self.base_file_iterator = Some(base_file_batches); + + // Step 2: If no records to merge (no log files), return base file data directly + if self.input_split.has_no_records_to_merge() { + let batches = self.base_file_iterator.as_ref().unwrap(); + log::debug!( + "[HoodieFileGroupReader] no log files → returning base file data directly ({base_rows} rows)" + ); + if batches.is_empty() { + // No base file and no log files — return empty batch. + return self.apply_output_converter(RecordBatch::new_empty(std::sync::Arc::new( + arrow_schema::Schema::empty(), + ))); + } + let result = + arrow::compute::concat_batches(&batches[0].schema(), batches).map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to concatenate base file batches: {e}" + )) + })?; + return self.apply_output_converter(result); + } + + // Step 3: Load record buffer (scan log files + create buffer) + // Mirrors Java: this.recordBuffer = recordBufferLoader.getRecordBuffer(...).getLeft(); + log::debug!( + "[HoodieFileGroupReader] scanning {} log file(s) with latest_commit_time={}", + self.input_split.log_file_paths.len(), + self.reader_context.latest_commit_time, + ); + let load_result = self + .record_buffer_loader + .get_record_buffer( + self.reader_context.clone(), + self.storage.clone(), + &self.input_split, + &self.reader_parameters, + &mut self.read_stats, + ) + .await?; + + self.record_buffer = Some(load_result.record_buffer); + self.valid_block_instants = load_result.valid_block_instants; + + let record_buffer = self.record_buffer.as_mut().unwrap(); + + log::debug!( + "[HoodieFileGroupReader] log scan complete: buffer_size={} valid_instants={:?} \ + stats: log_blocks={} log_records={} corrupt={} rollbacks={}", + record_buffer.size(), + self.valid_block_instants, + self.read_stats.total_log_blocks, + self.read_stats.total_log_records, + self.read_stats.total_corrupt_log_blocks, + self.read_stats.total_rollback_blocks, + ); + + // Step 4a: reader_schema is now set at buffer construction time + // (in KeyBasedFileGroupRecordBuffer::new), matching Java's + // FileGroupRecordBuffer constructor line 102. + + // Step 4b: Set base file iterator on the buffer + // Mirrors Java: recordBuffer.setBaseFileIterator(baseFileIterator); + let base_batches = self.base_file_iterator.take().unwrap_or_default(); + record_buffer.set_base_file_iterator(base_batches); + log::debug!( + "[HoodieFileGroupReader] set base file iterator ({base_rows} rows), starting merge_and_collect" + ); + + // Step 5: Merge and collect — take ownership of the buffer since + // merge_and_collect consumes it (mirrors Java's close() lifecycle). + let record_buffer = self.record_buffer.take().unwrap(); + let result = record_buffer.merge_and_collect()?; + log::debug!( + "[HoodieFileGroupReader] merge_and_collect complete: {} rows, {} columns", + result.num_rows(), + result.num_columns(), + ); + self.apply_output_converter(result) + } + + /// Read base file data and initialize the buffered record converter. + /// + /// Mirrors Java's `HoodieFileGroupReader.makeBaseFileIterator()` which: + /// 1. Returns empty iterator if no base file + /// 2. Creates `bufferedRecordConverter` via `BufferedRecordConverter.createConverter(...)` + /// 3. Reads the base file via `readerContext.getFileRecordIterator(...)` + async fn make_base_file_batches(&mut self) -> Result> { + match &self.input_split.base_file_path { + Some(path) => { + // Mirrors Java: this.bufferedRecordConverter = BufferedRecordConverter.createConverter( + // readerContext.getIteratorMode(), readerContext.getSchemaHandler().getRequiredSchema(), + // readerContext.getRecordContext(), orderingFieldNames); + // In Rust, the converter is set externally or already present; log if missing. + if self.buffered_record_converter.is_none() { + log::debug!( + "[HoodieFileGroupReader] makeBaseFileIterator: no bufferedRecordConverter set \ + (batch-level read does not require per-record conversion)" + ); + } + + // Mirrors Java line 159-162: + // readerContext.getFileRecordIterator(pathInfo, start, len, + // schemaHandler.getTableSchema(), schemaHandler.getRequiredSchema(), storage) + let batch = if let Some(required_schema) = &self.schema_handler.required_schema { + self.storage + .get_parquet_file_data_projected(path, required_schema) + .await + .map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to read base file '{path}' with projection: {e:?}" + )) + })? + } else { + self.storage + .get_parquet_file_data(path) + .await + .map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to read base file '{path}': {e:?}" + )) + })? + }; + // Mirrors Java HoodieAvroReaderContext.getFileRecordIterator + // (lines 218-228): when keyFilterOpt is set and the reader does + // not support a native key predicate (Parquet has none), apply + // a row-level filter on _hoodie_record_key. + let batch = apply_key_filter_to_batch( + batch, + self.reader_context.key_filter_opt.as_deref(), + )?; + Ok(vec![batch]) + } + None => Ok(Vec::new()), + } + } + + /// Apply the output converter to the final result, if set. + /// + /// Mirrors Java's `next()`: + /// ```java + /// if (outputConverter.isPresent()) { + /// return nextVal.project(outputConverter.get()); + /// } + /// ``` + fn apply_output_converter(&self, batch: RecordBatch) -> Result { + match &self.output_converter { + Some(converter) => converter.apply(batch), + None => Ok(batch), + } + } + + /// Filter base file records by instant range on `_hoodie_commit_time`. + /// + /// Mirrors Java's `HoodieReaderContext.applyInstantRangeFilter()` (lines 354-365): + /// ```java + /// if (HoodieTableMetadata.isMetadataTable(tablePath)) { + /// return fileRecordIterator; // skip for metadata table + /// } + /// InstantRange instantRange = getInstantRange().get(); + /// int commitTimePos = schemaHandler.getRequiredSchema().getField(COMMIT_TIME_METADATA_FIELD).pos(); + /// Predicate instantFilter = row -> instantRange.isInRange(getMetaFieldValue(row, commitTimePos)); + /// return new CloseableFilterIterator<>(fileRecordIterator, instantFilter); + /// ``` + /// + /// In Rust/Arrow, we use columnar filtering: extract the `_hoodie_commit_time` + /// column, build a boolean mask, and use `arrow::compute::filter_record_batch`. + fn apply_instant_range_filter(&self, batches: Vec) -> Result> { + let instant_range = match &self.reader_context.instant_range { + Some(range) => range, + None => return Ok(batches), + }; + + // Skip filtering for metadata table (mirrors Java line 356). + if crate::util::path::is_metadata_table_path(&self.reader_context.table_path) { + return Ok(batches); + } + + let timezone = self.reader_context.timezone(); + let mut filtered = Vec::with_capacity(batches.len()); + + for batch in batches { + if batch.num_rows() == 0 { + filtered.push(batch); + continue; + } + + // Find _hoodie_commit_time column position. + let commit_time_idx = match batch.schema().index_of("_hoodie_commit_time") { + Ok(idx) => idx, + Err(_) => { + // Column not present — cannot filter. Pass through. + // This can happen if the schema didn't include commit_time + // (e.g., schema was not properly prepared). + log::warn!( + "[HoodieFileGroupReader] applyInstantRangeFilter: \ + _hoodie_commit_time column not found in base file schema, \ + skipping filter" + ); + filtered.push(batch); + continue; + } + }; + + let commit_time_col = batch + .column(commit_time_idx) + .as_any() + .downcast_ref::(); + + let commit_time_col = match commit_time_col { + Some(col) => col, + None => { + log::warn!( + "[HoodieFileGroupReader] applyInstantRangeFilter: \ + _hoodie_commit_time is not a StringArray, skipping filter" + ); + filtered.push(batch); + continue; + } + }; + + // Build boolean mask: true for rows in range. + let mut mask_builder = + arrow_array::builder::BooleanBuilder::with_capacity(batch.num_rows()); + for i in 0..commit_time_col.len() { + if commit_time_col.is_null(i) { + mask_builder.append_value(false); + } else { + let ts = commit_time_col.value(i); + let in_range = instant_range.is_in_range(ts, &timezone).map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to check instant range for '{ts}': {e}" + )) + })?; + mask_builder.append_value(in_range); + } + } + let mask = mask_builder.finish(); + + let filtered_batch = + arrow::compute::filter_record_batch(&batch, &mask).map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to filter base file batch by instant range: {e}" + )) + })?; + filtered.push(filtered_batch); + } + + Ok(filtered) + } + + // ========================================================================= + // Setters (mirrors Java's mutable field assignments) + // ========================================================================= + + /// Set the output converter. + /// Mirrors Java: `this.outputConverter = readerContext.getSchemaHandler().getOutputConverter()`. + pub fn set_output_converter(&mut self, converter: Box) { + self.output_converter = Some(converter); + } + + /// Set the buffered record converter. + /// Mirrors Java: `this.bufferedRecordConverter = BufferedRecordConverter.createConverter(...)`. + pub fn set_buffered_record_converter(&mut self, converter: Box) { + self.buffered_record_converter = Some(converter); + } + + // ========================================================================= + // Accessors + // ========================================================================= + + /// Returns the read statistics collected during the read. + pub fn read_stats(&self) -> &HoodieReadStats { + &self.read_stats + } + + /// Returns the valid block instants from log scanning. + pub fn valid_block_instants(&self) -> &[String] { + &self.valid_block_instants + } +} + +// ========================================================================= +// Builder +// ========================================================================= + +/// Builder for `HoodieFileGroupReader`. +/// +/// Mirrors Java's `HoodieFileGroupReader.Builder`. +#[derive(Debug, Default)] +pub struct HoodieFileGroupReaderBuilder { + reader_context: Option>, + storage: Option>, + input_split: Option, + reader_parameters: ReaderParameters, + data_schema: Option, + requested_schema: Option, +} + +impl HoodieFileGroupReaderBuilder { + pub fn with_reader_context(mut self, ctx: Arc) -> Self { + self.reader_context = Some(ctx); + self + } + + pub fn with_storage(mut self, storage: Arc) -> Self { + self.storage = Some(storage); + self + } + + pub fn with_input_split(mut self, input_split: InputSplit) -> Self { + self.input_split = Some(input_split); + self + } + + pub fn with_reader_parameters(mut self, params: ReaderParameters) -> Self { + self.reader_parameters = params; + self + } + + /// Set the data schema (full table schema). + /// Mirrors Java's `Builder.withDataSchema(Schema dataSchema)`. + pub fn with_data_schema(mut self, schema: SchemaRef) -> Self { + self.data_schema = Some(schema); + self + } + + /// Set the requested schema (column projection). + /// Mirrors Java's `Builder.withRequestedSchema(Schema requestedSchema)`. + pub fn with_requested_schema(mut self, schema: SchemaRef) -> Self { + self.requested_schema = Some(schema); + self + } + + pub fn build(self) -> Result { + let reader_context = self + .reader_context + .ok_or_else(|| CoreError::ReadFileSliceError("reader_context is required".into()))?; + let storage = self + .storage + .ok_or_else(|| CoreError::ReadFileSliceError("storage is required".into()))?; + let input_split = self + .input_split + .ok_or_else(|| CoreError::ReadFileSliceError("input_split is required".into()))?; + + let reader = HoodieFileGroupReader::new( + reader_context, + storage, + input_split, + self.reader_parameters, + self.data_schema, + self.requested_schema, + ); + + Ok(reader) + } +} + +// ========================================================================= +// Key filter helper +// ========================================================================= + +/// Filter rows in `batch` so only those whose `_hoodie_record_key` matches +/// the given key-spec (derived from `key_filter_opt`) survive. If +/// `key_filter_opt` is None or doesn't yield a KeySpec, returns the batch +/// unchanged. +/// +/// Mirrors Java HoodieAvroReaderContext.getFileRecordIterator (lines +/// 218-228) for the non-key-predicate-supporting reader case. +fn apply_key_filter_to_batch( + batch: RecordBatch, + key_filter_opt: Option<&dyn crate::expression::predicate::Predicate>, +) -> Result { + use crate::file_group::reader::key_spec::create_key_spec; + let key_spec = match create_key_spec(key_filter_opt) { + Some(spec) => spec, + None => return Ok(batch), + }; + + // Find the _hoodie_record_key column. + let key_col_idx = match batch.schema().index_of("_hoodie_record_key") { + Ok(i) => i, + Err(_) => { + // Column not present (e.g., virtual keys). Return batch as-is — + // the filter cannot apply. + log::debug!( + "[HoodieFileGroupReader] key_filter_opt set but _hoodie_record_key \ + column not present in base file; skipping base-file filter" + ); + return Ok(batch); + } + }; + + let key_col = batch + .column(key_col_idx) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + CoreError::ReadFileSliceError( + "_hoodie_record_key column is not StringArray".to_string(), + ) + })?; + + // Build a BooleanArray mask: true for rows whose key matches the spec. + let mask: arrow_array::BooleanArray = (0..batch.num_rows()) + .map(|i| { + if key_col.is_null(i) { + Some(false) + } else { + Some(key_spec.matches(key_col.value(i))) + } + }) + .collect(); + + arrow::compute::filter_record_batch(&batch, &mask) + .map_err(|e| CoreError::ReadFileSliceError(format!("filter_record_batch: {e}"))) +} diff --git a/crates/core/src/file_group/reader/output_converter.rs b/crates/core/src/file_group/reader/output_converter.rs new file mode 100644 index 00000000..6cbd96bc --- /dev/null +++ b/crates/core/src/file_group/reader/output_converter.rs @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors Java's `Option> outputConverter` field +//! on `HoodieFileGroupReader`. +//! +//! The output converter applies a final transformation (typically schema +//! projection) to each record returned by the reader. In Java it is +//! obtained from `readerContext.getSchemaHandler().getOutputConverter()`. + +use crate::Result; +use crate::error::CoreError; +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; + +/// A converter that transforms output records (e.g., schema projection). +/// +/// Mirrors Java's `UnaryOperator` used as `outputConverter` in +/// `HoodieFileGroupReader`. Obtained from `FileGroupReaderSchemaHandler`. +pub trait OutputConverter: Send + Sync + std::fmt::Debug { + /// Apply the conversion to a record batch. + fn apply(&self, batch: RecordBatch) -> Result; +} + +/// Projects a `RecordBatch` from `required_schema` down to `requested_schema`. +/// +/// Mirrors Java's `readerContext.getRecordContext().projectRecord(requiredSchema, requestedSchema)`. +/// Selects only the columns present in the target schema, in the order they appear. +#[derive(Debug)] +pub struct ProjectionConverter { + /// Column names to project to (the requested schema columns), in order. + target_columns: Vec, +} + +impl ProjectionConverter { + pub fn new(target_schema: &SchemaRef) -> Self { + let target_columns = target_schema + .fields() + .iter() + .map(|f| f.name().clone()) + .collect(); + Self { target_columns } + } +} + +impl OutputConverter for ProjectionConverter { + fn apply(&self, batch: RecordBatch) -> Result { + let indices: Vec = self + .target_columns + .iter() + .map(|name| { + batch.schema().index_of(name).map_err(|_| { + CoreError::ReadFileSliceError(format!( + "Output projection: column '{}' not found in batch schema. \ + Available: [{}]", + name, + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>() + .join(", ") + )) + }) + }) + .collect::>>()?; + + batch + .project(&indices) + .map_err(|e| CoreError::ReadFileSliceError(format!("Output projection failed: {e}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int64Array, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use std::sync::Arc; + + /// Test that ProjectionConverter correctly narrows a RecordBatch. + #[test] + fn test_output_converter_projects_columns() { + let schema = Arc::new(Schema::new(vec![ + Field::new("col_a", DataType::Utf8, true), + Field::new("col_b", DataType::Int64, true), + Field::new("col_c", DataType::Utf8, true), + ])); + + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["a1", "a2"])), + Arc::new(Int64Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["c1", "c2"])), + ], + ) + .unwrap(); + + // Project to only col_a and col_c. + let target_schema = Arc::new(Schema::new(vec![ + Field::new("col_a", DataType::Utf8, true), + Field::new("col_c", DataType::Utf8, true), + ])); + let converter = ProjectionConverter::new(&target_schema); + let projected = converter.apply(batch).unwrap(); + + assert_eq!(projected.num_columns(), 2); + assert_eq!(projected.schema().field(0).name(), "col_a"); + assert_eq!(projected.schema().field(1).name(), "col_c"); + assert_eq!(projected.num_rows(), 2); + } + + /// Test that ProjectionConverter returns all columns when target matches source. + #[test] + fn test_output_converter_noop_when_same_schema() { + let schema = Arc::new(Schema::new(vec![ + Field::new("col_a", DataType::Utf8, true), + Field::new("col_b", DataType::Int64, true), + ])); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["a1"])), + Arc::new(Int64Array::from(vec![1])), + ], + ) + .unwrap(); + + let converter = ProjectionConverter::new(&schema); + let projected = converter.apply(batch).unwrap(); + + assert_eq!(projected.num_columns(), 2); + } + + /// Test that ProjectionConverter errors on missing column. + #[test] + fn test_output_converter_error_on_missing_column() { + let schema = Arc::new(Schema::new(vec![Field::new("col_a", DataType::Utf8, true)])); + + let batch = + RecordBatch::try_new(schema, vec![Arc::new(StringArray::from(vec!["a1"]))]).unwrap(); + + let target_schema = Arc::new(Schema::new(vec![ + Field::new("col_a", DataType::Utf8, true), + Field::new("col_missing", DataType::Utf8, true), + ])); + let converter = ProjectionConverter::new(&target_schema); + let result = converter.apply(batch); + + assert!(result.is_err()); + } +} diff --git a/crates/core/src/file_group/reader/read_stats.rs b/crates/core/src/file_group/reader/read_stats.rs new file mode 100644 index 00000000..f39cea93 --- /dev/null +++ b/crates/core/src/file_group/reader/read_stats.rs @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.HoodieReadStats`. +//! +//! Mutable accumulator for read statistics, written during log scanning +//! and buffer processing, read by the caller after the read completes. + +/// Mutable accumulator for file group read statistics. +#[derive(Debug, Clone, Default)] +pub struct HoodieReadStats { + pub num_inserts: u64, + pub num_updates: u64, + pub num_deletes: u64, + pub total_log_read_time_ms: u64, + pub total_log_records: u64, + pub total_log_files_compacted: u64, + pub total_log_blocks: u64, + pub total_corrupt_log_blocks: u64, + pub total_rollback_blocks: u64, +} diff --git a/crates/core/src/file_group/reader/reader_context.rs b/crates/core/src/file_group/reader/reader_context.rs new file mode 100644 index 00000000..0c72dd73 --- /dev/null +++ b/crates/core/src/file_group/reader/reader_context.rs @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors Java's `HoodieReaderContext` — the engine-agnostic reader context +//! that flows through the entire file group reader call stack. +//! +//! In Java, `HoodieReaderContext` is an engine-specific object (Spark, Flink) +//! that carries merge mode, instant range, schema handler, etc. In Rust, we work +//! directly with Arrow `RecordBatch`es, so this is a plain data struct carrying +//! the structured configuration that the reader stack needs. + +use super::record_context::RecordContext; +use super::schema_handler::FileGroupReaderSchemaHandler; +use crate::config::table::HudiTableConfig; +use crate::expression::predicate::Predicate; +use crate::timeline::selector::InstantRange; +use std::collections::HashMap; +use std::sync::Arc; + +/// Reader context that flows through the file group reader call stack. +/// +/// Mirrors Java's `HoodieReaderContext`, carrying structured reader +/// configuration instead of raw config key-value maps. +/// +/// ## Java counterpart +/// +/// | Java field / method | Rust field | +/// |--------------------------------------------|-----------------------------------------| +/// | `readerContext.getTablePath()` | `table_path` | +/// | `readerContext.getLatestCommitTime()` | `latest_commit_time` | +/// | `readerContext.getMergeMode()` | `merge_mode` | +/// | `readerContext.getInstantRange()` | `instant_range` | +/// | `readerContext.getRecordContext().format()` | `base_file_format` | +/// | `readerContext.getHasLogFiles()` | `has_log_files` | +/// | `readerContext.getRecordContext()` | `record_context` | +/// | `readerContext.getSchemaHandler()` | `schema_handler` | +/// | `metaClient.getTableConfig()` (config map) | `table_config` | +/// | `props` (hoodie reader config overrides) | `hoodie_reader_config` | +/// | `readerContext.getKeyFilterOpt()` | `key_filter_opt` | +#[derive(Debug, Clone)] +pub struct ReaderContext { + pub table_path: String, + pub latest_commit_time: String, + pub base_file_format: String, + pub has_log_files: bool, + pub has_bootstrap_base_file: bool, + pub needs_bootstrap_merge: bool, + pub should_merge_use_record_position: bool, + pub enable_logical_timestamp_field_repair: bool, + pub iterator_mode: String, + pub merge_mode: String, + pub merge_strategy_id: String, + pub instant_range: Option, + /// The engine-specific record context for record-level operations. + /// + /// Mirrors Java's `HoodieReaderContext.recordContext` field. + /// In Java this is a persistent mutable field set at construction and + /// shared across all consumers. In Rust it is set once and shared via + /// `Arc`. + /// + /// Constructed from `table_config` + `partition_path`, mirroring Java's + /// `RecordContext(tableConfig, typeConverter)`. + pub record_context: RecordContext, + /// Schema management for the read pipeline. + /// + /// Mirrors Java's `HoodieReaderContext.schemaHandler` field + /// (`FileGroupReaderSchemaHandler`). + pub schema_handler: FileGroupReaderSchemaHandler, + pub table_config: HashMap, + pub hoodie_reader_config: HashMap, + /// Mirrors Java `HoodieReaderContext.keyFilterOpt` — an optional + /// predicate (typically `In` or `StringStartsWithAny`) used to + /// narrow which records are scanned. None by default. See spec §5.1. + pub key_filter_opt: Option>, +} + +impl ReaderContext { + /// Get the record context for record-level operations. + /// + /// Mirrors Java's `HoodieReaderContext.getRecordContext()`. + /// + /// In Java, `RecordContext` is a persistent field on the reader context, + /// shared across all consumers. In Rust, it is stored as a field and + /// returned by reference. + pub fn get_record_context(&self) -> &RecordContext { + &self.record_context + } + + /// Convenience accessor — mirrors Java's + /// `readerContext.getRecordContext().recordKeyField`. + pub fn record_key_field(&self) -> &str { + &self.record_context.record_key_field + } + + /// Return ALL record key field names for schema computation. + /// + /// Mirrors Java's `getMandatoryFieldsForMerging()` (lines 250-258): + /// ```java + /// if (cfg.populateMetaFields()) { + /// requiredFields.add(HoodieRecord.RECORD_KEY_METADATA_FIELD); + /// } else { + /// requiredFields.addAll(Arrays.asList(cfg.getRecordKeyFields().get())); + /// } + /// ``` + /// + /// When `populateMetaFields` is true, returns `["_hoodie_record_key"]`. + /// When false (virtual keys), returns ALL configured record key fields + /// (supports composite keys like `["pk1", "pk2"]`). + /// + /// This differs from `record_key_field()` which returns only the first + /// field (used for per-record key extraction in the buffer). + pub fn record_key_fields(&self) -> Vec { + let populate = self + .table_config + .get(HudiTableConfig::PopulatesMetaFields.as_ref()) + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(true); + if populate { + vec![ + crate::metadata::meta_field::MetaField::RecordKey + .as_ref() + .to_string(), + ] + } else { + self.table_config + .get(HudiTableConfig::RecordKeyFields.as_ref()) + .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect()) + .unwrap_or_else(|| { + vec![ + crate::metadata::meta_field::MetaField::RecordKey + .as_ref() + .to_string(), + ] + }) + } + } + + /// Convenience accessor — mirrors Java's ordering field names used by + /// the merge pipeline. + pub fn ordering_field_names(&self) -> &[String] { + &self.record_context.ordering_field_names + } + + /// Convenience accessor for the partition path stored on record_context. + pub fn partition_path(&self) -> &str { + &self.record_context.partition_path + } + + /// Get the timeline timezone from table config, defaulting to "utc". + pub fn timezone(&self) -> String { + self.table_config + .get(HudiTableConfig::TimelineTimezone.as_ref()) + .cloned() + .unwrap_or_else(|| "utc".to_string()) + } + + /// Mirrors Java `HoodieReaderContext.getKeyFilterOpt()`. + pub fn get_key_filter_opt(&self) -> Option> { + self.key_filter_opt.clone() + } + + /// Rebuild the stored `record_context` from the current `table_config` + /// and the given `partition_path`. + /// + /// Use this after mutating `table_config` to keep `record_context` in sync. + pub fn rebuild_record_context(&mut self, partition_path: String) { + self.record_context = RecordContext::new(&self.table_config, partition_path); + } + + /// Create an empty reader context (for legacy/test code). + pub fn empty() -> Self { + Self { + table_path: String::new(), + latest_commit_time: String::new(), + base_file_format: String::new(), + has_log_files: false, + has_bootstrap_base_file: false, + needs_bootstrap_merge: false, + should_merge_use_record_position: false, + enable_logical_timestamp_field_repair: false, + iterator_mode: String::new(), + merge_mode: String::new(), + merge_strategy_id: String::new(), + instant_range: None, + record_context: RecordContext::default(), + schema_handler: FileGroupReaderSchemaHandler::new(), + table_config: HashMap::new(), + hoodie_reader_config: HashMap::new(), + key_filter_opt: None, + } + } +} + +#[cfg(test)] +mod key_filter_opt_tests { + use super::*; + use crate::expression::predicates::predicates_factory; + use crate::expression::{Expression, Literal, NameReference}; + use std::sync::Arc; + + #[test] + fn empty_reader_context_has_no_key_filter() { + let ctx = ReaderContext::empty(); + assert!(ctx.get_key_filter_opt().is_none()); + } + + #[test] + fn key_filter_opt_can_be_set_and_retrieved() { + let mut ctx = ReaderContext::empty(); + let pred = predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string("k1")) as Box], + ); + ctx.key_filter_opt = Some(Arc::new(pred)); + assert!(ctx.get_key_filter_opt().is_some()); + } +} diff --git a/crates/core/src/file_group/reader/reader_parameters.rs b/crates/core/src/file_group/reader/reader_parameters.rs new file mode 100644 index 00000000..88dd3746 --- /dev/null +++ b/crates/core/src/file_group/reader/reader_parameters.rs @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.ReaderParameters`. +//! +//! Pure query-object holding the four boolean flags that control how the +//! file group reader behaves. + +/// Configuration flags for the file group reader. +/// +/// These are extracted from the reader context and drive the buffer +/// selection logic in [`super::buffer::loader::DefaultFileGroupRecordBufferLoader`]. +#[derive(Debug, Clone)] +pub struct ReaderParameters { + /// Whether to use record positions for merging (position-based merge). + pub use_record_position: bool, + + /// Whether to emit delete records in the output. + pub emit_delete: bool, + + /// Whether to sort the output records. + pub sort_output: bool, + + /// Whether to allow in-flight (uncommitted) instants during log scanning. + pub allow_inflight_instants: bool, +} + +impl Default for ReaderParameters { + fn default() -> Self { + Self { + use_record_position: false, + emit_delete: false, + sort_output: false, + allow_inflight_instants: false, + } + } +} diff --git a/crates/core/src/file_group/reader/record_context.rs b/crates/core/src/file_group/reader/record_context.rs new file mode 100644 index 00000000..2cff335e --- /dev/null +++ b/crates/core/src/file_group/reader/record_context.rs @@ -0,0 +1,787 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.engine.RecordContext`. +//! +//! In Java Hudi, `RecordContext` is an abstract class with engine-specific +//! implementations (Spark, Flink, Avro). It provides APIs for record operations +//! including key extraction, ordering value extraction, delete detection, +//! binary conversion (`toBinaryRow`), sealing (`seal`), and schema management. +//! +//! In hudi-rs, the engine is always Arrow, so `RecordContext` is a concrete +//! struct. Binary conversion uses Arrow IPC serialization. + +use crate::Result; +use crate::config::table::HudiTableConfig; +use crate::error::CoreError; +use crate::file_group::reader::buffered_record::{BufferedRecord, OrderingValue}; +use crate::file_group::reader::delete_context::DeleteContext; +use crate::metadata::meta_field::MetaField; +use arrow_array::{Array, RecordBatch, StringArray}; +use arrow_ipc::reader::StreamReader; +use arrow_ipc::writer::StreamWriter; +use arrow_schema::{DataType, SchemaRef}; +use std::collections::HashMap; +use std::io::Cursor; + +use super::buffer::row_extraction::slice_row; + +/// Record context for Arrow engine record operations. +/// +/// Mirrors Java's `org.apache.hudi.common.engine.RecordContext`. +/// +/// In Java, `RecordContext` is the engine-specific "glue" that lets the +/// engine-agnostic `FileGroupRecordBuffer` and `BufferedRecordMerger` +/// infrastructure manipulate engine records without knowing the type system. +/// +/// ## Instance methods (record-level operations needing config): +/// - `get_record_keys` — mirrors `RecordContext.getRecordKey(T, Schema)` +/// - `get_ordering_values` — mirrors `RecordContext.getOrderingValue(T, Schema, List)` +/// - `batch_to_buffered_records` — combines key + ordering extraction +/// - `delete_batch_to_buffered_records` — converts delete batches +/// - `is_delete_record` — mirrors `RecordContext.isDeleteRecord(T, DeleteContext)` +/// +/// ## Associated functions (engine-level, no config needed): +/// - `to_binary_row` — mirrors `RecordContext.toBinaryRow(Schema, T)` +/// - `seal` — mirrors `RecordContext.seal(T)` +/// - `from_binary` — Arrow-specific IPC deserialization +/// - `get_schema_from_buffer_record` — mirrors `RecordContext.getSchemaFromBufferRecord` +#[derive(Debug, Clone)] +pub struct RecordContext { + /// Which column contains the record key (e.g. `_hoodie_record_key` or a PK column). + /// Mirrors Java's `recordKeyExtractor` strategy. + pub record_key_field: String, + + /// Ordering (precombine) field names for conflict resolution. + /// Mirrors Java's `orderingFieldNames` used in `getOrderingValue()`. + pub ordering_field_names: Vec, + + /// Whether meta fields are populated (affects key extraction strategy). + /// When true, reads `_hoodie_record_key` directly. + /// When false, computes key from configured record key columns. + pub populate_meta_fields: bool, + + /// Partition path for constructing delete rows. + /// Mirrors Java's `RecordContext.partitionPath`. + pub partition_path: String, +} + +impl Default for RecordContext { + fn default() -> Self { + Self::new(&HashMap::new(), String::new()) + } +} + +impl RecordContext { + /// Create a new RecordContext from table configuration. + /// + /// Mirrors Java's `RecordContext(HoodieTableConfig tableConfig, JavaTypeConverter typeConverter)`. + /// + /// In Java, the constructor reads `tableConfig.populateMetaFields()` to choose + /// the key extraction strategy (metadata vs virtual keys), and later + /// `initOrderingValueConverter(schema, orderingFieldNames)` is called via + /// `setSchemaHandler()`. + /// + /// In hudi-rs, we derive all fields from the `table_config` map in one shot: + /// - `populate_meta_fields` from `hoodie.populate.meta.fields` (default: true) + /// - `record_key_field` from `_hoodie_record_key` (meta) or `hoodie.table.recordkey.fields` (virtual) + /// - `ordering_field_names` from `hoodie.table.precombine.field` or `hoodie.table.ordering.fields` + pub fn new(table_config: &HashMap, partition_path: String) -> Self { + let populate_meta_fields = table_config + .get(HudiTableConfig::PopulatesMetaFields.as_ref()) + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(true); + + let record_key_field = if populate_meta_fields { + MetaField::RecordKey.as_ref().to_string() + } else { + // Virtual keys mode: use the first record key field from config. + // Java: virtualKeyExtractor(tableConfig.getRecordKeyFields()) + table_config + .get(HudiTableConfig::RecordKeyFields.as_ref()) + .and_then(|fields| fields.split(',').next().map(|f| f.trim().to_string())) + .unwrap_or_else(|| MetaField::RecordKey.as_ref().to_string()) + }; + + let ordering_field_names = table_config + .get(HudiTableConfig::PrecombineField.as_ref()) + .or_else(|| table_config.get("hoodie.table.ordering.fields")) + .map(|f| vec![f.clone()]) + .unwrap_or_default(); + + Self { + record_key_field, + ordering_field_names, + populate_meta_fields, + partition_path, + } + } + + // ========================================================================= + // Instance methods — record-level operations (mirrors Java RecordContext) + // ========================================================================= + + /// Extract record key strings from a RecordBatch column. + /// + /// Mirrors Java's `RecordContext.getRecordKey(T record, Schema schema)`, + /// applied to each row in the batch. + /// + /// Uses `self.record_key_field` to locate the key column. + pub fn get_record_keys(&self, batch: &RecordBatch) -> Result> { + let col_idx = batch + .schema() + .index_of(&self.record_key_field) + .map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Key field '{}' not found in schema: {e}", + self.record_key_field + )) + })?; + + let key_array = batch + .column(col_idx) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + CoreError::ReadFileSliceError(format!( + "Key field '{}' is not a StringArray", + self.record_key_field + )) + })?; + + Ok((0..key_array.len()) + .map(|i| key_array.value(i).to_string()) + .collect()) + } + + /// Extract ordering values from the first ordering field column. + /// + /// Mirrors Java's `RecordContext.getOrderingValue(T, Schema, List)`, + /// applied to each row in the batch. + /// + /// Uses `self.ordering_field_names` to locate the ordering column. + /// Supports Int32, Int64, and Utf8 column types. + /// Returns None if no ordering fields specified or column not found. + pub fn get_ordering_values(&self, batch: &RecordBatch) -> Option>> { + let field_name = self.ordering_field_names.first()?; + let col_idx = batch.schema().index_of(field_name).ok()?; + let col = batch.column(col_idx); + + let values: Vec> = match col.data_type() { + DataType::Int64 => { + let arr = col.as_any().downcast_ref::()?; + (0..arr.len()) + .map(|i| { + if arr.is_null(i) { + None + } else { + Some(OrderingValue::Long(arr.value(i))) + } + }) + .collect() + } + DataType::Int32 => { + let arr = col.as_any().downcast_ref::()?; + (0..arr.len()) + .map(|i| { + if arr.is_null(i) { + None + } else { + Some(OrderingValue::Long(arr.value(i) as i64)) + } + }) + .collect() + } + DataType::Utf8 => { + let arr = col.as_any().downcast_ref::()?; + (0..arr.len()) + .map(|i| { + if arr.is_null(i) { + None + } else { + Some(OrderingValue::String(arr.value(i).to_string())) + } + }) + .collect() + } + _ => return None, + }; + Some(values) + } + + /// Convert a multi-row RecordBatch into a Vec of (key, BufferedRecord) pairs. + /// + /// Each row becomes an individual `BufferedRecord` with a single-row `RecordBatch` + /// as its data payload. Combines `get_record_keys` + `get_ordering_values` + + /// `is_delete_record` per row. + /// + /// Mirrors the Java pattern in `KeyBasedFileGroupRecordBuffer.processDataBlock()`: + /// ```text + /// for each record in dataBlock.getEngineRecordIterator(): + /// key = recordContext.getRecordKey(record, schema) + /// ordering = recordContext.getOrderingValue(record, schema, orderingFields) + /// isDelete = recordContext.isDeleteRecord(record, deleteContext) + /// buffered = BufferedRecords.fromEngineRecord(record, key, ordering, isDelete) + /// ``` + pub fn batch_to_buffered_records( + &self, + batch: &RecordBatch, + delete_context: Option<&DeleteContext>, + ) -> Result> { + if batch.num_rows() == 0 { + return Ok(Vec::new()); + } + + let keys = self.get_record_keys(batch)?; + let ordering_values = self.get_ordering_values(batch); + let mut records = Vec::with_capacity(batch.num_rows()); + + for (row_idx, key) in keys.into_iter().enumerate() { + let ordering_value = ordering_values + .as_ref() + .and_then(|vals| vals.get(row_idx).cloned()) + .flatten(); + // Mirrors Java: boolean isDelete = recordContext.isDeleteRecord(nextRecord, deleteContext) + let is_delete = self.is_delete_record(batch, row_idx, delete_context); + let record = if is_delete { + BufferedRecord::new_delete(key.clone(), ordering_value) + } else { + let row_batch = slice_row(batch, row_idx); + BufferedRecord::new_data(key.clone(), row_batch, ordering_value) + }; + records.push((key, record)); + } + + Ok(records) + } + + /// Convert a delete RecordBatch into a Vec of (key, BufferedRecord) pairs. + /// + /// Delete batches have schema: (recordKey, partitionPath, orderingVal). + /// + /// Mirrors the Java pattern: + /// ```text + /// for each deleteRecord in deleteBlock.getRecordsToDelete(): + /// ordering = recordContext.getOrderingValue(deleteRecord) + /// buffered = BufferedRecords.fromDeleteRecord(deleteRecord, ordering, ...) + /// ``` + pub fn delete_batch_to_buffered_records( + &self, + batch: &RecordBatch, + ) -> Result> { + if batch.num_rows() == 0 { + return Ok(Vec::new()); + } + + // Delete batch schema: recordKey (col 0), partitionPath (col 1), orderingVal (col 2) + let key_array = batch + .column(0) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + CoreError::ReadFileSliceError( + "Delete batch column 0 is not a StringArray".to_string(), + ) + })?; + + let mut records = Vec::with_capacity(batch.num_rows()); + for i in 0..key_array.len() { + let key = key_array.value(i).to_string(); + // TODO: Extract ordering value from column 2 + let record = BufferedRecord::new_delete(key.clone(), None); + records.push((key, record)); + } + + Ok(records) + } + + /// Check if a record is a delete record. + /// + /// Mirrors Java's `RecordContext.isDeleteRecord(T record, DeleteContext deleteContext)`. + /// + /// Checks (in order, matching Java): + /// 1. Built-in: `_hoodie_is_deleted` field is true + /// 2. Operation: `_hoodie_operation` field is DELETE or UPDATE_BEFORE + /// 3. Custom: user-configured delete marker field matches configured value + /// + /// When `delete_context` is `Some`, uses pre-computed field positions and + /// custom delete marker config from the context. When `None`, falls back + /// to inline schema lookups (no custom marker support). + pub fn is_delete_record( + &self, + batch: &RecordBatch, + row_idx: usize, + delete_context: Option<&DeleteContext>, + ) -> bool { + self.is_built_in_delete_record(batch, row_idx, delete_context) + || self.is_delete_hoodie_operation(batch, row_idx, delete_context) + || self.is_custom_delete_record(batch, row_idx, delete_context) + } + + /// Check 1: Built-in `_hoodie_is_deleted` field. + /// + /// Mirrors Java's `RecordContext.isBuiltInDeleteRecord(T, DeleteContext)`. + fn is_built_in_delete_record( + &self, + batch: &RecordBatch, + row_idx: usize, + delete_context: Option<&DeleteContext>, + ) -> bool { + // Fast path: if DeleteContext says the field doesn't exist, skip + if let Some(ctx) = delete_context { + if !ctx.has_built_in_delete_field { + return false; + } + } + + let schema = batch.schema(); + if let Some((idx, _)) = schema.column_with_name("_hoodie_is_deleted") { + if let Some(arr) = batch + .column(idx) + .as_any() + .downcast_ref::() + { + if !arr.is_null(row_idx) && arr.value(row_idx) { + return true; + } + } + } + false + } + + /// Check 2: `_hoodie_operation` field is DELETE or UPDATE_BEFORE. + /// + /// Mirrors Java's `RecordContext.isDeleteHoodieOperation(T, DeleteContext)`. + fn is_delete_hoodie_operation( + &self, + batch: &RecordBatch, + row_idx: usize, + delete_context: Option<&DeleteContext>, + ) -> bool { + // Use pre-computed position from DeleteContext if available + let col_idx = if let Some(ctx) = delete_context { + match ctx.hoodie_operation_pos { + Some(pos) => pos, + None => return false, // field not in schema + } + } else { + // Fallback: inline schema lookup + match batch.schema().column_with_name("_hoodie_operation") { + Some((idx, _)) => idx, + None => return false, + } + }; + + if let Some(arr) = batch.column(col_idx).as_any().downcast_ref::() { + if !arr.is_null(row_idx) { + let op = arr.value(row_idx); + if op == "DELETE" || op == "UPDATE_BEFORE" { + return true; + } + } + } + false + } + + /// Check 3: Custom delete marker field matches configured value. + /// + /// Mirrors Java's `RecordContext.isCustomDeleteRecord(T, DeleteContext)`. + /// Only applies when DeleteContext has a custom_delete_marker configured. + fn is_custom_delete_record( + &self, + batch: &RecordBatch, + row_idx: usize, + delete_context: Option<&DeleteContext>, + ) -> bool { + let (key_field, marker_value) = match delete_context { + Some(ctx) => match &ctx.custom_delete_marker { + Some((k, v)) => (k.as_str(), v.as_str()), + None => return false, + }, + None => return false, // No custom marker without DeleteContext + }; + + let schema = batch.schema(); + if let Some((idx, _)) = schema.column_with_name(key_field) { + let col = batch.column(idx); + // String column + if let Some(arr) = col.as_any().downcast_ref::() { + if !arr.is_null(row_idx) && arr.value(row_idx) == marker_value { + return true; + } + } + // Int32 column — compare stringified value against marker + if let Some(arr) = col.as_any().downcast_ref::() { + if !arr.is_null(row_idx) && arr.value(row_idx).to_string() == marker_value { + return true; + } + } + // Int64 column — compare stringified value against marker + if let Some(arr) = col.as_any().downcast_ref::() { + if !arr.is_null(row_idx) && arr.value(row_idx).to_string() == marker_value { + return true; + } + } + } + false + } + + // ========================================================================= + // Associated functions — engine-level operations (no config needed) + // ========================================================================= + + /// Get the schema from a buffered record's data. + /// + /// Mirrors Java's `RecordContext.getSchemaFromBufferRecord(BufferedRecord)`. + /// + /// In Java, this decodes the schema from the record's schema ID via a + /// schema registry. In Arrow, we extract the schema directly from the + /// RecordBatch. + pub fn get_schema_from_buffer_record(record: &BufferedRecord) -> Option { + record.data.as_ref().map(|batch| batch.schema()) + } + + /// Convert an engine record to binary format. + /// + /// Mirrors Java's `RecordContext.toBinaryRow(HoodieSchema, T)`. + /// + /// For Spark: converts `InternalRow` -> `UnsafeRow` (compact off-heap binary). + /// For Arrow: serializes `RecordBatch` -> Arrow IPC stream bytes. + pub fn to_binary_row(_schema: &SchemaRef, record: &RecordBatch) -> Vec { + let mut buf = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut buf, &record.schema()).expect("IPC writer creation"); + writer.write(record).expect("IPC write"); + writer.finish().expect("IPC finish"); + } + buf + } + + /// Seal the binary record to ensure data referenced in memory does not change. + /// + /// Mirrors Java's `RecordContext.seal(T)`. + /// + /// For Spark: calls `UnsafeRow.copy()` to make the row self-contained. + /// For Arrow: no-op -- IPC bytes are already self-contained (owned `Vec`). + pub fn seal(bytes: Vec) -> Vec { + bytes + } + + /// Deserialize binary IPC bytes back to a RecordBatch. + /// + /// This is the Rust-specific "unwrap" counterpart to `to_binary_row`. + /// In Java, unwrapping is implicit because `UnsafeRow` IS an `InternalRow`. + /// In Rust/Arrow, we must explicitly deserialize IPC bytes back to `RecordBatch`. + pub fn from_binary(bytes: &[u8]) -> Result { + let cursor = Cursor::new(bytes); + let mut reader = StreamReader::try_new(cursor, None).map_err(|e| { + CoreError::ReadFileSliceError(format!("IPC reader creation failed: {e}")) + })?; + reader + .next() + .ok_or_else(|| { + CoreError::ReadFileSliceError("IPC stream contained no batches".to_string()) + })? + .map_err(|e| CoreError::ReadFileSliceError(format!("IPC read failed: {e}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_group::reader::delete_context::DeleteContext; + use arrow_array::{BooleanArray, Int64Array, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use std::sync::Arc; + + fn make_test_batch() -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, false), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["k1"])), + Arc::new(Int64Array::from(vec![42])), + ], + ) + .unwrap() + } + + fn make_record_context() -> RecordContext { + let table_config = HashMap::from([ + ( + HudiTableConfig::PopulatesMetaFields.as_ref().to_string(), + "true".to_string(), + ), + ( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ), + ]); + RecordContext::new(&table_config, "partition/path".to_string()) + } + + fn make_keyed_batch(num_rows: usize) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("name", DataType::Utf8, true), + Field::new("ts", DataType::Int64, false), + ])); + let keys: Vec = (0..num_rows).map(|i| format!("key_{i}")).collect(); + let names: Vec> = (0..num_rows).map(|i| Some(format!("name_{i}"))).collect(); + let timestamps: Vec = (0..num_rows).map(|i| i as i64).collect(); + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(keys)), + Arc::new(StringArray::from(names)), + Arc::new(Int64Array::from(timestamps)), + ], + ) + .unwrap() + } + + // ========================================================================= + // Associated function tests (IPC roundtrip, schema extraction) + // ========================================================================= + + #[test] + fn test_to_binary_row_and_from_binary_roundtrip() { + let batch = make_test_batch(); + let schema = batch.schema(); + + let bytes = RecordContext::to_binary_row(&schema, &batch); + assert!(!bytes.is_empty()); + + let sealed = RecordContext::seal(bytes); + + let restored = RecordContext::from_binary(&sealed).unwrap(); + assert_eq!(restored.num_rows(), 1); + assert_eq!(restored.schema(), schema); + assert_eq!( + restored + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "k1" + ); + assert_eq!( + restored + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 42 + ); + } + + #[test] + fn test_get_schema_from_buffer_record() { + let batch = make_test_batch(); + let schema = batch.schema(); + let record = BufferedRecord::new_data("k1".to_string(), batch, None); + let extracted = RecordContext::get_schema_from_buffer_record(&record); + assert_eq!(extracted.unwrap(), schema); + } + + #[test] + fn test_get_schema_from_delete_record_returns_none() { + let record = BufferedRecord::new_delete("k1".to_string(), None); + let extracted = RecordContext::get_schema_from_buffer_record(&record); + assert!(extracted.is_none()); + } + + // ========================================================================= + // Instance method tests (key extraction, ordering, batch conversion) + // ========================================================================= + + #[test] + fn test_get_record_keys() { + let ctx = make_record_context(); + let batch = make_keyed_batch(3); + let keys = ctx.get_record_keys(&batch).unwrap(); + assert_eq!(keys, vec!["key_0", "key_1", "key_2"]); + } + + #[test] + fn test_get_record_keys_missing_field() { + // Virtual keys mode with a nonexistent field + let table_config = HashMap::from([ + ( + HudiTableConfig::PopulatesMetaFields.as_ref().to_string(), + "false".to_string(), + ), + ( + HudiTableConfig::RecordKeyFields.as_ref().to_string(), + "nonexistent".to_string(), + ), + ]); + let ctx = RecordContext::new(&table_config, String::new()); + let batch = make_keyed_batch(1); + assert!(ctx.get_record_keys(&batch).is_err()); + } + + #[test] + fn test_get_ordering_values_int64() { + let ctx = make_record_context(); + let batch = make_keyed_batch(3); + let values = ctx.get_ordering_values(&batch).unwrap(); + assert_eq!(values.len(), 3); + assert_eq!(values[0], Some(OrderingValue::Long(0))); + assert_eq!(values[1], Some(OrderingValue::Long(1))); + assert_eq!(values[2], Some(OrderingValue::Long(2))); + } + + #[test] + fn test_get_ordering_values_no_fields() { + // No precombine/ordering field → empty ordering_field_names + let ctx = RecordContext::new(&HashMap::new(), String::new()); + let batch = make_keyed_batch(3); + assert!(ctx.get_ordering_values(&batch).is_none()); + } + + #[test] + fn test_batch_to_buffered_records() { + let ctx = make_record_context(); + let batch = make_keyed_batch(3); + let records = ctx.batch_to_buffered_records(&batch, None).unwrap(); + assert_eq!(records.len(), 3); + assert_eq!(records[0].0, "key_0"); + assert_eq!(records[1].0, "key_1"); + assert_eq!(records[2].0, "key_2"); + assert!(!records[0].1.is_delete()); + assert!(!records[0].1.is_empty()); + // Check ordering values were extracted + assert_eq!(records[0].1.ordering_value, Some(OrderingValue::Long(0))); + } + + #[test] + fn test_batch_to_buffered_records_empty() { + let ctx = make_record_context(); + let batch = make_keyed_batch(0); + let records = ctx.batch_to_buffered_records(&batch, None).unwrap(); + assert!(records.is_empty()); + } + + #[test] + fn test_is_delete_record_hoodie_is_deleted() { + let ctx = RecordContext::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("_hoodie_is_deleted", DataType::Boolean, true), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])), + Arc::new(BooleanArray::from(vec![Some(true), Some(false)])), + ], + ) + .unwrap(); + + assert!(ctx.is_delete_record(&batch, 0, None)); + assert!(!ctx.is_delete_record(&batch, 1, None)); + } + + #[test] + fn test_is_delete_record_hoodie_operation() { + let ctx = RecordContext::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("_hoodie_operation", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["k1", "k2", "k3"])), + Arc::new(StringArray::from(vec![ + Some("DELETE"), + Some("UPDATE_BEFORE"), + Some("INSERT"), + ])), + ], + ) + .unwrap(); + + assert!(ctx.is_delete_record(&batch, 0, None)); + assert!(ctx.is_delete_record(&batch, 1, None)); + assert!(!ctx.is_delete_record(&batch, 2, None)); + } + + #[test] + fn test_is_delete_record_no_markers() { + let ctx = RecordContext::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("value", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["k1"])), + Arc::new(Int64Array::from(vec![42])), + ], + ) + .unwrap(); + + assert!(!ctx.is_delete_record(&batch, 0, None)); + } + + #[test] + fn test_is_custom_delete_record_int32_marker() { + let ctx = RecordContext::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("_hoodie_record_key", DataType::Utf8, false), + Field::new("counter", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])), + Arc::new(arrow_array::Int32Array::from(vec![3, 1])), + ], + ) + .unwrap(); + let delete_ctx = DeleteContext { + custom_delete_marker: Some(("counter".to_string(), "3".to_string())), + has_built_in_delete_field: false, + hoodie_operation_pos: None, + reader_schema: schema, + }; + // counter=3 matches marker → delete + assert!(ctx.is_delete_record(&batch, 0, Some(&delete_ctx))); + // counter=1 does not match → not delete + assert!(!ctx.is_delete_record(&batch, 1, Some(&delete_ctx))); + } + + #[test] + fn test_default_record_context() { + let ctx = RecordContext::default(); + assert_eq!(ctx.record_key_field, "_hoodie_record_key"); + assert!(ctx.ordering_field_names.is_empty()); + assert!(ctx.populate_meta_fields); + assert!(ctx.partition_path.is_empty()); + } +} diff --git a/crates/core/src/file_group/reader/record_merger.rs b/crates/core/src/file_group/reader/record_merger.rs new file mode 100644 index 00000000..a7baa909 --- /dev/null +++ b/crates/core/src/file_group/reader/record_merger.rs @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.BufferedRecordMerger` and +//! `org.apache.hudi.common.table.read.BufferedRecordMergerFactory`. +//! +//! The `BufferedRecordMerger` trait defines how records are merged during +//! log scanning (delta merge: log-vs-log) and at read time (final merge: +//! base-vs-log). + +use super::buffered_record::{BufferedRecord, DeleteRecord}; +use crate::Result; + +/// Trait for merging buffered records during the file group read pipeline. +/// +/// Mirrors Java's `BufferedRecordMerger` interface. +/// +/// Consumed by `KeyBasedFileGroupRecordBuffer`: +/// - `delta_merge` in `process_next_data_record` (log-vs-log within buffer) +/// - `delta_merge_delete` in `process_next_deleted_record` (delete-vs-existing) +/// - `final_merge` in `has_next_base_record` (base-vs-log at read time) +pub trait BufferedRecordMerger: Send + Sync + std::fmt::Debug { + /// Merge a new log record with an existing buffered record. + /// + /// Returns `Some(merged)` if the record should be kept, `None` if dropped. + /// + /// Called during log scanning when a new record for the same key arrives. + fn delta_merge( + &self, + new_record: &BufferedRecord, + existing_record: Option<&BufferedRecord>, + ) -> Result>; + + /// Merge a delete record with an existing buffered record. + /// + /// Returns `Some(delete)` if the delete wins, `None` if the existing record survives. + fn delta_merge_delete( + &self, + delete_record: &DeleteRecord, + existing_record: Option<&BufferedRecord>, + ) -> Result>; + + /// Merge a base file record with a log record (final merge at read time). + /// + /// The `older_record` is from the base file, `newer_record` is from the log buffer. + fn final_merge( + &self, + older_record: &BufferedRecord, + newer_record: &BufferedRecord, + ) -> Result; +} + +/// Event-time based record merger. +/// +/// Resolves conflicts by comparing ordering values: the record with the +/// higher ordering value wins. This is the default merge strategy for +/// `COMMIT_TIME_ORDERING` and `EVENT_TIME_ORDERING` modes. +/// +/// Created by `BufferedRecordMergerFactory` based on the merge mode. +#[derive(Debug)] +pub struct EventTimeRecordMerger; + +impl BufferedRecordMerger for EventTimeRecordMerger { + fn delta_merge( + &self, + new_record: &BufferedRecord, + existing_record: Option<&BufferedRecord>, + ) -> Result> { + match existing_record { + None => Ok(Some(new_record.clone())), + Some(existing) => { + // Compare ordering values: higher wins + match (&new_record.ordering_value, &existing.ordering_value) { + (Some(new_val), Some(existing_val)) if new_val >= existing_val => { + Ok(Some(new_record.clone())) + } + (Some(_), Some(_)) => Ok(Some(existing.clone())), + // If either has no ordering value, new record wins (latest-write-wins) + _ => Ok(Some(new_record.clone())), + } + } + } + } + + fn delta_merge_delete( + &self, + delete_record: &DeleteRecord, + existing_record: Option<&BufferedRecord>, + ) -> Result> { + match existing_record { + None => Ok(Some(delete_record.clone())), + Some(existing) => { + match (&delete_record.ordering_value, &existing.ordering_value) { + (Some(del_val), Some(existing_val)) if del_val >= existing_val => { + Ok(Some(delete_record.clone())) + } + (Some(_), Some(_)) => Ok(None), // existing record survives + _ => Ok(Some(delete_record.clone())), + } + } + } + } + + /// Mirrors Java's `shouldKeepNewerRecord(olderRecord, newerRecord)`. + /// + /// Compares ordering values: higher wins. If either has no ordering value, + /// the newer (log) record wins. + /// + /// TODO: Java has special handling for `isCommitTimeOrderingDelete()` where + /// DELETE records always win regardless of ordering value. Not yet implemented. + fn final_merge( + &self, + older_record: &BufferedRecord, + newer_record: &BufferedRecord, + ) -> Result { + match (&newer_record.ordering_value, &older_record.ordering_value) { + (Some(new_val), Some(old_val)) if new_val >= old_val => Ok(newer_record.clone()), + (Some(_), Some(_)) => Ok(older_record.clone()), + // If either has no ordering value, newer (log) record wins + _ => Ok(newer_record.clone()), + } + } +} + +/// Commit-time based record merger. +/// +/// Mirrors Java's `CommitTimeRecordMerger`. In this mode, the most recently +/// written record always wins — no ordering value comparison is performed. +/// +/// Semantics: +/// - `delta_merge`: new record always wins (unconditional overwrite) +/// - `delta_merge_delete`: delete always wins (unconditional) +/// - `final_merge`: log record always wins over base record +#[derive(Debug)] +pub struct CommitTimeRecordMerger; + +impl BufferedRecordMerger for CommitTimeRecordMerger { + fn delta_merge( + &self, + new_record: &BufferedRecord, + _existing_record: Option<&BufferedRecord>, + ) -> Result> { + // In commit time ordering, last writer always wins — no ordering comparison + Ok(Some(new_record.clone())) + } + + fn delta_merge_delete( + &self, + delete_record: &DeleteRecord, + _existing_record: Option<&BufferedRecord>, + ) -> Result> { + // Delete always wins in commit time ordering + Ok(Some(delete_record.clone())) + } + + fn final_merge( + &self, + _older_record: &BufferedRecord, + newer_record: &BufferedRecord, + ) -> Result { + // Log record always wins over base file record + Ok(newer_record.clone()) + } +} + +/// Factory for creating `BufferedRecordMerger` instances. +/// +/// Mirrors Java's `BufferedRecordMergerFactory`. +pub struct BufferedRecordMergerFactory; + +impl BufferedRecordMergerFactory { + /// Create a merger based on the merge mode. + /// + /// - `"COMMIT_TIME_ORDERING"` → `CommitTimeRecordMerger` (last writer wins) + /// - `"EVENT_TIME_ORDERING"` → `EventTimeRecordMerger` (ordering value comparison) + /// - Others → Error (CUSTOM merge mode not yet supported) + pub fn create(merge_mode: &str) -> Result> { + match merge_mode { + "COMMIT_TIME_ORDERING" => Ok(Box::new(CommitTimeRecordMerger)), + "EVENT_TIME_ORDERING" => Ok(Box::new(EventTimeRecordMerger)), + unsupported => Err(crate::error::CoreError::ReadFileSliceError(format!( + "Unsupported merge mode for record merger: '{unsupported}'" + ))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::file_group::reader::buffered_record::OrderingValue; + + fn make_data_record(key: &str, ordering: Option) -> BufferedRecord { + BufferedRecord { + record_key: key.to_string(), + data: None, + binary_data: None, + ordering_value: ordering.map(OrderingValue::Long), + is_delete: false, + } + } + + fn make_delete_record(key: &str, ordering: Option) -> DeleteRecord { + DeleteRecord { + record_key: key.to_string(), + partition_path: String::new(), + ordering_value: ordering.map(OrderingValue::Long), + } + } + + // ========================================================================= + // CommitTimeRecordMerger tests + // ========================================================================= + + /// Given: CommitTimeRecordMerger, new_record with lower ordering value than existing + /// When: delta_merge(new_record, Some(existing_record)) + /// Then: new_record is returned (ordering value is irrelevant in commit-time mode) + #[test] + fn test_commit_time_delta_merge_new_always_wins() { + let merger = CommitTimeRecordMerger; + let new_rec = make_data_record("k1", Some(0)); + let existing = make_data_record("k1", Some(100)); + + let result = merger.delta_merge(&new_rec, Some(&existing)).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap().record_key, "k1"); + } + + /// Given: CommitTimeRecordMerger, no existing record + /// When: delta_merge(new_record, None) + /// Then: new_record is returned + #[test] + fn test_commit_time_delta_merge_no_existing() { + let merger = CommitTimeRecordMerger; + let new_rec = make_data_record("k1", Some(1)); + + let result = merger.delta_merge(&new_rec, None).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap().record_key, "k1"); + } + + /// Given: CommitTimeRecordMerger, delete with lower ordering than existing data + /// When: delta_merge_delete(delete, Some(existing)) + /// Then: delete is returned (always wins in commit-time mode) + #[test] + fn test_commit_time_delta_merge_delete_always_wins() { + let merger = CommitTimeRecordMerger; + let delete = make_delete_record("k1", Some(0)); + let existing = make_data_record("k1", Some(100)); + + let result = merger.delta_merge_delete(&delete, Some(&existing)).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap().record_key, "k1"); + } + + /// Given: CommitTimeRecordMerger + /// When: final_merge(base_record, log_record) + /// Then: log_record is returned (log always wins over base) + #[test] + fn test_commit_time_final_merge_log_wins() { + let merger = CommitTimeRecordMerger; + let base = make_data_record("k1", Some(100)); + let log = make_data_record("k1", Some(1)); + + let result = merger.final_merge(&base, &log).unwrap(); + assert_eq!(result.record_key, "k1"); + // Log record wins (ordering=1), not base (ordering=100) + assert_eq!(result.ordering_value, Some(OrderingValue::Long(1))); + } + + // ========================================================================= + // EventTimeRecordMerger tests + // ========================================================================= + + /// Given: EventTimeRecordMerger, new record has higher ordering (ts=2) than existing (ts=1) + /// When: delta_merge(new, Some(existing)) + /// Then: new_record wins (higher ordering value) + #[test] + fn test_event_time_delta_merge_higher_ordering_wins() { + let merger = EventTimeRecordMerger; + let new_rec = make_data_record("k1", Some(2)); + let existing = make_data_record("k1", Some(1)); + + let result = merger.delta_merge(&new_rec, Some(&existing)).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap().ordering_value, Some(OrderingValue::Long(2))); + } + + /// Given: EventTimeRecordMerger, new record has lower ordering (ts=0) than existing (ts=1) + /// When: delta_merge(new, Some(existing)) + /// Then: existing_record survives (higher ordering value) + #[test] + fn test_event_time_delta_merge_lower_ordering_loses() { + let merger = EventTimeRecordMerger; + let new_rec = make_data_record("k1", Some(0)); + let existing = make_data_record("k1", Some(1)); + + let result = merger.delta_merge(&new_rec, Some(&existing)).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap().ordering_value, Some(OrderingValue::Long(1))); + } + + /// Given: EventTimeRecordMerger, equal ordering values + /// When: delta_merge(new(ts=1), Some(existing(ts=1))) + /// Then: new_record wins (>= comparison) + #[test] + fn test_event_time_delta_merge_equal_ordering_new_wins() { + let merger = EventTimeRecordMerger; + let new_rec = make_data_record("new", Some(1)); + let existing = make_data_record("existing", Some(1)); + + let result = merger.delta_merge(&new_rec, Some(&existing)).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap().record_key, "new"); + } + + /// Given: EventTimeRecordMerger + /// When: delta_merge_delete(delete(ts=2), Some(existing(ts=1))) + /// Then: delete wins (higher ordering) + /// When: delta_merge_delete(delete(ts=0), Some(existing(ts=1))) + /// Then: existing survives (None returned, lower ordering delete loses) + #[test] + fn test_event_time_delta_merge_delete_ordering_check() { + let merger = EventTimeRecordMerger; + + // Delete with higher ordering wins + let delete_high = make_delete_record("k1", Some(2)); + let existing = make_data_record("k1", Some(1)); + let result = merger + .delta_merge_delete(&delete_high, Some(&existing)) + .unwrap(); + assert!(result.is_some(), "delete(ts=2) should win over data(ts=1)"); + + // Delete with lower ordering loses + let delete_low = make_delete_record("k1", Some(0)); + let result = merger + .delta_merge_delete(&delete_low, Some(&existing)) + .unwrap(); + assert!(result.is_none(), "delete(ts=0) should lose to data(ts=1)"); + } + + /// Given: EventTimeRecordMerger, no existing record + /// When: delta_merge(new, None) + /// Then: new_record is returned + #[test] + fn test_event_time_delta_merge_no_existing() { + let merger = EventTimeRecordMerger; + let new_rec = make_data_record("k1", Some(5)); + + let result = merger.delta_merge(&new_rec, None).unwrap(); + assert!(result.is_some()); + } + + // ========================================================================= + // Factory tests + // ========================================================================= + + #[test] + fn test_factory_commit_time_ordering() { + let merger = BufferedRecordMergerFactory::create("COMMIT_TIME_ORDERING").unwrap(); + let new_rec = make_data_record("k", Some(0)); + let existing = make_data_record("k", Some(100)); + // CommitTime: new always wins regardless of ordering + let result = merger.delta_merge(&new_rec, Some(&existing)).unwrap(); + assert_eq!(result.unwrap().ordering_value, Some(OrderingValue::Long(0))); + } + + #[test] + fn test_factory_event_time_ordering() { + let merger = BufferedRecordMergerFactory::create("EVENT_TIME_ORDERING").unwrap(); + let new_rec = make_data_record("k", Some(0)); + let existing = make_data_record("k", Some(100)); + // EventTime: higher ordering wins → existing survives + let result = merger.delta_merge(&new_rec, Some(&existing)).unwrap(); + assert_eq!( + result.unwrap().ordering_value, + Some(OrderingValue::Long(100)) + ); + } +} diff --git a/crates/core/src/file_group/reader/schema_handler.rs b/crates/core/src/file_group/reader/schema_handler.rs new file mode 100644 index 00000000..d7c71624 --- /dev/null +++ b/crates/core/src/file_group/reader/schema_handler.rs @@ -0,0 +1,922 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler`. +//! +//! Manages the various schemas involved in reading a file group: +//! table schema, requested schema, required schema, and the schema +//! used for updates (log file merging). + +use crate::file_group::reader::delete_context::DeleteContext; +use crate::file_group::reader::output_converter::{OutputConverter, ProjectionConverter}; +use crate::schema::avro_schema_utils::are_schemas_projection_equivalent; +use arrow_schema::{Field, Schema, SchemaRef}; +use std::sync::Arc; + +/// The `_hoodie_commit_time` meta field name. +/// Mirrors Java's `HoodieRecord.COMMIT_TIME_METADATA_FIELD`. +const COMMIT_TIME_FIELD: &str = "_hoodie_commit_time"; + +/// Schema handler for file group reading. +/// +/// Manages the different schema views needed during the read pipeline: +/// - **table_schema**: The full schema of the Hudi table. +/// - **requested_schema**: The schema requested by the query (column projection). +/// - **required_schema**: The minimum schema required for reading (includes +/// merge keys and ordering fields even if not in requested_schema). +/// - **data_schema**: The schema used for reading base files. +/// - **schema_for_updates**: Schema for incoming log records (initially == required_schema). +/// - **delete_context**: Canonical delete detection context (single source of truth). +/// +/// In Java Hudi, this class also handles internal schema evolution. +/// That is not yet implemented in hudi-rs. +#[derive(Debug, Clone, Default)] +pub struct FileGroupReaderSchemaHandler { + /// The full table schema. + pub table_schema: Option, + + /// The schema requested by the query (column projection). + pub requested_schema: Option, + + /// The data schema (used for base file reading). + pub data_schema: Option, + + /// The required schema for merge operations. + /// Includes record key and ordering fields even if not requested. + pub required_schema: Option, + + /// Schema for incoming update records (log file merging). + /// Initially equals `required_schema`. May be set to a different schema + /// by `StreamingFileGroupRecordBufferLoader` for incoming records + /// without metadata fields. + /// + /// Mirrors Java's `FileGroupReaderSchemaHandler.schemaForUpdates`. + schema_for_updates: Option, + + /// Canonical delete context, created during `prepare_required_schema()`. + /// Single source of truth — downstream consumers (record buffer, etc.) + /// should use this instead of creating their own. + /// + /// Mirrors Java's `FileGroupReaderSchemaHandler.deleteContext`. + delete_context: Option, +} + +impl FileGroupReaderSchemaHandler { + pub fn new() -> Self { + Self::default() + } + + pub fn with_table_schema(mut self, schema: SchemaRef) -> Self { + self.table_schema = Some(schema); + self + } + + pub fn with_data_schema(mut self, schema: SchemaRef) -> Self { + self.data_schema = Some(schema); + self + } + + pub fn with_requested_schema(mut self, schema: SchemaRef) -> Self { + self.requested_schema = Some(schema); + self + } + + /// Get the canonical delete context. + /// + /// Mirrors Java's `FileGroupReaderSchemaHandler.getDeleteContext()`. + /// Returns `None` only if `prepare_required_schema()` has not been called. + pub fn delete_context(&self) -> Option<&DeleteContext> { + self.delete_context.as_ref() + } + + /// Get the schema for updates (log record merging). + /// + /// Mirrors Java's `FileGroupReaderSchemaHandler.getSchemaForUpdates()`. + pub fn schema_for_updates(&self) -> Option<&SchemaRef> { + self.schema_for_updates.as_ref() + } + + /// Set the schema for updates. + /// + /// Mirrors Java's `FileGroupReaderSchemaHandler.setSchemaForUpdates(Schema)`. + /// Used by streaming buffer loaders for incoming records without metadata. + pub fn set_schema_for_updates(&mut self, schema: SchemaRef) { + self.schema_for_updates = Some(schema); + } + + /// Generate the required schema for reading files. + /// + /// Mirrors Java's `FileGroupReaderSchemaHandler.generateRequiredSchema()` + /// (lines 187-229). + /// + /// For COW (no log files): returns `requested_schema` as-is, unless + /// `has_instant_range` is true — then adds `_hoodie_commit_time` for + /// base file filtering. + /// + /// For MOR (with log files): adds mandatory fields needed for merging + /// (record key, ordering fields, delete markers, commit time) to + /// `requested_schema`. + /// + /// # Arguments + /// * `has_log_files` — Whether the file group has log files (MOR gate). + /// * `record_key_fields` — All record key field names (single for meta-field + /// tables, potentially multiple for virtual-key composite keys). + /// * `ordering_field_names` — Precombine/ordering field names. + /// * `delete_context` — Delete detection context. + /// * `has_instant_range` — Whether an instant range filter is active. + /// * `merge_mode` — The merge mode string (e.g. "COMMIT_TIME_ORDERING"). + pub fn generate_required_schema( + &self, + has_log_files: bool, + record_key_fields: &[String], + ordering_field_names: &[String], + delete_context: &DeleteContext, + has_instant_range: bool, + merge_mode: &str, + ) -> Option { + // The base schema to start from: requested_schema, or data_schema as fallback. + let base_schema = self + .requested_schema + .as_ref() + .or(self.data_schema.as_ref())?; + + // The field source for looking up field definitions. + let field_source = self.table_schema.as_ref().or(self.data_schema.as_ref())?; + + // COW path: no log files. + // Mirrors Java lines 190-197: + // if (!readerContext.getHasLogFiles()) { + // if (hasInstantRange && !findNestedField(requestedSchema, COMMIT_TIME_METADATA_FIELD)) { + // addedFields.add(getField(tableSchema, COMMIT_TIME_METADATA_FIELD)); + // return appendFields(requestedSchema, addedFields); + // } + // return requestedSchema; + // } + if !has_log_files { + if has_instant_range && base_schema.column_with_name(COMMIT_TIME_FIELD).is_none() { + if let Some((_, field)) = field_source.column_with_name(COMMIT_TIME_FIELD) { + let mut fields: Vec> = base_schema.fields().to_vec(); + fields.push(Arc::new(field.clone())); + return Some(Arc::new(Schema::new(fields))); + } + } + return Some(base_schema.clone()); + } + + // CUSTOM merge mode is not supported in hudi-rs. + // Mirrors Java lines 209-213. + if merge_mode.eq_ignore_ascii_case("CUSTOM") { + panic!( + "CUSTOM merge mode is not supported in hudi-rs. \ + Use COMMIT_TIME_ORDERING or EVENT_TIME_ORDERING." + ); + } + + // MOR path: collect mandatory fields for merging. + // Mirrors Java's getMandatoryFieldsForMerging() (lines 231-278). + let mut mandatory_field_names: Vec<&str> = Vec::new(); + + // Add _hoodie_commit_time if instant range is active (Java lines 246-248). + if has_instant_range { + mandatory_field_names.push(COMMIT_TIME_FIELD); + } + + // Add record key fields (Java lines 251-258). + // Java checks populateMetaFields to decide between _hoodie_record_key + // and the configured record key fields. The caller already resolves + // this and passes the correct fields. + for key_field in record_key_fields { + mandatory_field_names.push(key_field.as_str()); + } + + // Add ordering/precombine fields only for EVENT_TIME_ORDERING + // (Java lines 260-263). + if merge_mode.eq_ignore_ascii_case("EVENT_TIME_ORDERING") { + for ordering_field in ordering_field_names { + mandatory_field_names.push(ordering_field.as_str()); + } + } + + // Add _hoodie_is_deleted if it exists in table schema (Java lines 265-267). + if delete_context.has_built_in_delete_field { + mandatory_field_names.push("_hoodie_is_deleted"); + } + + // Add custom delete marker field if configured (Java lines 269-271). + if let Some((key_field, _)) = &delete_context.custom_delete_marker { + mandatory_field_names.push(key_field.as_str()); + } + + // Add _hoodie_operation if it exists in table schema (Java lines 273-274). + if field_source.column_with_name("_hoodie_operation").is_some() { + mandatory_field_names.push("_hoodie_operation"); + } + + // Append only fields not already in the base schema (Java line 219). + let mut extra_fields: Vec> = Vec::new(); + for &field_name in &mandatory_field_names { + if base_schema.column_with_name(field_name).is_none() { + if let Some((_, field)) = field_source.column_with_name(field_name) { + // Deduplicate: only add if not already in extra_fields. + if !extra_fields.iter().any(|f| f.name() == field_name) { + extra_fields.push(Arc::new(field.clone())); + } + } + } + } + + if extra_fields.is_empty() { + Some(base_schema.clone()) + } else { + let mut fields: Vec> = base_schema.fields().to_vec(); + fields.extend(extra_fields); + Some(Arc::new(Schema::new(fields))) + } + } + + /// Prepare the required schema: generate it and store it. + /// + /// Mirrors Java's `prepareRequiredSchema()` (lines 280-288) and the + /// constructor body (lines 104-106) which creates `DeleteContext`, + /// computes `requiredSchema`, and sets `schemaForUpdates`. + /// + /// Creates a `DeleteContext` (stored as canonical source of truth), + /// calls `generate_required_schema()`, and sets `schema_for_updates`. + /// + /// # Arguments + /// * `has_log_files` — Whether the file group has log files. + /// * `record_key_fields` — All record key field names. + /// * `ordering_field_names` — Precombine/ordering field names. + /// * `props` — Table config properties. + /// * `has_instant_range` — Whether an instant range filter is active. + /// * `merge_mode` — The merge mode string. + pub fn prepare_required_schema( + &mut self, + has_log_files: bool, + record_key_fields: &[String], + ordering_field_names: &[String], + props: &std::collections::HashMap, + has_instant_range: bool, + merge_mode: &str, + ) { + // Create and store the canonical DeleteContext (Java line 104). + let delete_context = if let Some(table_schema) = &self.table_schema { + DeleteContext::new(props, table_schema) + } else { + DeleteContext::from_props(props) + }; + self.delete_context = Some(delete_context.clone()); + + // Compute required schema (Java line 105). + self.required_schema = self.generate_required_schema( + has_log_files, + record_key_fields, + ordering_field_names, + &delete_context, + has_instant_range, + merge_mode, + ); + + // Initialize schema_for_updates = required_schema (Java line 106). + self.schema_for_updates = self.required_schema.clone(); + } + + /// Get the output converter for projecting from required_schema to requested_schema. + /// + /// Mirrors Java's `getOutputConverter()` (lines 143-148): + /// Returns `Some(ProjectionConverter)` when required_schema has more fields + /// than requested_schema; `None` when they are equivalent or when schemas + /// are not set. + pub fn get_output_converter(&self) -> Option> { + let required = self.required_schema.as_ref()?; + let requested = self.requested_schema.as_ref()?; + + // If schemas are projection-equivalent (same fields in same order), + // no converter is needed. + if are_schemas_projection_equivalent(required, requested) { + return None; + } + + Some(Box::new(ProjectionConverter::new(requested))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::DataType; + use std::collections::HashMap; + + /// Helper to create a simple Arrow schema from field names (all Utf8). + fn make_schema(field_names: &[&str]) -> SchemaRef { + let fields: Vec = field_names + .iter() + .map(|name| Field::new(*name, DataType::Utf8, true)) + .collect(); + Arc::new(Schema::new(fields)) + } + + /// Helper to create a schema with mixed types for more realistic testing. + fn make_table_schema() -> SchemaRef { + let fields = vec![ + Field::new("_hoodie_commit_time", DataType::Utf8, true), + Field::new("_hoodie_commit_seqno", DataType::Utf8, true), + Field::new("_hoodie_record_key", DataType::Utf8, true), + Field::new("_hoodie_partition_path", DataType::Utf8, true), + Field::new("_hoodie_file_name", DataType::Utf8, true), + Field::new("begin_lat", DataType::Float64, true), + Field::new("begin_lon", DataType::Float64, true), + Field::new("rider", DataType::Utf8, true), + Field::new("tip_history", DataType::Utf8, true), + Field::new("timestamp", DataType::Int64, true), + Field::new("_hoodie_is_deleted", DataType::Boolean, true), + Field::new("_hoodie_operation", DataType::Utf8, true), + ]; + Arc::new(Schema::new(fields)) + } + + // ========================================================================= + // COW tests + // ========================================================================= + + /// COW (no log files) → required_schema == requested_schema. + #[test] + fn test_cow_required_schema_equals_requested() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "tip_history", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema.clone()); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + false, // no log files (COW) + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + false, // no instant range + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + assert_eq!(required, requested_schema); + } + + /// COW with instant range → adds _hoodie_commit_time if not in requested schema. + /// Mirrors Java lines 191-194. + #[test] + fn test_cow_with_instant_range_adds_commit_time() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema.clone()); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + false, // COW + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + true, // has instant range + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"_hoodie_commit_time"), + "COW with instant range should add _hoodie_commit_time" + ); + assert!(field_names.contains(&"begin_lat")); + assert!(field_names.contains(&"rider")); + } + + /// COW with instant range but commit_time already requested → no duplicate. + #[test] + fn test_cow_with_instant_range_no_duplicate_commit_time() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["_hoodie_commit_time", "begin_lat"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema.clone()); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + false, + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + true, + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + // Should be same as requested since commit_time is already present. + assert_eq!(required, requested_schema); + } + + // ========================================================================= + // MOR tests + // ========================================================================= + + /// MOR with subset projection → record key field appended. + #[test] + fn test_mor_adds_record_key_field() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "tip_history", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema.clone()); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, // has log files (MOR) + &["_hoodie_record_key".to_string()], + &[], // no ordering fields (COMMIT_TIME_ORDERING) + &delete_context, + false, + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"_hoodie_record_key"), + "required_schema should include record key field" + ); + assert!( + field_names.contains(&"begin_lat"), + "required_schema should include requested fields" + ); + assert!( + field_names.contains(&"_hoodie_is_deleted"), + "required_schema should include _hoodie_is_deleted when present in table schema" + ); + assert!( + field_names.contains(&"_hoodie_operation"), + "required_schema should include _hoodie_operation when present in table schema" + ); + } + + /// MOR with instant range → adds _hoodie_commit_time to mandatory fields. + /// Mirrors Java lines 246-248. + #[test] + fn test_mor_with_instant_range_adds_commit_time() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + true, // has instant range + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"_hoodie_commit_time"), + "MOR with instant range should add _hoodie_commit_time" + ); + assert!(field_names.contains(&"_hoodie_record_key")); + } + + /// COMMIT_TIME_ORDERING should NOT include ordering fields. + #[test] + fn test_mor_commit_time_ordering_excludes_ordering_fields() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &["timestamp".to_string()], + &delete_context, + false, + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + !field_names.contains(&"timestamp"), + "COMMIT_TIME_ORDERING should NOT include ordering field" + ); + assert!( + field_names.contains(&"_hoodie_record_key"), + "required_schema should still include record key field" + ); + } + + /// EVENT_TIME_ORDERING → ordering field appended. + #[test] + fn test_mor_adds_ordering_field() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "tip_history", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema.clone()); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &["timestamp".to_string()], + &delete_context, + false, + "EVENT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"timestamp"), + "required_schema should include ordering field" + ); + assert!( + field_names.contains(&"_hoodie_record_key"), + "required_schema should include record key field" + ); + } + + /// MOR with _hoodie_is_deleted in table schema → added to required. + #[test] + fn test_mor_adds_delete_marker_fields() { + let table_schema = make_table_schema(); // has _hoodie_is_deleted + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + false, + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!(field_names.contains(&"_hoodie_is_deleted")); + assert!(field_names.contains(&"_hoodie_operation")); + } + + /// MOR with custom delete marker config → custom field added. + #[test] + fn test_mor_adds_custom_delete_marker() { + let table_schema = + make_schema(&["_hoodie_record_key", "col_a", "col_b", "is_deleted_custom"]); + let requested_schema = make_schema(&["col_a"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema); + + let mut props = HashMap::new(); + props.insert( + "hoodie.datasource.write.payload.delete.field".to_string(), + "is_deleted_custom".to_string(), + ); + props.insert( + "hoodie.datasource.write.payload.delete.marker".to_string(), + "true".to_string(), + ); + let delete_context = DeleteContext::new(&props, &table_schema); + + let required = handler + .generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + false, + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"is_deleted_custom"), + "required_schema should include custom delete marker field" + ); + } + + /// When all mandatory fields already in requested → no extra fields. + #[test] + fn test_mor_all_mandatory_fields_already_present() { + let table_schema = make_table_schema(); + let requested_schema = table_schema.clone(); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema.clone()); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &["timestamp".to_string()], + &delete_context, + false, + "EVENT_TIME_ORDERING", + ) + .unwrap(); + + assert_eq!(required, requested_schema); + } + + /// CUSTOM merge mode should panic. + #[test] + #[should_panic(expected = "CUSTOM merge mode is not supported")] + fn test_custom_merge_mode_panics() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + handler.generate_required_schema( + true, + &["_hoodie_record_key".to_string()], + &[], + &delete_context, + false, + "CUSTOM", + ); + } + + // ========================================================================= + // Composite record key tests + // ========================================================================= + + /// MOR with composite record keys → all key fields added. + #[test] + fn test_mor_composite_record_keys() { + let table_schema = make_schema(&["pk1", "pk2", "col_a", "_hoodie_is_deleted"]); + let requested_schema = make_schema(&["col_a"]); + + let handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema.clone()) + .with_requested_schema(requested_schema); + + let delete_context = DeleteContext::new(&HashMap::new(), &table_schema); + let required = handler + .generate_required_schema( + true, + &["pk1".to_string(), "pk2".to_string()], // composite keys + &[], + &delete_context, + false, + "COMMIT_TIME_ORDERING", + ) + .unwrap(); + + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"pk1"), + "should include first key field" + ); + assert!( + field_names.contains(&"pk2"), + "should include second key field" + ); + assert!( + field_names.contains(&"col_a"), + "should include requested fields" + ); + } + + // ========================================================================= + // Output converter tests + // ========================================================================= + + #[test] + fn test_get_output_converter_none_when_equal() { + let schema = make_schema(&["col_a", "col_b"]); + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(schema.clone()) + .with_data_schema(schema.clone()) + .with_requested_schema(schema.clone()); + handler.required_schema = Some(schema); + + assert!(handler.get_output_converter().is_none()); + } + + #[test] + fn test_get_output_converter_some_when_different() { + let requested = make_schema(&["col_a"]); + let required = make_schema(&["col_a", "col_b"]); + let mut handler = FileGroupReaderSchemaHandler::new().with_requested_schema(requested); + handler.required_schema = Some(required); + + assert!(handler.get_output_converter().is_some()); + } + + // ========================================================================= + // prepare_required_schema end-to-end tests + // ========================================================================= + + #[test] + fn test_prepare_required_schema_event_time() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema) + .with_data_schema(make_table_schema()) + .with_requested_schema(requested_schema); + + handler.prepare_required_schema( + true, // has log files + &["_hoodie_record_key".to_string()], + &["timestamp".to_string()], + &HashMap::new(), + false, + "EVENT_TIME_ORDERING", + ); + + let required = handler.required_schema.as_ref().unwrap(); + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!(field_names.contains(&"begin_lat")); + assert!(field_names.contains(&"rider")); + assert!(field_names.contains(&"_hoodie_record_key")); + assert!(field_names.contains(&"timestamp")); + assert!(field_names.contains(&"_hoodie_is_deleted")); + + // schema_for_updates should equal required_schema + assert_eq!( + handler.schema_for_updates(), + handler.required_schema.as_ref() + ); + // delete_context should be stored + assert!(handler.delete_context().is_some()); + } + + #[test] + fn test_prepare_required_schema_commit_time() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema) + .with_data_schema(make_table_schema()) + .with_requested_schema(requested_schema); + + handler.prepare_required_schema( + true, + &["_hoodie_record_key".to_string()], + &["timestamp".to_string()], + &HashMap::new(), + false, + "COMMIT_TIME_ORDERING", + ); + + let required = handler.required_schema.as_ref().unwrap(); + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!(field_names.contains(&"begin_lat")); + assert!(field_names.contains(&"rider")); + assert!(field_names.contains(&"_hoodie_record_key")); + assert!( + !field_names.contains(&"timestamp"), + "COMMIT_TIME_ORDERING should NOT include ordering field" + ); + assert!(field_names.contains(&"_hoodie_is_deleted")); + } + + /// prepare_required_schema with instant range stores DeleteContext and adds commit time. + #[test] + fn test_prepare_required_schema_with_instant_range() { + let table_schema = make_table_schema(); + let requested_schema = make_schema(&["begin_lat", "rider"]); + + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema) + .with_data_schema(make_table_schema()) + .with_requested_schema(requested_schema); + + handler.prepare_required_schema( + true, // MOR + &["_hoodie_record_key".to_string()], + &[], + &HashMap::new(), + true, // has instant range + "COMMIT_TIME_ORDERING", + ); + + let required = handler.required_schema.as_ref().unwrap(); + let field_names: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + assert!( + field_names.contains(&"_hoodie_commit_time"), + "should add _hoodie_commit_time when instant range is active" + ); + + // DeleteContext should be stored + let dc = handler.delete_context().unwrap(); + assert!(dc.has_built_in_delete_field); + } +} diff --git a/crates/core/src/file_group/reader/update_processor.rs b/crates/core/src/file_group/reader/update_processor.rs new file mode 100644 index 00000000..13a59935 --- /dev/null +++ b/crates/core/src/file_group/reader/update_processor.rs @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors `org.apache.hudi.common.table.read.UpdateProcessor`. +//! +//! Strategy interface for processing record updates during the +//! base-file-vs-log merge phase. The default implementation simply +//! passes through the merged record; more advanced implementations +//! can track stats, invoke callbacks, etc. + +use super::buffered_record::BufferedRecord; +use super::read_stats::HoodieReadStats; +use crate::Result; + +/// Strategy for processing record updates during merge iteration. +/// +/// Mirrors Java's `UpdateProcessor` interface. +/// +/// Created by factory method based on merge mode and configuration. +/// Wrapped optionally in a `CallbackProcessor` for update callbacks. +pub trait UpdateProcessor: Send + Sync + std::fmt::Debug { + /// Process an update (base record merged with log record). + /// + /// Returns the record to emit, or `None` to skip. + fn process_update( + &self, + record_key: &str, + previous_record: Option<&BufferedRecord>, + merged_record: &BufferedRecord, + is_delete: bool, + ) -> Result>; +} + +/// Default update processor: passes through the merged record as-is. +/// +/// Corresponds to Java's `StandardUpdateProcessor`. +#[derive(Debug)] +pub struct StandardUpdateProcessor; + +impl UpdateProcessor for StandardUpdateProcessor { + fn process_update( + &self, + _record_key: &str, + _previous_record: Option<&BufferedRecord>, + merged_record: &BufferedRecord, + _is_delete: bool, + ) -> Result> { + Ok(Some(merged_record.clone())) + } +} + +/// Factory for creating `UpdateProcessor` instances. +pub fn create_update_processor( + _read_stats: &HoodieReadStats, + _emit_deletes: bool, +) -> Box { + Box::new(StandardUpdateProcessor) +} diff --git a/crates/core/src/file_group/reader.rs b/crates/core/src/file_group/reader_v1.rs similarity index 83% rename from crates/core/src/file_group/reader.rs rename to crates/core/src/file_group/reader_v1.rs index 1031a398..fb508c50 100644 --- a/crates/core/src/file_group/reader.rs +++ b/crates/core/src/file_group/reader_v1.rs @@ -26,9 +26,12 @@ use crate::error::CoreError::ReadFileSliceError; use crate::expr::filter::{Filter, SchemableFilter}; use crate::file_group::file_slice::FileSlice; use crate::file_group::log_file::scanner::{LogFileScanner, ScanResult}; -use crate::file_group::record_batches::RecordBatches; +use crate::file_group::reader::HoodieFileGroupReader; +use crate::file_group::reader::input_split::InputSplit; +use crate::file_group::reader::reader_context::ReaderContext; +use crate::file_group::reader::reader_parameters::ReaderParameters; +use crate::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; use crate::hfile::{HFileReader, HFileRecord}; -use crate::merge::record_merger::RecordMerger; use crate::metadata::merger::FilesPartitionMerger; use crate::metadata::meta_field::MetaField; use crate::metadata::table_record::FilesPartitionRecord; @@ -104,6 +107,16 @@ impl FileGroupReader { }) } + /// Returns a reference to the Hudi configuration. + pub fn hudi_configs(&self) -> &Arc { + &self.hudi_configs + } + + /// Returns a reference to the storage layer. + pub fn storage(&self) -> &Arc { + &self.storage + } + /// Reads the data from the base file at the given relative path. /// /// # Arguments @@ -115,12 +128,46 @@ impl FileGroupReader { &self, relative_path: &str, ) -> Result { + log::debug!("FileGroupReader: reading base file '{relative_path}' (non-streaming)"); let records: RecordBatch = self .storage .get_parquet_file_data(relative_path) .map_err(|e| ReadFileSliceError(format!("Failed to read path {relative_path}: {e:?}"))) .await?; + let populates_meta_fields: bool = self + .hudi_configs + .get_or_default(HudiTableConfig::PopulatesMetaFields) + .into(); + if populates_meta_fields { + let start_ts = self + .hudi_configs + .try_get(HudiReadConfig::FileGroupStartTimestamp) + .map(|v| -> String { v.into() }); + match start_ts { + Some(ref start) => { + let end_ts = self + .hudi_configs + .try_get(HudiReadConfig::FileGroupEndTimestamp) + .map(|v| -> String { v.into() }); + log::debug!( + "FileGroupReader: commit time filter for '{relative_path}': \ + _hoodie_commit_time > '{start}'{}", + end_ts + .as_deref() + .map(|e| format!(" AND _hoodie_commit_time <= '{e}'")) + .unwrap_or_default(), + ); + } + None => { + log::debug!( + "FileGroupReader: no commit time filter for '{relative_path}' \ + (snapshot/time-travel query)" + ); + } + } + } + apply_commit_time_filter(&self.hudi_configs, records) } @@ -129,15 +176,15 @@ impl FileGroupReader { .hudi_configs .get_or_default(HudiTableConfig::TimelineTimezone) .into(); - let start_timestamp = self + let start_instant = self .hudi_configs .try_get(HudiReadConfig::FileGroupStartTimestamp) .map(|v| -> String { v.into() }); - let end_timestamp = self + let end_instant = self .hudi_configs .try_get(HudiReadConfig::FileGroupEndTimestamp) .map(|v| -> String { v.into() }); - InstantRange::new(timezone, start_timestamp, end_timestamp, false, true) + InstantRange::new(timezone, start_instant, end_instant, false, true) } /// Reads the data from the given file slice. @@ -158,8 +205,14 @@ impl FileGroupReader { } else { vec![] }; - self.read_file_slice_from_paths(&base_file_path, log_file_paths) - .await + match base_file_path { + Some(path) => self.read_file_slice_from_paths(&path, log_file_paths).await, + None => { + // Log-only file slice: delegate directly to the Java-style reader + self.read_file_slice_from_paths_log_only(log_file_paths) + .await + } + } } /// Reads a file slice from a base file and a list of log files. @@ -190,39 +243,110 @@ impl FileGroupReader { let base_file_only = log_file_paths.is_empty() || use_read_optimized; if base_file_only { + if log_file_paths.is_empty() { + log::debug!( + "FileGroupReader: COW/base-only read for '{base_file_path}' (no log files)" + ); + } else { + log::debug!( + "FileGroupReader: read-optimized mode for '{base_file_path}' \ + (skipping {} log file(s))", + log_file_paths.len() + ); + } self.read_file_slice_by_base_file_path(base_file_path).await } else { - let instant_range = self.create_instant_range_for_log_file_scan(); - let scan_result = LogFileScanner::new(self.hudi_configs.clone(), self.storage.clone()) - .scan(log_file_paths, &instant_range) - .await?; + log::debug!( + "FileGroupReader: MOR merge (java-style) for '{base_file_path}' \ + with {} log file(s): [{}]", + log_file_paths.len(), + log_file_paths.join(", "), + ); + + // Delegate to the Java-style HoodieFileGroupReader for proper + // key-based merge (supports COMMIT_TIME_ORDERING, etc.) + let input_split = InputSplit::new( + Some(base_file_path.to_string()), + None, + log_file_paths, + String::new(), + ); + + let _latest_instant_time = self + .hudi_configs + .try_get(HudiReadConfig::FileGroupEndTimestamp) + .map(|v| -> String { v.into() }) + .unwrap_or_else(|| "99991231235959999".to_string()); + + let mut reader_context = ReaderContext::empty(); + // Populate table_config from hudi_configs so RecordContext can + // derive ordering fields and key extraction strategy. + reader_context.table_config = self.hudi_configs.as_options(); + reader_context.rebuild_record_context(String::new()); + reader_context.has_log_files = true; + + // Populate schema_handler from base file parquet metadata. + // Mirrors Java: readerContext.setSchemaHandler(new FileGroupReaderSchemaHandler( + // readerContext, dataSchema, requestedSchema, ...)) + if let Ok(table_schema) = self.storage.get_parquet_file_schema(base_file_path).await { + let table_schema: arrow_schema::SchemaRef = Arc::new(table_schema); + let mut schema_handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema); + // requested_schema = None → "all columns" (user projection wired in follow-up) + schema_handler.prepare_required_schema( + true, // has_log_files + &reader_context.record_key_fields(), + &reader_context.ordering_field_names().to_vec(), + &reader_context.table_config, + reader_context.instant_range.is_some(), + &reader_context.merge_mode, + ); + reader_context.schema_handler = schema_handler; + } - let log_batches = match scan_result { - ScanResult::RecordBatches(batches) => batches, - ScanResult::Empty => RecordBatches::new(), - ScanResult::HFileRecords(_) => { - return Err(CoreError::LogBlockError( - "Unexpected HFile records in regular table log file".to_string(), - )); - } - }; + let mut reader = HoodieFileGroupReader::new( + Arc::new(reader_context), + self.storage.clone(), + input_split, + ReaderParameters::default(), + None, + None, + ); - let base_batch = self - .read_file_slice_by_base_file_path(base_file_path) - .await?; - let schema = base_batch.schema(); - let num_data_batches = log_batches.num_data_batches() + 1; - let num_delete_batches = log_batches.num_delete_batches(); - let mut all_batches = - RecordBatches::new_with_capacity(num_data_batches, num_delete_batches); - all_batches.push_data_batch(base_batch); - all_batches.extend(log_batches); - - let merger = RecordMerger::new(schema.clone(), self.hudi_configs.clone()); - merger.merge_record_batches(all_batches) + reader.read().await } } + /// Read a log-only file slice (no base file) via the Java-style HoodieFileGroupReader. + async fn read_file_slice_from_paths_log_only( + &self, + log_file_paths: Vec, + ) -> Result { + log::debug!( + "FileGroupReader: log-only MOR merge with {} log file(s): [{}]", + log_file_paths.len(), + log_file_paths.join(", "), + ); + + let input_split = InputSplit::new(None, None, log_file_paths, String::new()); + + let mut reader_context = ReaderContext::empty(); + reader_context.table_config = self.hudi_configs.as_options(); + reader_context.rebuild_record_context(String::new()); + + let mut reader = HoodieFileGroupReader::new( + Arc::new(reader_context), + self.storage.clone(), + input_split, + ReaderParameters::default(), + None, + None, + ); + + reader.read().await + } + // ========================================================================= // Streaming Read APIs // ========================================================================= @@ -280,8 +404,19 @@ impl FileGroupReader { vec![] }; - self.read_file_slice_from_paths_stream(&base_file_path, log_file_paths, options) - .await + match base_file_path { + Some(path) => { + self.read_file_slice_from_paths_stream(&path, log_file_paths, options) + .await + } + None => { + // Log-only: fall back to the non-streaming collect-and-merge path + let batch = self + .read_file_slice_from_paths_log_only(log_file_paths) + .await?; + Ok(Box::pin(futures::stream::once(async { Ok(batch) }))) + } + } } /// Reads a file slice from paths as a stream of record batches. @@ -359,12 +494,81 @@ impl FileGroupReader { .get_or_default(HudiReadConfig::StreamBatchSize) .into(); let batch_size = options.batch_size.unwrap_or(default_batch_size); + log::debug!( + "FileGroupReader: reading '{}' | batch_size={batch_size}{}", + relative_path, + if options.batch_size.is_some() { + " (from ReadOptions)" + } else { + " (default)" + }, + ); let mut parquet_options = ParquetReadOptions::new().with_batch_size(batch_size); // Add projection pushdown using column names (converted to indices internally // by get_parquet_file_stream using the same schema the projection is applied to) if let Some(ref projection_names) = options.projection { + log::debug!( + "FileGroupReader: column projection for '{}': [{}]", + relative_path, + projection_names.join(", "), + ); parquet_options = parquet_options.with_projection(projection_names.clone()); + } else { + log::debug!( + "FileGroupReader: no column projection for '{}' — all columns will be read", + relative_path, + ); + } + + // Log commit time filter (evaluated once here; applied per batch inside the stream) + let populates_meta_fields: bool = self + .hudi_configs + .get_or_default(HudiTableConfig::PopulatesMetaFields) + .into(); + if populates_meta_fields { + let start_ts = self + .hudi_configs + .try_get(HudiReadConfig::FileGroupStartTimestamp) + .map(|v| -> String { v.into() }); + match start_ts { + Some(ref start) => { + let end_ts = self + .hudi_configs + .try_get(HudiReadConfig::FileGroupEndTimestamp) + .map(|v| -> String { v.into() }); + log::debug!( + "FileGroupReader: commit time filter for '{}': \ + _hoodie_commit_time > '{}'{}", + relative_path, + start, + end_ts + .as_deref() + .map(|e| format!(" AND _hoodie_commit_time <= '{e}'")) + .unwrap_or_default(), + ); + } + None => { + log::debug!( + "FileGroupReader: no commit time filter for '{}' \ + (snapshot/time-travel query)", + relative_path, + ); + } + } + } else { + log::debug!( + "FileGroupReader: commit time filter skipped for '{}' \ + (PopulatesMetaFields=false)", + relative_path, + ); + } + + if options.row_predicate.is_some() { + log::debug!( + "FileGroupReader: user row predicate active for '{}'", + relative_path, + ); } let hudi_configs = self.hudi_configs.clone(); @@ -442,7 +646,9 @@ impl FileGroupReader { file_slice: &FileSlice, keys: &[&str], ) -> Result> { - let base_file_path = file_slice.base_file_relative_path()?; + let base_file_path = file_slice.base_file_relative_path()?.ok_or_else(|| { + ReadFileSliceError("Metadata table file slice must have a base file".to_string()) + })?; let log_file_paths: Vec = if file_slice.has_log_file() { file_slice .log_files @@ -494,9 +700,10 @@ impl FileGroupReader { vec![] } else { let instant_range = self.create_instant_range_for_log_file_scan(); - let scan_result = LogFileScanner::new(self.hudi_configs.clone(), self.storage.clone()) - .scan(log_file_paths, &instant_range) - .await?; + let scan_result = + LogFileScanner::new(Arc::new(ReaderContext::empty()), self.storage.clone()) + .scan(log_file_paths, &instant_range) + .await?; match scan_result { ScanResult::HFileRecords(records) => records, diff --git a/crates/core/src/internal_schema/mod.rs b/crates/core/src/internal_schema/mod.rs new file mode 100644 index 00000000..81eaf6c8 --- /dev/null +++ b/crates/core/src/internal_schema/mod.rs @@ -0,0 +1,23 @@ +// crates/core/src/internal_schema/mod.rs +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors `org.apache.hudi.internal.schema.{Type, Types}`. +//! +//! Schema-evolution code (`InternalSchema`, `AvroInternalSchemaConverter`, +//! schema-change actions/visitors, `SchemaChangeUtils`) is intentionally +//! out of scope — see the keyFilterOpt design spec §2. + +pub mod type_; +pub mod types; + +pub use type_::{Type, TypeID}; diff --git a/crates/core/src/internal_schema/type_.rs b/crates/core/src/internal_schema/type_.rs new file mode 100644 index 00000000..326344c3 --- /dev/null +++ b/crates/core/src/internal_schema/type_.rs @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors `org.apache.hudi.internal.schema.Type`. + +use std::fmt::Debug; + +/// Type identifier — mirrors Java's `Type.TypeID` enum. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TypeID { + Record, + Array, + Map, + Fixed, + String, + Binary, + Int, + Long, + Float, + Double, + Date, + Boolean, + Time, + Timestamp, + Decimal, + UUID, + DecimalBytes, + DecimalFixed, + TimeMillis, + TimestampMillis, + LocalTimestampMillis, + LocalTimestampMicros, +} + +impl TypeID { + /// Mirrors Java `TypeID.getName()` — lowercase snake_case of the enum name. + pub fn get_name(&self) -> &'static str { + match self { + TypeID::Record => "record", + TypeID::Array => "array", + TypeID::Map => "map", + TypeID::Fixed => "fixed", + TypeID::String => "string", + TypeID::Binary => "binary", + TypeID::Int => "int", + TypeID::Long => "long", + TypeID::Float => "float", + TypeID::Double => "double", + TypeID::Date => "date", + TypeID::Boolean => "boolean", + TypeID::Time => "time", + TypeID::Timestamp => "timestamp", + TypeID::Decimal => "decimal", + TypeID::UUID => "uuid", + TypeID::DecimalBytes => "decimal_bytes", + TypeID::DecimalFixed => "decimal_fixed", + TypeID::TimeMillis => "time_millis", + TypeID::TimestampMillis => "timestamp_millis", + TypeID::LocalTimestampMillis => "local_timestamp_millis", + TypeID::LocalTimestampMicros => "local_timestamp_micros", + } + } + + /// Mirrors Java `Type.fromValue(String)` — looks up a TypeID by its + /// uppercase Java enum name (case-insensitive). Returns Err for unknown. + pub fn from_value(value: &str) -> Result { + match value.to_ascii_uppercase().as_str() { + "RECORD" => Ok(TypeID::Record), + "ARRAY" => Ok(TypeID::Array), + "MAP" => Ok(TypeID::Map), + "FIXED" => Ok(TypeID::Fixed), + "STRING" => Ok(TypeID::String), + "BINARY" => Ok(TypeID::Binary), + "INT" => Ok(TypeID::Int), + "LONG" => Ok(TypeID::Long), + "FLOAT" => Ok(TypeID::Float), + "DOUBLE" => Ok(TypeID::Double), + "DATE" => Ok(TypeID::Date), + "BOOLEAN" => Ok(TypeID::Boolean), + "TIME" => Ok(TypeID::Time), + "TIMESTAMP" => Ok(TypeID::Timestamp), + "DECIMAL" => Ok(TypeID::Decimal), + "UUID" => Ok(TypeID::UUID), + "DECIMAL_BYTES" => Ok(TypeID::DecimalBytes), + "DECIMAL_FIXED" => Ok(TypeID::DecimalFixed), + "TIME_MILLIS" => Ok(TypeID::TimeMillis), + "TIMESTAMP_MILLIS" => Ok(TypeID::TimestampMillis), + "LOCAL_TIMESTAMP_MILLIS" => Ok(TypeID::LocalTimestampMillis), + "LOCAL_TIMESTAMP_MICROS" => Ok(TypeID::LocalTimestampMicros), + _ => Err(format!("Invalid value of Type: {value}")), + } + } + + /// Test helper — list all variants for round-trip testing. + #[cfg(test)] + pub fn all() -> Vec { + vec![ + TypeID::Record, + TypeID::Array, + TypeID::Map, + TypeID::Fixed, + TypeID::String, + TypeID::Binary, + TypeID::Int, + TypeID::Long, + TypeID::Float, + TypeID::Double, + TypeID::Date, + TypeID::Boolean, + TypeID::Time, + TypeID::Timestamp, + TypeID::Decimal, + TypeID::UUID, + TypeID::DecimalBytes, + TypeID::DecimalFixed, + TypeID::TimeMillis, + TypeID::TimestampMillis, + TypeID::LocalTimestampMillis, + TypeID::LocalTimestampMicros, + ] + } +} + +/// Mirrors Java `Type` interface. All concrete types implement this trait. +/// +/// Java has `extends Serializable` — Rust trait objects can derive `Debug` +/// and the concrete impls in `types.rs` derive `Clone, PartialEq, Eq, Hash`. +pub trait Type: Debug + Send + Sync { + /// Mirrors Java `Type.typeId()`. + fn type_id(&self) -> TypeID; + + /// Mirrors Java default `Type.isNestedType()`. Default false; nested + /// types override. + fn is_nested_type(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn type_id_get_name_returns_lowercase() { + assert_eq!(TypeID::Int.get_name(), "int"); + assert_eq!(TypeID::TimestampMillis.get_name(), "timestamp_millis"); + assert_eq!( + TypeID::LocalTimestampMicros.get_name(), + "local_timestamp_micros" + ); + } + + #[test] + fn type_id_from_value_round_trips() { + for variant in TypeID::all() { + let s = variant.get_name(); + assert_eq!(TypeID::from_value(s).unwrap(), variant); + } + } + + #[test] + fn type_id_from_value_rejects_unknown() { + assert!(TypeID::from_value("nope").is_err()); + } +} diff --git a/crates/core/src/internal_schema/types.rs b/crates/core/src/internal_schema/types.rs new file mode 100644 index 00000000..3b0f602c --- /dev/null +++ b/crates/core/src/internal_schema/types.rs @@ -0,0 +1,474 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors `org.apache.hudi.internal.schema.Types`. + +use crate::internal_schema::{Type, TypeID}; +use std::fmt; + +// ========================================================================= +// Singleton primitive types — all map to a single `Self` instance via `get()`. +// ========================================================================= + +macro_rules! singleton_primitive { + ($name:ident, $id:ident, $disp:literal) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct $name; + + impl $name { + pub fn get() -> Self { + $name + } + } + + impl Type for $name { + fn type_id(&self) -> TypeID { + TypeID::$id + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str($disp) + } + } + }; +} + +singleton_primitive!(BooleanType, Boolean, "boolean"); +singleton_primitive!(IntType, Int, "int"); +singleton_primitive!(LongType, Long, "long"); +singleton_primitive!(FloatType, Float, "float"); +singleton_primitive!(DoubleType, Double, "double"); +singleton_primitive!(DateType, Date, "date"); +singleton_primitive!(TimeType, Time, "time"); +singleton_primitive!(TimestampType, Timestamp, "timestamp"); +singleton_primitive!(TimeMillisType, TimeMillis, "time_millis"); +singleton_primitive!(TimestampMillisType, TimestampMillis, "timestamp_millis"); +singleton_primitive!( + LocalTimestampMillisType, + LocalTimestampMillis, + "local_timestamp_millis" +); +singleton_primitive!( + LocalTimestampMicrosType, + LocalTimestampMicros, + "local_timestamp_micros" +); +singleton_primitive!(StringType, String, "string"); +singleton_primitive!(BinaryType, Binary, "binary"); +singleton_primitive!(UUIDType, UUID, "uuid"); + +// ========================================================================= +// Parameterized primitive types +// ========================================================================= + +/// Mirrors Java `Types.FixedType(int size)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FixedType { + size: u32, +} + +impl FixedType { + pub fn new(size: u32) -> Self { + Self { size } + } + pub fn size(&self) -> u32 { + self.size + } +} + +impl Type for FixedType { + fn type_id(&self) -> TypeID { + TypeID::Fixed + } +} + +impl fmt::Display for FixedType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "fixed({})", self.size) + } +} + +/// Mirrors Java `Types.DecimalType(int precision, int scale)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DecimalType { + precision: u8, + scale: u8, +} + +impl DecimalType { + pub fn new(precision: u8, scale: u8) -> Self { + Self { precision, scale } + } + pub fn precision(&self) -> u8 { + self.precision + } + pub fn scale(&self) -> u8 { + self.scale + } +} + +impl Type for DecimalType { + fn type_id(&self) -> TypeID { + TypeID::Decimal + } +} + +impl fmt::Display for DecimalType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "decimal({}, {})", self.precision, self.scale) + } +} + +// ========================================================================= +// Nested types +// ========================================================================= + +/// Mirrors Java `Types.Field`. +/// +/// Java fields: `int id`, `boolean isOptional`, `String name`, `Type type`, `String doc`. +/// Default Java constructor sets `doc = null`. +#[derive(Debug)] +pub struct Field { + id: i32, + is_optional: bool, + name: String, + field_type: Box, + doc: Option, +} + +impl Field { + pub fn new( + id: i32, + name: impl Into, + field_type: Box, + is_optional: bool, + ) -> Self { + Self { + id, + name: name.into(), + field_type, + is_optional, + doc: None, + } + } + + pub fn with_doc(mut self, doc: impl Into) -> Self { + self.doc = Some(doc.into()); + self + } + + pub fn id(&self) -> i32 { + self.id + } + pub fn name(&self) -> &str { + &self.name + } + pub fn field_type(&self) -> &dyn Type { + self.field_type.as_ref() + } + pub fn is_optional(&self) -> bool { + self.is_optional + } + pub fn doc(&self) -> Option<&str> { + self.doc.as_deref() + } +} + +impl PartialEq for Field { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.is_optional == other.is_optional + && self.name == other.name + && self.field_type.type_id() == other.field_type.type_id() + && self.doc == other.doc + } +} + +impl Eq for Field {} + +/// Mirrors Java `Types.RecordType`. +#[derive(Debug)] +pub struct RecordType { + fields: Vec, +} + +impl RecordType { + pub fn new(fields: Vec) -> Self { + Self { fields } + } + + pub fn fields(&self) -> &[Field] { + &self.fields + } + + /// Mirrors Java `RecordType.fieldType(String name)`. + pub fn field_type_by_name(&self, name: &str) -> Option<&dyn Type> { + self.fields + .iter() + .find(|f| f.name == name) + .map(|f| f.field_type()) + } + + /// Mirrors Java `RecordType.field(int id)` — find by field id. + pub fn field_by_id(&self, id: i32) -> Option<&Field> { + self.fields.iter().find(|f| f.id == id) + } + + /// Find a field by name (Rust convenience accessor). + pub fn field_by_name(&self, name: &str) -> Option<&Field> { + self.fields.iter().find(|f| f.name == name) + } +} + +impl Type for RecordType { + fn type_id(&self) -> TypeID { + TypeID::Record + } + fn is_nested_type(&self) -> bool { + true + } +} + +impl fmt::Display for RecordType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "record<{}>", self.fields.len()) + } +} + +/// Mirrors Java `Types.ArrayType`. +#[derive(Debug)] +pub struct ArrayType { + element_id: i32, + element_optional: bool, + element_type: Box, +} + +impl ArrayType { + pub fn new(element_id: i32, element_optional: bool, element_type: Box) -> Self { + Self { + element_id, + element_optional, + element_type, + } + } + + pub fn element_id(&self) -> i32 { + self.element_id + } + pub fn is_element_optional(&self) -> bool { + self.element_optional + } + pub fn element_type(&self) -> &dyn Type { + self.element_type.as_ref() + } +} + +impl Type for ArrayType { + fn type_id(&self) -> TypeID { + TypeID::Array + } + fn is_nested_type(&self) -> bool { + true + } +} + +impl fmt::Display for ArrayType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "array<{}>", self.element_type.type_id().get_name()) + } +} + +/// Mirrors Java `Types.MapType`. +#[derive(Debug)] +pub struct MapType { + key_id: i32, + value_id: i32, + value_optional: bool, + key_type: Box, + value_type: Box, +} + +impl MapType { + pub fn new( + key_id: i32, + value_id: i32, + value_optional: bool, + key_type: Box, + value_type: Box, + ) -> Self { + Self { + key_id, + value_id, + value_optional, + key_type, + value_type, + } + } + + pub fn key_id(&self) -> i32 { + self.key_id + } + pub fn value_id(&self) -> i32 { + self.value_id + } + pub fn is_value_optional(&self) -> bool { + self.value_optional + } + pub fn key_type(&self) -> &dyn Type { + self.key_type.as_ref() + } + pub fn value_type(&self) -> &dyn Type { + self.value_type.as_ref() + } +} + +impl Type for MapType { + fn type_id(&self) -> TypeID { + TypeID::Map + } + fn is_nested_type(&self) -> bool { + true + } +} + +impl fmt::Display for MapType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "map<{},{}>", + self.key_type.type_id().get_name(), + self.value_type.type_id().get_name() + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::internal_schema::TypeID; + + #[test] + fn primitive_type_ids() { + assert_eq!(BooleanType::get().type_id(), TypeID::Boolean); + assert_eq!(IntType::get().type_id(), TypeID::Int); + assert_eq!(LongType::get().type_id(), TypeID::Long); + assert_eq!(FloatType::get().type_id(), TypeID::Float); + assert_eq!(DoubleType::get().type_id(), TypeID::Double); + assert_eq!(DateType::get().type_id(), TypeID::Date); + assert_eq!(TimeType::get().type_id(), TypeID::Time); + assert_eq!(TimestampType::get().type_id(), TypeID::Timestamp); + assert_eq!(TimeMillisType::get().type_id(), TypeID::TimeMillis); + assert_eq!( + TimestampMillisType::get().type_id(), + TypeID::TimestampMillis + ); + assert_eq!( + LocalTimestampMillisType::get().type_id(), + TypeID::LocalTimestampMillis + ); + assert_eq!( + LocalTimestampMicrosType::get().type_id(), + TypeID::LocalTimestampMicros + ); + assert_eq!(StringType::get().type_id(), TypeID::String); + assert_eq!(BinaryType::get().type_id(), TypeID::Binary); + assert_eq!(UUIDType::get().type_id(), TypeID::UUID); + } + + #[test] + fn primitive_type_to_string() { + assert_eq!(BooleanType::get().to_string(), "boolean"); + assert_eq!(IntType::get().to_string(), "int"); + assert_eq!(StringType::get().to_string(), "string"); + } + + #[test] + fn primitive_types_are_not_nested() { + assert!(!BooleanType::get().is_nested_type()); + assert!(!IntType::get().is_nested_type()); + assert!(!StringType::get().is_nested_type()); + } + + #[test] + fn primitive_types_equal_their_singletons() { + assert_eq!(IntType::get(), IntType::get()); + // Different types are never equal even if both are PrimitiveType. + // (PrimitiveType equality in Java compares typeId only.) + } + + #[test] + fn fixed_type_carries_size() { + let f = FixedType::new(16); + assert_eq!(f.type_id(), TypeID::Fixed); + assert_eq!(f.size(), 16); + assert_eq!(f.to_string(), "fixed(16)"); + } + + #[test] + fn decimal_type_carries_precision_and_scale() { + let d = DecimalType::new(10, 2); + assert_eq!(d.type_id(), TypeID::Decimal); + assert_eq!(d.precision(), 10); + assert_eq!(d.scale(), 2); + assert_eq!(d.to_string(), "decimal(10, 2)"); + } + + #[test] + fn field_carries_id_name_type_nullable() { + let f = Field::new(1, "id", Box::new(IntType::get()), false); + assert_eq!(f.id(), 1); + assert_eq!(f.name(), "id"); + assert!(!f.is_optional()); + assert_eq!(f.field_type().type_id(), TypeID::Int); + } + + #[test] + fn record_type_lookup_field_by_name() { + let fields = vec![ + Field::new(1, "id", Box::new(IntType::get()), false), + Field::new(2, "name", Box::new(StringType::get()), true), + ]; + let rec = RecordType::new(fields); + assert_eq!(rec.type_id(), TypeID::Record); + assert!(rec.is_nested_type()); + assert_eq!(rec.field_by_name("id").unwrap().id(), 1); + assert_eq!(rec.field_by_name("name").unwrap().id(), 2); + assert!(rec.field_by_name("missing").is_none()); + } + + #[test] + fn array_type_carries_element_type() { + let arr = ArrayType::new(2, true, Box::new(IntType::get())); + assert_eq!(arr.type_id(), TypeID::Array); + assert!(arr.is_nested_type()); + assert!(arr.is_element_optional()); + assert_eq!(arr.element_type().type_id(), TypeID::Int); + } + + #[test] + fn map_type_carries_key_value_types() { + let m = MapType::new( + 3, + 4, + true, + Box::new(StringType::get()), + Box::new(LongType::get()), + ); + assert_eq!(m.type_id(), TypeID::Map); + assert!(m.is_nested_type()); + assert_eq!(m.key_type().type_id(), TypeID::String); + assert_eq!(m.value_type().type_id(), TypeID::Long); + assert!(m.is_value_optional()); + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index fbbe9d98..320fcf69 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -46,12 +46,14 @@ //! } //! ``` -mod avro_to_arrow; +pub mod avro_to_arrow; pub mod config; pub mod error; pub mod expr; +pub mod expression; // ← NEW pub mod file_group; pub mod hfile; +pub mod internal_schema; // ← NEW pub mod keygen; pub mod merge; pub mod metadata; diff --git a/crates/core/src/metadata/merger.rs b/crates/core/src/metadata/merger.rs index ff1ba812..2c85e35e 100644 --- a/crates/core/src/metadata/merger.rs +++ b/crates/core/src/metadata/merger.rs @@ -451,8 +451,10 @@ mod tests { // Set up scanner let dir_url = parse_uri(metadata_dir.to_str().unwrap())?; let storage = Storage::new_with_base_url(dir_url)?; - let hudi_configs = Arc::new(HudiConfigs::empty()); - let scanner = LogFileScanner::new(hudi_configs, Arc::clone(&storage)); + let scanner = LogFileScanner::new( + Arc::new(crate::file_group::reader::reader_context::ReaderContext::empty()), + Arc::clone(&storage), + ); let instant_range = InstantRange::up_to("99991231235959999", "utc"); // Scan all log files diff --git a/crates/core/src/schema/avro_schema_utils.rs b/crates/core/src/schema/avro_schema_utils.rs new file mode 100644 index 00000000..5a19747d --- /dev/null +++ b/crates/core/src/schema/avro_schema_utils.rs @@ -0,0 +1,429 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Mirrors Java's `org.apache.hudi.avro.AvroSchemaUtils`. +//! +//! Provides schema comparison utilities for checking projection equivalence +//! between Arrow schemas. + +use arrow_schema::{DataType, Fields, SchemaRef}; + +/// Check if two schemas are projection-equivalent. +/// +/// Mirrors Java's `AvroSchemaUtils.areSchemasProjectionEquivalent()` which +/// delegates to `AvroSchemaComparatorForRecordProjection`. +/// +/// The comparison recurses through nested types (Struct, List, Map, etc.): +/// - **Record/Struct fields**: matched pairwise by case-insensitive name, then +/// types are compared recursively. +/// - **List / LargeList**: element types compared recursively. +/// - **Map**: entry struct (key + value) compared recursively. +/// - **All other types** (primitives, FixedSizeBinary, Decimal, Dictionary, +/// etc.): standard [`DataType`] equality which already checks parameters +/// such as precision, scale, and fixed-size width. +/// +/// Nullable flag on fields is intentionally ignored — Java unwraps nullable +/// unions before comparing, which produces the same effect. +pub fn are_schemas_projection_equivalent(a: &SchemaRef, b: &SchemaRef) -> bool { + record_fields_equivalent(a.fields(), b.fields()) +} + +/// Compare two sets of record fields for projection equivalence. +/// +/// Fields are compared pairwise in order. Two fields match when their names +/// are equal (case-insensitive, mirroring Java's +/// `AvroSchemaComparatorForRecordProjection.validateField`) and their data +/// types are recursively projection-equivalent. +fn record_fields_equivalent(a: &Fields, b: &Fields) -> bool { + if a.len() != b.len() { + return false; + } + a.iter().zip(b.iter()).all(|(fa, fb)| { + fa.name().eq_ignore_ascii_case(fb.name()) + && types_equivalent(fa.data_type(), fb.data_type()) + }) +} + +/// Recursively compare two Arrow [`DataType`]s for projection equivalence. +fn types_equivalent(a: &DataType, b: &DataType) -> bool { + match (a, b) { + // Struct (record): recurse into fields with case-insensitive name check. + (DataType::Struct(fa), DataType::Struct(fb)) => record_fields_equivalent(fa, fb), + // List / LargeList: recurse on element type. + (DataType::List(ea), DataType::List(eb)) + | (DataType::LargeList(ea), DataType::LargeList(eb)) => { + types_equivalent(ea.data_type(), eb.data_type()) + } + // Map: compare keys_sorted flag and recurse on the entry struct + // (which contains key and value fields). + (DataType::Map(ea, sa), DataType::Map(eb, sb)) => { + sa == sb && types_equivalent(ea.data_type(), eb.data_type()) + } + // All other types (primitives, FixedSizeBinary, Decimal, Dictionary, etc.): + // standard DataType equality already compares all parameters. + _ => a == b, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::{DataType, Field, Schema}; + use std::sync::Arc; + + /// Helper: build an Arrow schema from `(name, DataType)` pairs. + fn make_schema(fields: &[(&str, DataType)]) -> SchemaRef { + Arc::new(Schema::new( + fields + .iter() + .map(|(name, dt)| Field::new(*name, dt.clone(), true)) + .collect::>(), + )) + } + + /// Helper: build a simple schema where every field is Utf8. + fn make_simple_schema(names: &[&str]) -> SchemaRef { + Arc::new(Schema::new( + names + .iter() + .map(|n| Field::new(*n, DataType::Utf8, true)) + .collect::>(), + )) + } + + // ========================================================================= + // Ported from Java TestAvroSchemaUtils.java — record-level tests + // ========================================================================= + + /// Java: testAreSchemasProjectionEquivalentRecordSchemas + /// Two record schemas with the same field name are projection-equivalent + /// regardless of schema/record name (Arrow schemas have no "record name"). + #[test] + fn test_are_schemas_projection_equivalent_record_schemas() { + let s1 = make_schema(&[("f1", DataType::Int32)]); + let s2 = make_schema(&[("f1", DataType::Int32)]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentDifferentFieldCountInRecords + #[test] + fn test_are_schemas_projection_equivalent_different_field_count_in_records() { + let s1 = make_schema(&[("a", DataType::Int32)]); + let s2: SchemaRef = Arc::new(Schema::empty()); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentNestedRecordSchemas + /// Nested struct fields — top-level field names match so these are equivalent. + #[test] + fn test_are_schemas_projection_equivalent_nested_record_schemas() { + let inner1 = DataType::Struct(vec![Field::new("x", DataType::Utf8, true)].into()); + let inner2 = DataType::Struct(vec![Field::new("x", DataType::Utf8, true)].into()); + let s1 = make_schema(&[("inner", inner1)]); + let s2 = make_schema(&[("inner", inner2)]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentArraySchemas + /// Schemas with identically-named array fields. + #[test] + fn test_are_schemas_projection_equivalent_array_schemas() { + let s1 = make_schema(&[( + "arr", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + )]); + let s2 = make_schema(&[( + "arr", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + )]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentDifferentElementTypeInArray + /// Same field name but different list element types → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_different_element_type_in_array() { + let s1 = make_schema(&[( + "arr", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + )]); + let s2 = make_schema(&[( + "arr", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + )]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentMapSchemas + #[test] + fn test_are_schemas_projection_equivalent_map_schemas() { + let s1 = make_schema(&[( + "m", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + )]); + let s2 = make_schema(&[( + "m", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + )]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentDifferentMapValueTypes + /// Same field name but different map value types → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_different_map_value_types() { + let s1 = make_schema(&[( + "m", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + )]); + let s2 = make_schema(&[( + "m", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), + ] + .into(), + ), + false, + )), + false, + ), + )]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentNullableSchemaComparison + /// One field nullable, the other not — same field name and type → equivalent. + /// Java unwraps nullable unions before comparing; Arrow nullable is a field + /// property that we intentionally ignore, producing the same result. + #[test] + fn test_are_schemas_projection_equivalent_nullable_schema_comparison() { + let s1 = make_schema(&[("f", DataType::Int32)]); + // In Arrow, nullable is a field property, not a union wrapper. + let s2 = Arc::new(Schema::new(vec![Field::new("f", DataType::Int32, false)])); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentListVsString + /// Same field name but List type vs String type → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_list_vs_string() { + let s1 = make_schema(&[( + "f", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + )]); + let s2 = make_schema(&[("f", DataType::Utf8)]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + assert!(!are_schemas_projection_equivalent(&s2, &s1)); + } + + /// Java: testAreSchemasProjectionEquivalentMapVsString + /// Same field name but Map type vs String type → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_map_vs_string() { + let s1 = make_schema(&[( + "f", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), + ] + .into(), + ), + false, + )), + false, + ), + )]); + let s2 = make_schema(&[("f", DataType::Utf8)]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + assert!(!are_schemas_projection_equivalent(&s2, &s1)); + } + + /// Java: testAreSchemasProjectionEquivalentEqualFixedSchemas + #[test] + fn test_are_schemas_projection_equivalent_equal_fixed_schemas() { + let s1 = make_schema(&[("f", DataType::FixedSizeBinary(16))]); + let s2 = make_schema(&[("f", DataType::FixedSizeBinary(16))]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentDifferentFixedSize + /// Same field name but different FixedSizeBinary sizes → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_different_fixed_size() { + let s1 = make_schema(&[("f", DataType::FixedSizeBinary(8))]); + let s2 = make_schema(&[("f", DataType::FixedSizeBinary(4))]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentEnums + /// Arrow uses Dictionary encoding as the closest analog to Avro enums. + #[test] + fn test_are_schemas_projection_equivalent_enums() { + let dict_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let s1 = make_schema(&[("e", dict_type.clone())]); + let s2 = make_schema(&[("e", dict_type)]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentDifferentEnumSymbols + /// Different Dictionary index types model incompatible enum encodings → not + /// equivalent. + #[test] + fn test_are_schemas_projection_equivalent_different_enum_symbols() { + let s1 = make_schema(&[( + "e", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + )]); + let s2 = make_schema(&[( + "e", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + )]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentEnumSymbolSubset + /// Avro allows the first enum to be a prefix-subset of the second's symbols. + /// Arrow Dictionary types do not carry symbol lists, so two identical + /// Dictionary types are always equivalent in both directions. + #[test] + fn test_are_schemas_projection_equivalent_enum_symbol_subset() { + let dict_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let s1 = make_schema(&[("e", dict_type.clone())]); + let s2 = make_schema(&[("e", dict_type)]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + assert!(are_schemas_projection_equivalent(&s2, &s1)); + } + + /// Java: testAreSchemasProjectionEquivalentEqualDecimalLogicalTypes + #[test] + fn test_are_schemas_projection_equivalent_equal_decimal_logical_types() { + let s1 = make_schema(&[("d", DataType::Decimal128(12, 2))]); + let s2 = make_schema(&[("d", DataType::Decimal128(12, 2))]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentDifferentPrecision + /// Same field name but different Decimal precision → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_different_precision() { + let s1 = make_schema(&[("d", DataType::Decimal128(12, 2))]); + let s2 = make_schema(&[("d", DataType::Decimal128(13, 2))]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentLogicalVsNoLogicalType + /// Decimal field vs plain Binary field — same field name but different types + /// → not equivalent. + #[test] + fn test_are_schemas_projection_equivalent_logical_vs_no_logical_type() { + let s1 = make_schema(&[("d", DataType::Decimal128(10, 2))]); + let s2 = make_schema(&[("d", DataType::Binary)]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + /// Java: testAreSchemasProjectionEquivalentSameReferenceSchema + #[test] + fn test_are_schemas_projection_equivalent_same_reference_schema() { + let s = make_simple_schema(&["f"]); + assert!(are_schemas_projection_equivalent(&s, &s)); + } + + /// Java: testAreSchemasProjectionEquivalentNullSchemaComparison + /// Rust uses references so null is not applicable. We test empty schemas + /// and schemas with different names instead to cover the boundary case. + #[test] + fn test_are_schemas_projection_equivalent_empty_schemas() { + let s1: SchemaRef = Arc::new(Schema::empty()); + let s2: SchemaRef = Arc::new(Schema::empty()); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + // ========================================================================= + // Additional edge-case tests (no direct Java equivalent) + // ========================================================================= + + #[test] + fn test_are_schemas_projection_equivalent_different_field_names() { + let s1 = make_simple_schema(&["a"]); + let s2 = make_simple_schema(&["b"]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } + + #[test] + fn test_are_schemas_projection_equivalent_case_insensitive_field_names() { + let s1 = make_schema(&[("Field_A", DataType::Int32)]); + let s2 = make_schema(&[("field_a", DataType::Int32)]); + assert!(are_schemas_projection_equivalent(&s1, &s2)); + } + + #[test] + fn test_are_schemas_projection_equivalent_field_order_matters() { + let s1 = make_simple_schema(&["a", "b"]); + let s2 = make_simple_schema(&["b", "a"]); + assert!(!are_schemas_projection_equivalent(&s1, &s2)); + } +} diff --git a/crates/core/src/schema/delete.rs b/crates/core/src/schema/delete.rs index 351e9dcb..5eacdafa 100644 --- a/crates/core/src/schema/delete.rs +++ b/crates/core/src/schema/delete.rs @@ -50,7 +50,9 @@ pub fn avro_schema_for_delete_record(delete_record_value: &AvroValue) -> Result< } }; - // Extract the ordering value type position from the union (always at field position 2) + // Extract the ordering value type position from the union (always at field position 2). + // The ordering value is a union of [null, BooleanWrapper, IntWrapper, LongWrapper, ...]. + // Each wrapper is a record with a single "value" field. let ordering_val = &fields[2].1; let ordering_val_type_pos = match ordering_val { AvroValue::Union(type_pos, _) => { @@ -213,27 +215,47 @@ mod tests { assert_eq!(union.variants()[0], AvroSchema::Null); assert_eq!(union.variants()[1], minimized_type); } else { + // The orderingVal union contains wrapper record types: + // [null, BooleanWrapper, IntWrapper, LongWrapper, FloatWrapper, + // DoubleWrapper, BytesWrapper, StringWrapper, DateWrapper, + // DecimalWrapper, TimeMicrosWrapper, TimestampMicrosWrapper, ArrayWrapper] assert_eq!(union.variants().len(), 13); assert_eq!(union.variants()[0], AvroSchema::Null); - assert_eq!(union.variants()[1], AvroSchema::Int); - assert_eq!(union.variants()[2], AvroSchema::Long); - assert_eq!(union.variants()[3], AvroSchema::Float); - assert_eq!(union.variants()[4], AvroSchema::Double); - assert_eq!(union.variants()[5], AvroSchema::Bytes); - assert_eq!(union.variants()[6], AvroSchema::String); - assert_eq!( - union.variants()[7], - AvroSchema::Decimal(DecimalSchema { - precision: 30, - scale: 15, - inner: Box::new(AvroSchema::Bytes) - }) - ); - assert_eq!(union.variants()[8], AvroSchema::Date); - assert_eq!(union.variants()[9], AvroSchema::TimeMillis); - assert_eq!(union.variants()[10], AvroSchema::TimeMicros); - assert_eq!(union.variants()[11], AvroSchema::TimestampMillis); - assert_eq!(union.variants()[12], AvroSchema::TimestampMicros); + // Remaining variants are wrapper records — just check they are records + let wrapper_names = [ + "BooleanWrapper", + "IntWrapper", + "LongWrapper", + "FloatWrapper", + "DoubleWrapper", + "BytesWrapper", + "StringWrapper", + "DateWrapper", + "DecimalWrapper", + "TimeMicrosWrapper", + "TimestampMicrosWrapper", + "ArrayWrapper", + ]; + for (i, name) in wrapper_names.iter().enumerate() { + match &union.variants()[i + 1] { + AvroSchema::Record(RecordSchema { + name: schema_name, .. + }) => { + assert_eq!( + schema_name.name, + *name, + "wrapper at index {} mismatch", + i + 1 + ); + } + other => panic!( + "Expected Record schema for {} at index {}, got {:?}", + name, + i + 1, + other + ), + } + } } } _ => panic!("Expected a Union schema for orderingVal"), @@ -242,6 +264,8 @@ mod tests { #[test] fn test_schema_for_delete_record() { + // The ordering value uses wrapper records: union index 2 = IntWrapper + let int_wrapper = AvroValue::Record(vec![("value".to_string(), AvroValue::Int(42))]); let delete_record_value = AvroValue::Record(vec![ ( "recordKey".to_string(), @@ -253,7 +277,7 @@ mod tests { ), ( "orderingVal".to_string(), - AvroValue::Union(1, Box::new(AvroValue::Int(42))), + AvroValue::Union(2, Box::new(int_wrapper)), ), ]); @@ -264,7 +288,20 @@ mod tests { match schema { AvroSchema::Record(RecordSchema { fields, .. }) => { - validate_delete_fields(&fields, Some(AvroSchema::Int)); + // The minimized type is the IntWrapper record (at union index 2) + match &fields[2].schema { + AvroSchema::Union(union) => { + assert_eq!(union.variants().len(), 2); + assert_eq!(union.variants()[0], AvroSchema::Null); + match &union.variants()[1] { + AvroSchema::Record(RecordSchema { name, .. }) => { + assert_eq!(name.name, "IntWrapper"); + } + other => panic!("Expected IntWrapper record, got {:?}", other), + } + } + _ => panic!("Expected Union schema"), + } } _ => panic!("Expected a Record schema"), } diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rs index e3515810..ffe53712 100644 --- a/crates/core/src/schema/mod.rs +++ b/crates/core/src/schema/mod.rs @@ -22,6 +22,7 @@ use crate::schema::resolver::sanitize_avro_schema_str; use arrow_schema::{Schema, SchemaRef}; use serde_json::Value; +pub mod avro_schema_utils; pub mod delete; pub mod resolver; diff --git a/crates/core/src/storage/mod.rs b/crates/core/src/storage/mod.rs index e0c04917..205e3445 100644 --- a/crates/core/src/storage/mod.rs +++ b/crates/core/src/storage/mod.rs @@ -281,6 +281,15 @@ impl Storage { let reader = ParquetObjectReader::new(obj_store, obj_path).with_file_size(meta.size); let builder = ParquetRecordBatchStreamBuilder::new(reader).await?; let schema = builder.schema().clone(); + + let all_cols: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + log::debug!( + "Parquet full read (non-streaming) for '{relative_path}': \ + reading all {n} cols: [{cols}]", + n = all_cols.len(), + cols = all_cols.join(", "), + ); + let mut stream = builder.build()?; let mut batches = Vec::new(); @@ -295,6 +304,49 @@ impl Storage { Ok(concat_batches(&schema, &batches)?) } + /// Read a Parquet file with column projection (non-streaming). + /// + /// Like [`get_parquet_file_data`], but only reads the columns present in + /// `projection_schema`. Reuses the projection infrastructure from + /// [`get_parquet_file_stream`]. + /// + /// Mirrors Java's `readerContext.getFileRecordIterator(pathInfo, start, len, + /// tableSchema, requiredSchema, storage)` where `requiredSchema` drives + /// column pruning. + pub async fn get_parquet_file_data_projected( + &self, + relative_path: &str, + projection_schema: &arrow_schema::SchemaRef, + ) -> Result { + let column_names: Vec = projection_schema + .fields() + .iter() + .map(|f| f.name().clone()) + .collect(); + + log::debug!( + "Parquet projected read (non-streaming) for '{relative_path}': \ + projecting {n} cols: [{cols}]", + n = column_names.len(), + cols = column_names.join(", "), + ); + + let options = ParquetReadOptions::new().with_projection(column_names); + let mut stream = self.get_parquet_file_stream(relative_path, options).await?; + let schema = stream.schema.clone(); + + let mut batches = Vec::new(); + while let Some(r) = stream.stream.next().await { + batches.push(r?); + } + + if batches.is_empty() { + return Ok(RecordBatch::new_empty(schema)); + } + + Ok(concat_batches(&schema, &batches)?) + } + /// Get a streaming reader for a Parquet file. /// /// Returns a [ParquetFileStream] that yields record batches as they are read, @@ -317,12 +369,37 @@ impl Storage { let mut builder = ParquetRecordBatchStreamBuilder::new(reader).await?; if let Some(batch_size) = options.batch_size { + log::debug!("Parquet reader: batch_size={batch_size} for '{relative_path}'"); builder = builder.with_batch_size(batch_size); } - // Handle projection: convert column names to indices using builder's schema + // Handle projection: convert column names to indices using builder's schema. + // Build `projected_schema` that reflects the actual columns in the stream. + let projected_schema: SchemaRef; if let Some(ref column_names) = options.projection { let arrow_schema = builder.schema(); + let file_cols: Vec<&str> = arrow_schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + let pruned_cols: Vec<&str> = file_cols + .iter() + .copied() + .filter(|&c| !column_names.iter().any(|n| n == c)) + .collect(); + log::debug!( + "Parquet column projection for '{relative_path}': \ + file schema [{file_schema}] ({n_file} cols), \ + projecting [{proj}] ({n_proj} cols), \ + pruning [{pruned}] ({n_pruned} cols)", + file_schema = file_cols.join(", "), + n_file = file_cols.len(), + proj = column_names.join(", "), + n_proj = column_names.len(), + pruned = pruned_cols.join(", "), + n_pruned = pruned_cols.len(), + ); let projection: Vec = column_names .iter() .map(|name| { @@ -340,18 +417,42 @@ impl Storage { }) .collect::>>()?; + // Build the projected schema in parquet root column order (ascending index). + // ProjectionMask::roots produces columns sorted by their parquet index, + // so the schema must match that order. + let mut sorted_projection = projection.clone(); + sorted_projection.sort(); + let projected_fields: Vec> = sorted_projection + .iter() + .map(|&idx| arrow_schema.field(idx).clone().into()) + .collect(); + projected_schema = std::sync::Arc::new(arrow_schema::Schema::new(projected_fields)); + let projection_mask = parquet::arrow::ProjectionMask::roots( builder.parquet_schema(), projection.iter().copied(), ); builder = builder.with_projection(projection_mask); + } else { + let arrow_schema = builder.schema(); + let all_cols: Vec<&str> = arrow_schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + log::debug!( + "Parquet column projection for '{relative_path}': no projection, \ + reading all {n} cols: [{cols}]", + n = all_cols.len(), + cols = all_cols.join(", "), + ); + projected_schema = arrow_schema.clone(); } - let schema = builder.schema().clone(); let stream = builder.build()?; Ok(ParquetFileStream { - schema, + schema: projected_schema, stream: Box::pin(stream), }) } diff --git a/crates/core/src/storage/reader.rs b/crates/core/src/storage/reader.rs index b0123165..a28aad60 100644 --- a/crates/core/src/storage/reader.rs +++ b/crates/core/src/storage/reader.rs @@ -35,6 +35,13 @@ impl StorageReader { let reader = BufReader::with_capacity(bytes.len(), Cursor::new(bytes)); Ok(Self { reader }) } + + /// Get a reference to the underlying file bytes. + /// + /// `Bytes` is ref-counted — cloning is O(1). + pub fn get_ref_bytes(&self) -> &Bytes { + self.reader.get_ref().get_ref() + } } impl Read for StorageReader { diff --git a/crates/core/src/table/file_pruner.rs b/crates/core/src/table/file_pruner.rs index 500a2a3b..5dfb0c2e 100644 --- a/crates/core/src/table/file_pruner.rs +++ b/crates/core/src/table/file_pruner.rs @@ -61,6 +61,13 @@ impl FilePruner { .map(|f| f.name().as_str()) .collect(); + // Capture expressions before shadowing for logging + let input_exprs: Vec = and_filters + .iter() + .filter(|f| !partition_columns.contains(f.field_name.as_str())) + .map(|f| format!("{} {} {}", f.field_name, f.operator, f.field_value)) + .collect(); + // Only keep filters on non-partition columns that exist in the table schema let and_filters: Vec = and_filters .iter() @@ -68,6 +75,16 @@ impl FilePruner { .filter_map(|filter| SchemableFilter::try_from((filter.clone(), table_schema)).ok()) .collect(); + if and_filters.is_empty() { + log::debug!("FilePruner: no data-column filters — all files included"); + } else { + log::debug!( + "FilePruner: {} data-column filter(s): [{}]", + and_filters.len(), + input_exprs.join(" AND "), + ); + } + Ok(FilePruner { and_filters }) } @@ -110,7 +127,16 @@ impl FilePruner { // Check if this filter can prune the file if self.can_prune_by_filter(filter, col_stats) { - return false; // File can be pruned + log::debug!( + "FilePruner: pruning — filter '{} {}' excludes file \ + (col '{}': min={:?}, max={:?})", + filter.field.name(), + filter.operator, + col_stats.column_name, + col_stats.min_value, + col_stats.max_value, + ); + return false; } } diff --git a/crates/core/src/table/fs_view.rs b/crates/core/src/table/fs_view.rs index a3740c1e..322adbc9 100644 --- a/crates/core/src/table/fs_view.rs +++ b/crates/core/src/table/fs_view.rs @@ -154,15 +154,22 @@ impl FileSystemView { as_of_timestamp: &str, ) -> Vec { if file_pruner.is_empty() { + log::debug!("FilePruner: no data-column filters, skipping stats pruning"); return file_groups; } - let mut retained = Vec::with_capacity(file_groups.len()); + let total = file_groups.len(); + let mut retained = Vec::with_capacity(total); for mut fg in file_groups { if let Some(fsl) = fg.get_file_slice_mut_as_of(as_of_timestamp) { let relative_path = match fsl.base_file_relative_path() { - Ok(path) => path, + Ok(Some(path)) => path, + Ok(None) => { + // Log-only slice: no base file to prune from, include it + retained.push(fg); + continue; + } Err(e) => { log::warn!( "Cannot get base file path for pruning: {e}. Including file group." @@ -194,10 +201,18 @@ impl FileSystemView { } }; + log::debug!( + "FilePruner: evaluating '{}' | {} row(s), {} column stat(s) loaded", + relative_path, + stats.num_rows.unwrap_or(0), + stats.columns.len(), + ); + if file_pruner.should_include(&stats) { + log::debug!("FilePruner: including '{relative_path}'"); retained.push(fg); } else { - log::debug!("Pruned file {relative_path} based on column stats"); + log::debug!("FilePruner: pruned '{relative_path}' based on column stats"); } } else { // No file slice as of timestamp, include the file group @@ -206,6 +221,10 @@ impl FileSystemView { } } + log::debug!( + "FilePruner: retained {}/{total} file group(s) after stats pruning", + retained.len(), + ); retained } @@ -367,7 +386,16 @@ mod tests { .collect::>(); assert_eq!(file_ids, vec!["a079bdb3-731c-4894-b855-abfcd6921007-0"]); for fsl in file_slices.iter() { - assert_eq!(fsl.base_file.file_metadata.as_ref().unwrap().num_records, 4); + assert_eq!( + fsl.base_file + .as_ref() + .unwrap() + .file_metadata + .as_ref() + .unwrap() + .num_records, + 4 + ); } } @@ -408,7 +436,16 @@ mod tests { .collect::>(); assert_eq!(file_ids, vec!["ebcb261d-62d3-4895-90ec-5b3c9622dff4-0"]); for fsl in file_slices.iter() { - assert_eq!(fsl.base_file.file_metadata.as_ref().unwrap().num_records, 1); + assert_eq!( + fsl.base_file + .as_ref() + .unwrap() + .file_metadata + .as_ref() + .unwrap() + .num_records, + 1 + ); } } @@ -460,7 +497,16 @@ mod tests { .collect::>(); assert_eq!(file_ids, vec!["a22e8257-e249-45e9-ba46-115bc85adcba-0"]); for fsl in file_slices.iter() { - assert_eq!(fsl.base_file.file_metadata.as_ref().unwrap().num_records, 2); + assert_eq!( + fsl.base_file + .as_ref() + .unwrap() + .file_metadata + .as_ref() + .unwrap() + .num_records, + 2 + ); } } } diff --git a/crates/core/src/table/listing.rs b/crates/core/src/table/listing.rs index ebf1bb55..cae8af3b 100644 --- a/crates/core/src/table/listing.rs +++ b/crates/core/src/table/listing.rs @@ -136,7 +136,6 @@ impl FileLister { } let mut file_groups: Vec = Vec::new(); - // TODO support creating file groups without base files for (file_id, base_files) in file_id_to_base_files.into_iter() { let mut file_group = FileGroup::new(file_id.to_owned(), partition_path.to_string()); @@ -147,6 +146,14 @@ impl FileLister { file_groups.push(file_group); } + + // Build FileGroups for log-only file groups (no base files). + for (file_id, log_files) in file_id_to_log_files.into_iter() { + let mut file_group = FileGroup::new(file_id.to_owned(), partition_path.to_string()); + file_group.add_log_files(log_files)?; + file_groups.push(file_group); + } + Ok(file_groups) } diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index e861ed1f..54f7579b 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -76,10 +76,10 @@ //! for file_slice_vec in file_slices { //! let file_group_vec = file_slice_vec //! .iter() -//! .map(|f| { -//! let relative_path = f.base_file_relative_path().unwrap(); +//! .filter_map(|f| { +//! let relative_path = f.base_file_relative_path().unwrap()?; //! let url = join_url_segments(&base_uri, &[relative_path.as_str()]).unwrap(); -//! url.path().to_string() +//! Some(url.path().to_string()) //! }) //! .collect(); //! parquet_file_groups.push(file_group_vec) @@ -104,7 +104,7 @@ use crate::config::table::HudiTableConfig::PartitionFields; use crate::config::table::{HudiTableConfig, TableTypeValue}; use crate::expr::filter::{Filter, from_str_tuples}; use crate::file_group::file_slice::FileSlice; -use crate::file_group::reader::FileGroupReader; +use crate::file_group::reader_v1::FileGroupReader; use crate::metadata::METADATA_TABLE_PARTITION_FIELD; use crate::schema::resolver::{ resolve_avro_schema, resolve_avro_schema_with_meta_fields, resolve_data_schema, resolve_schema, @@ -851,7 +851,7 @@ mod tests { let mut file_paths = Vec::new(); let base_url = table.base_url(); for f in table.get_file_slices(filters.to_vec()).await? { - let relative_path = f.base_file_relative_path()?; + let relative_path = f.base_file_relative_path()?.unwrap(); let file_url = join_url_segments(&base_url, &[relative_path.as_str()])?; file_paths.push(file_url.to_string()); } @@ -1262,7 +1262,7 @@ mod tests { ); let file_slice = file_slices[0]; assert_eq!( - file_slice.base_file.file_name(), + file_slice.base_file.as_ref().unwrap().file_name(), "92e64357-e4d1-4639-a9d3-c3535829d0aa-0_1-53-79_20250121000647668.parquet" ); assert_eq!( @@ -1294,7 +1294,7 @@ mod tests { assert_eq!( file_slices .iter() - .map(|f| f.base_file_relative_path().unwrap()) + .map(|f| f.base_file_relative_path().unwrap().unwrap()) .collect::>(), vec!["a079bdb3-731c-4894-b855-abfcd6921007-0_0-203-274_20240418173551906.parquet",] ); @@ -1308,7 +1308,7 @@ mod tests { assert_eq!( file_slices .iter() - .map(|f| f.base_file_relative_path().unwrap()) + .map(|f| f.base_file_relative_path().unwrap().unwrap()) .collect::>(), vec!["a079bdb3-731c-4894-b855-abfcd6921007-0_0-203-274_20240418173551906.parquet",] ); @@ -1324,7 +1324,7 @@ mod tests { assert_eq!( file_slices .iter() - .map(|f| f.base_file_relative_path().unwrap()) + .map(|f| f.base_file_relative_path().unwrap().unwrap()) .collect::>(), vec!["a079bdb3-731c-4894-b855-abfcd6921007-0_0-182-253_20240418173550988.parquet",] ); @@ -1340,7 +1340,7 @@ mod tests { assert_eq!( file_slices .iter() - .map(|f| f.base_file_relative_path().unwrap()) + .map(|f| f.base_file_relative_path().unwrap().unwrap()) .collect::>(), Vec::::new() ); diff --git a/crates/core/src/table/partition.rs b/crates/core/src/table/partition.rs index 37699d77..37bb0ee6 100644 --- a/crates/core/src/table/partition.rs +++ b/crates/core/src/table/partition.rs @@ -94,6 +94,27 @@ impl PartitionPruner { .get_or_default(HudiTableConfig::IsPartitionPathUrlencoded) .into(); let is_partitioned = is_table_partitioned(hudi_configs); + + if and_filters.is_empty() { + log::debug!( + "PartitionPruner: no filters — all partitions included \ + (hive_style={is_hive_style}, url_encoded={is_url_encoded}, \ + partitioned={is_partitioned})" + ); + } else { + let exprs: Vec = transformed_filters + .iter() + .map(|f| format!("{} {} {}", f.field_name, f.operator, f.field_value)) + .collect(); + log::debug!( + "PartitionPruner: {} filter(s): [{}] \ + (hive_style={is_hive_style}, url_encoded={is_url_encoded}, \ + partitioned={is_partitioned})", + and_filters.len(), + exprs.join(" AND "), + ); + } + Ok(PartitionPruner { schema, is_hive_style, @@ -126,22 +147,48 @@ impl PartitionPruner { /// Returns `true` if the partition path should be included based on the filters. pub fn should_include(&self, partition_path: &str) -> bool { + if self.and_filters.is_empty() { + return true; + } + let segments = match self.parse_segments(partition_path) { Ok(s) => s, - Err(_) => return true, // Include the partition regardless of parsing error + Err(e) => { + log::debug!( + "PartitionPruner: parse error for '{}': {e} — including conservatively", + partition_path + ); + return true; + } }; - self.and_filters.iter().all(|filter| { - match segments.get(filter.field.name()) { - Some(segment_value) => { - match filter.apply_comparison(segment_value) { + let included = + self.and_filters + .iter() + .all(|filter| match segments.get(filter.field.name()) { + Some(segment_value) => match filter.apply_comparison(segment_value) { Ok(scalar) => scalar.value(0), - Err(_) => true, // Include the partition when comparison error occurs - } - } - None => true, // Include the partition when filtering field does not match any field in the partition - } - }) + Err(_) => true, + }, + None => true, + }); + + if !included { + let filter_exprs: Vec = self + .and_filters + .iter() + .map(|f| format!("{} {}", f.field.name(), f.operator)) + .collect(); + log::debug!( + "PartitionPruner: pruning '{}' (filters: [{}])", + partition_path, + filter_exprs.join(" AND "), + ); + } else { + log::debug!("PartitionPruner: including '{}'", partition_path); + } + + included } /// Returns true if the table uses a timestamp-based key generator, diff --git a/crates/core/src/timeline/mod.rs b/crates/core/src/timeline/mod.rs index 76ced9b9..f3ffb665 100644 --- a/crates/core/src/timeline/mod.rs +++ b/crates/core/src/timeline/mod.rs @@ -21,7 +21,7 @@ pub mod completion_time; pub mod instant; pub mod loader; pub mod lsm_tree; -pub(crate) mod selector; +pub mod selector; pub(crate) mod util; pub mod view; diff --git a/crates/core/src/timeline/selector.rs b/crates/core/src/timeline/selector.rs index 44047bb7..6277b8e6 100644 --- a/crates/core/src/timeline/selector.rs +++ b/crates/core/src/timeline/selector.rs @@ -29,8 +29,8 @@ use std::sync::Arc; #[derive(Debug, Clone)] pub struct InstantRange { timezone: String, - start_timestamp: Option, - end_timestamp: Option, + start_instant: Option, + end_instant: Option, start_inclusive: bool, end_inclusive: bool, } @@ -38,48 +38,48 @@ pub struct InstantRange { impl InstantRange { pub fn new( timezone: String, - start_timestamp: Option, - end_timestamp: Option, + start_instant: Option, + end_instant: Option, start_inclusive: bool, end_inclusive: bool, ) -> Self { Self { timezone, - start_timestamp, - end_timestamp, + start_instant, + end_instant, start_inclusive, end_inclusive, } } /// Create a new [InstantRange] with a closed end timestamp range. - pub fn up_to(end_timestamp: &str, timezone: &str) -> Self { + pub fn up_to(end_instant: &str, timezone: &str) -> Self { Self::new( timezone.to_string(), None, - Some(end_timestamp.to_string()), + Some(end_instant.to_string()), false, true, ) } /// Create a new [InstantRange] with an open timestamp range. - pub fn within(start_timestamp: &str, end_timestamp: &str, timezone: &str) -> Self { + pub fn within(start_instant: &str, end_instant: &str, timezone: &str) -> Self { Self::new( timezone.to_string(), - Some(start_timestamp.to_string()), - Some(end_timestamp.to_string()), + Some(start_instant.to_string()), + Some(end_instant.to_string()), false, false, ) } /// Create a new [InstantRange] with an open start and closed end timestamp range. - pub fn within_open_closed(start_timestamp: &str, end_timestamp: &str, timezone: &str) -> Self { + pub fn within_open_closed(start_instant: &str, end_instant: &str, timezone: &str) -> Self { Self::new( timezone.to_string(), - Some(start_timestamp.to_string()), - Some(end_timestamp.to_string()), + Some(start_instant.to_string()), + Some(end_instant.to_string()), false, true, ) @@ -89,15 +89,15 @@ impl InstantRange { &self.timezone } - pub fn start_timestamp(&self) -> Result>> { - self.start_timestamp + pub fn start_instant(&self) -> Result>> { + self.start_instant .as_deref() .map(|timestamp| Instant::parse_datetime(timestamp, &self.timezone)) .transpose() } - pub fn end_timestamp(&self) -> Result>> { - self.end_timestamp + pub fn end_instant(&self) -> Result>> { + self.end_instant .as_deref() .map(|timestamp| Instant::parse_datetime(timestamp, &self.timezone)) .transpose() @@ -105,7 +105,7 @@ impl InstantRange { pub fn is_in_range(&self, timestamp: &str, timezone: &str) -> Result { let t = Instant::parse_datetime(timestamp, timezone)?; - if let Some(start) = self.start_timestamp()? { + if let Some(start) = self.start_instant()? { if self.start_inclusive { if t < start { return Ok(false); @@ -115,7 +115,7 @@ impl InstantRange { } } - if let Some(end) = self.end_timestamp()? { + if let Some(end) = self.end_instant()? { if self.end_inclusive { if t > end { return Ok(false); @@ -385,8 +385,8 @@ mod tests { ); assert_eq!(range.timezone(), "UTC"); - assert_eq!(range.start_timestamp.as_deref(), Some("20240101000000000")); - assert_eq!(range.end_timestamp.as_deref(), Some("20241231235959999")); + assert_eq!(range.start_instant.as_deref(), Some("20240101000000000")); + assert_eq!(range.end_instant.as_deref(), Some("20241231235959999")); assert!(range.start_inclusive); assert!(!range.end_inclusive); } @@ -396,8 +396,8 @@ mod tests { let range = InstantRange::up_to("20241231235959999", "UTC"); assert_eq!(range.timezone(), "UTC"); - assert!(range.start_timestamp.is_none()); - assert_eq!(range.end_timestamp.as_deref(), Some("20241231235959999")); + assert!(range.start_instant.is_none()); + assert_eq!(range.end_instant.as_deref(), Some("20241231235959999")); assert!(!range.start_inclusive); assert!(range.end_inclusive); } @@ -407,8 +407,8 @@ mod tests { let range = InstantRange::within("20240101000000000", "20241231235959999", "UTC"); assert_eq!(range.timezone(), "UTC"); - assert_eq!(range.start_timestamp.as_deref(), Some("20240101000000000")); - assert_eq!(range.end_timestamp.as_deref(), Some("20241231235959999")); + assert_eq!(range.start_instant.as_deref(), Some("20240101000000000")); + assert_eq!(range.end_instant.as_deref(), Some("20241231235959999")); assert!(!range.start_inclusive); assert!(!range.end_inclusive); } @@ -419,8 +419,8 @@ mod tests { InstantRange::within_open_closed("20240101000000000", "20241231235959999", "UTC"); assert_eq!(range.timezone(), "UTC"); - assert_eq!(range.start_timestamp.as_deref(), Some("20240101000000000")); - assert_eq!(range.end_timestamp.as_deref(), Some("20241231235959999")); + assert_eq!(range.start_instant.as_deref(), Some("20240101000000000")); + assert_eq!(range.end_instant.as_deref(), Some("20241231235959999")); assert!(!range.start_inclusive); assert!(range.end_inclusive); } diff --git a/crates/core/tests/file_group_reader_tests.rs b/crates/core/tests/file_group_reader_tests.rs new file mode 100644 index 00000000..ca27eeac --- /dev/null +++ b/crates/core/tests/file_group_reader_tests.rs @@ -0,0 +1,1846 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! E2E integration tests for the new `HoodieFileGroupReader`. +//! +//! Tests read real Hudi v9 MOR file groups (COMMIT_TIME_ORDERING) with +//! base parquet + Avro log files through the new reader pipeline. +//! +//! ## Test data: v9_mor_8i4u_commit_time +//! +//! Schema: id INT, name STRING, age INT, ts STRING, city STRING (partitioned by city) +//! Commit 1: INSERT 8 rows → base .parquet per partition +//! Commit 2: UPSERT 4 rows (ids 1,3,5,7) → .log files +//! +//! Expected merged view (8 rows): +//! id=1 Alice-V2 age=31 city=sf +//! id=2 Bob age=25 city=sf +//! id=3 Carol-V2 age=36 city=nyc +//! id=4 Dave age=28 city=nyc +//! id=5 Eve-V2 age=33 city=chi +//! id=6 Frank age=40 city=chi +//! id=7 Grace-V2 age=28 city=la +//! id=8 Hank age=45 city=la + +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use hudi_core::config::HudiConfigs; +use hudi_core::config::table::HudiTableConfig; +use hudi_core::error::Result; +use hudi_core::expression::predicates::predicates_factory; +use hudi_core::expression::{Expression, Literal, NameReference, Predicate}; +use hudi_core::file_group::reader::HoodieFileGroupReader; +use hudi_core::file_group::reader::input_split::InputSplit; +use hudi_core::file_group::reader::reader_context::ReaderContext; +use hudi_core::file_group::reader::reader_parameters::ReaderParameters; +use hudi_core::storage::Storage; +use hudi_core::table::builder::OptionResolver; +use hudi_test::QuickstartTripsTable; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use std::fs::File; +use std::sync::Arc; +use std::sync::Arc as StdArc; + +/// Create HudiConfigs and Storage from table path using OptionResolver. +async fn create_configs_and_storage(table_path: &str) -> Result<(Arc, Arc)> { + let empty_opts: Vec<(&str, &str)> = vec![]; + let mut resolver = OptionResolver::new_with_options(table_path, empty_opts); + resolver.resolve_options().await?; + let hudi_configs = Arc::new(HudiConfigs::new(resolver.hudi_options)); + let storage = Storage::new(Arc::new(resolver.storage_options), hudi_configs.clone())?; + Ok((hudi_configs, storage)) +} + +/// Read a single file group through the new HoodieFileGroupReader. +async fn read_file_group( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, +) -> Result { + let (_hudi_configs, storage) = create_configs_and_storage(table_path).await?; + + let base_path = if base_file.is_empty() { + None + } else if partition.is_empty() { + Some(base_file.to_string()) + } else { + Some(format!("{}/{}", partition, base_file)) + }; + let log_paths: Vec = log_files + .iter() + .map(|lf| { + if partition.is_empty() { + lf.to_string() + } else { + format!("{}/{}", partition, lf) + } + }) + .collect(); + + let input_split = InputSplit::new(base_path, None, log_paths, partition.to_string()); + + let mut reader_context = ReaderContext::empty(); + reader_context.latest_commit_time = "99991231235959999".to_string(); + reader_context.merge_mode = "COMMIT_TIME_ORDERING".to_string(); + reader_context.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + reader_context.rebuild_record_context(partition.to_string()); + + let mut reader = HoodieFileGroupReader::new( + Arc::new(reader_context), + storage, + input_split, + ReaderParameters::default(), + None, // data_schema — None means no projection + None, // requested_schema + ); + + reader.read().await +} + +/// Read a file group with column projection. +/// +/// Passes `data_schema` + `requested_schema` to `HoodieFileGroupReader::new()`, +/// which mirrors Java's constructor: it creates the schema handler internally, +/// calls `prepare_required_schema()` to add mandatory merge fields, and sets +/// the `output_converter` for final projection. +async fn read_file_group_with_projection( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + requested_schema: SchemaRef, +) -> Result { + let (_hudi_configs, storage) = create_configs_and_storage(table_path).await?; + + let base_path = if base_file.is_empty() { + None + } else if partition.is_empty() { + Some(base_file.to_string()) + } else { + Some(format!("{}/{}", partition, base_file)) + }; + let log_paths: Vec = log_files + .iter() + .map(|lf| { + if partition.is_empty() { + lf.to_string() + } else { + format!("{}/{}", partition, lf) + } + }) + .collect(); + + let input_split = InputSplit::new(base_path.clone(), None, log_paths, partition.to_string()); + + let mut reader_context = ReaderContext::empty(); + reader_context.latest_commit_time = "99991231235959999".to_string(); + reader_context.merge_mode = "COMMIT_TIME_ORDERING".to_string(); + reader_context.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + reader_context.rebuild_record_context(partition.to_string()); + + // Read table schema from base file parquet metadata. + let data_schema: Option = if let Some(ref bp) = base_path { + storage + .get_parquet_file_schema(bp) + .await + .ok() + .map(|s| Arc::new(s) as SchemaRef) + } else { + None + }; + + // Pass data_schema + requested_schema to the constructor. + // The constructor creates the schema handler, calls prepare_required_schema, + // and sets the output_converter — matching Java's pattern. + let mut reader = HoodieFileGroupReader::new( + Arc::new(reader_context), + storage, + input_split, + ReaderParameters::default(), + data_schema, + Some(requested_schema), + ); + + reader.read().await +} + +/// Extract (id, name, age) tuples from a RecordBatch, sorted by id. +fn extract_id_name_age(batch: &arrow_array::RecordBatch) -> Vec<(i32, String, i32)> { + QuickstartTripsTable::id_name_age(batch) +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=sf (2 base rows, 1 update in log) +// ============================================================================= + +/// E2E: Read city=sf partition — base file + log file merge. +/// +/// Given: v9 MOR COMMIT_TIME_ORDERING table, city=sf partition +/// Base: (id=1, Alice, 30), (id=2, Bob, 25) +/// Log: (id=1, Alice-V2, 31) — update for id=1 +/// When: Read file group through HoodieFileGroupReader +/// Then: 2 rows: (1, Alice-V2, 31) and (2, Bob, 25) +#[tokio::test] +async fn test_e2e_v9_mor_commit_time_sf_merge() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let result = read_file_group( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + ) + .await?; + + let records = extract_id_name_age(&result); + assert_eq!(records.len(), 2, "sf partition should have 2 rows"); + assert_eq!( + records[0], + (1, "Alice-V2".to_string(), 31), + "id=1 should be updated" + ); + assert_eq!( + records[1], + (2, "Bob".to_string(), 25), + "id=2 should be unchanged" + ); + + Ok(()) +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=nyc (2 base rows, 1 update in log) +// ============================================================================= + +/// E2E: Read city=nyc partition — base file + log file merge. +/// +/// Given: Base: (id=3, Carol, 35), (id=4, Dave, 28) +/// Log: (id=3, Carol-V2, 36) +/// When: Read file group +/// Then: 2 rows: (3, Carol-V2, 36) and (4, Dave, 28) +#[tokio::test] +async fn test_e2e_v9_mor_commit_time_nyc_merge() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let result = read_file_group( + &table_path, + "city=nyc", + "cae699a1-42f7-4226-9bd3-7f0e49496028-0_3-13-36_20260408053032350.parquet", + vec![".cae699a1-42f7-4226-9bd3-7f0e49496028-0_20260408053037787.log.1_3-27-76"], + ) + .await?; + + let records = extract_id_name_age(&result); + assert_eq!(records.len(), 2, "nyc partition should have 2 rows"); + assert_eq!( + records[0], + (3, "Carol-V2".to_string(), 36), + "id=3 should be updated" + ); + assert_eq!( + records[1], + (4, "Dave".to_string(), 28), + "id=4 should be unchanged" + ); + + Ok(()) +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=chi (2 base rows, 1 update in log) +// ============================================================================= + +/// E2E: Read city=chi partition — base file + log file merge. +/// +/// Given: Base: (id=5, Eve, 32), (id=6, Frank, 40) +/// Log: (id=5, Eve-V2, 33) +/// When: Read file group +/// Then: 2 rows: (5, Eve-V2, 33) and (6, Frank, 40) +#[tokio::test] +async fn test_e2e_v9_mor_commit_time_chi_merge() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let result = read_file_group( + &table_path, + "city=chi", + "8fb566bd-0f0d-45ee-9f4b-391c1ceb9dda-0_2-13-35_20260408053032350.parquet", + vec![".8fb566bd-0f0d-45ee-9f4b-391c1ceb9dda-0_20260408053037787.log.1_2-27-75"], + ) + .await?; + + let records = extract_id_name_age(&result); + assert_eq!(records.len(), 2, "chi partition should have 2 rows"); + assert_eq!( + records[0], + (5, "Eve-V2".to_string(), 33), + "id=5 should be updated" + ); + assert_eq!( + records[1], + (6, "Frank".to_string(), 40), + "id=6 should be unchanged" + ); + + Ok(()) +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: city=la (2 base rows, 1 update in log) +// ============================================================================= + +/// E2E: Read city=la partition — base file + log file merge. +/// +/// Given: Base: (id=7, Grace, 27), (id=8, Hank, 45) +/// Log: (id=7, Grace-V2, 28) +/// When: Read file group +/// Then: 2 rows: (7, Grace-V2, 28) and (8, Hank, 45) +#[tokio::test] +async fn test_e2e_v9_mor_commit_time_la_merge() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let result = read_file_group( + &table_path, + "city=la", + "31d2005b-4c79-46f4-aca5-6519809b2503-0_1-13-34_20260408053032350.parquet", + vec![".31d2005b-4c79-46f4-aca5-6519809b2503-0_20260408053037787.log.1_1-27-74"], + ) + .await?; + + let records = extract_id_name_age(&result); + assert_eq!(records.len(), 2, "la partition should have 2 rows"); + assert_eq!( + records[0], + (7, "Grace-V2".to_string(), 28), + "id=7 should be updated" + ); + assert_eq!( + records[1], + (8, "Hank".to_string(), 45), + "id=8 should be unchanged" + ); + + Ok(()) +} + +// ============================================================================= +// v9 MOR: Base-only read (no log files) +// ============================================================================= + +/// E2E: Read base file only (no log files) — tests the no-merge path. +/// +/// Given: v9 MOR table, city=sf partition, base file only (no log files) +/// When: Read file group with empty log_files list +/// Then: Original base records returned (Alice with age=30, Bob with age=25) +#[tokio::test] +async fn test_e2e_v9_mor_base_only_read() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let result = read_file_group( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![], // no log files + ) + .await?; + + let records = extract_id_name_age(&result); + assert_eq!(records.len(), 2, "base-only should have 2 rows"); + assert_eq!( + records[0], + (1, "Alice".to_string(), 30), + "id=1 should be original (no update)" + ); + assert_eq!( + records[1], + (2, "Bob".to_string(), 25), + "id=2 should be original" + ); + + Ok(()) +} + +// ============================================================================= +// v9 MOR COMMIT_TIME_ORDERING: Non-partitioned, 2 log files (delete + update) +// ============================================================================= + +/// Extract (id, name, price) tuples from a RecordBatch, sorted by id. +fn extract_id_name_price(batch: &arrow_array::RecordBatch) -> Vec<(i32, String, f64)> { + QuickstartTripsTable::id_name_price(batch) +} + +/// E2E: Read non-partitioned file group with 2 log files — reproduces multi-log merge. +/// +/// Given: v9 MOR COMMIT_TIME_ORDERING non-partitioned table +/// Base: 7 rows (ids 0-6) from INSERT +/// Log 1: MERGE INTO DELETE (ids 0,1,2) — delete block +/// Log 2: MERGE INTO UPDATE (ids 4,5,6 → D2/E2/F2) — avro data block +/// When: Read file group through HoodieFileGroupReader +/// Then: 4 rows after merge: (3,C,30.0), (4,D2,45.0), (5,E2,55.0), (6,F2,65.0) +#[tokio::test] +async fn test_e2e_v9_mor_commit_time_nonpart_multi_log() -> Result<()> { + let table_path = QuickstartTripsTable::V9MorNonpart3Commits.path_to_mor_avro(); + + let result = read_file_group( + &table_path, + "", // non-partitioned + "960a29a0-0f78-401d-85b1-1cbc44b34121-0_0-846-1597_20260409002001492.parquet", + vec![ + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002002957.log.1_0-868-1644", + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002003963.log.1_0-890-1691", + ], + ) + .await?; + + let records = extract_id_name_price(&result); + assert_eq!( + records.len(), + 4, + "should have 4 rows after merge (3 deleted, 3 updated)" + ); + assert_eq!( + records[0], + (3, "C".to_string(), 30.0), + "id=3 should be unchanged" + ); + assert_eq!( + records[1], + (4, "D2".to_string(), 45.0), + "id=4 should be updated" + ); + assert_eq!( + records[2], + (5, "E2".to_string(), 55.0), + "id=5 should be updated" + ); + assert_eq!( + records[3], + (6, "F2".to_string(), 65.0), + "id=6 should be updated" + ); + + Ok(()) +} + +// ============================================================================= +// MOR File Slice Layout Tests +// +// These tests read the 4 MOR v9 tables created by Hudi Spark's +// TestMORFileSliceLayouts and validate the HoodieFileGroupReader output +// against the gold data (SELECT * result saved as parquet). +// +// All tables: v9, MOR, COMMIT_TIME_ORDERING, non-partitioned, 1 file group. +// ============================================================================= + +/// Read gold parquet data from disk. +fn read_gold_parquet(gold_dir: &str) -> arrow_array::RecordBatch { + let entries: Vec<_> = std::fs::read_dir(gold_dir) + .expect("gold_data dir should exist") + .filter_map(|e| e.ok()) + .filter(|e| e.path().extension().map_or(false, |ext| ext == "parquet")) + .collect(); + assert!(!entries.is_empty(), "no parquet files in {gold_dir}"); + let file = File::open(entries[0].path()).expect("open gold parquet"); + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .expect("build parquet reader") + .build() + .expect("build reader"); + let batches: Vec<_> = reader.map(|b| b.expect("read batch")).collect(); + arrow::compute::concat_batches(&batches[0].schema(), &batches).expect("concat gold batches") +} + +/// Sort a RecordBatch by the "key" column (String type, ascending). +fn sort_by_key(batch: &arrow_array::RecordBatch) -> arrow_array::RecordBatch { + let key_col = batch + .column(batch.schema().index_of("key").expect("key column")) + .clone(); + let indices = arrow_ord::sort::sort_to_indices(&key_col, None, None).expect("sort"); + let columns: Vec<_> = batch + .columns() + .iter() + .map(|col| arrow_select::take::take(col, &indices, None).expect("take")) + .collect(); + arrow_array::RecordBatch::try_new(batch.schema(), columns).expect("sorted batch") +} + +/// Compare actual reader output against gold data. +/// +/// Compares non-hoodie-metadata columns (skipping `_hoodie_*` columns) +/// by sorting both by "key" and checking row count + per-cell values. +fn assert_matches_gold( + actual: &arrow_array::RecordBatch, + gold: &arrow_array::RecordBatch, + table_name: &str, +) { + let actual_sorted = sort_by_key(actual); + let gold_sorted = sort_by_key(gold); + + assert_eq!( + actual_sorted.num_rows(), + gold_sorted.num_rows(), + "[{table_name}] row count mismatch: actual={} gold={}", + actual_sorted.num_rows(), + gold_sorted.num_rows(), + ); + + // Compare user columns (skip _hoodie_* metadata columns) + let user_cols: Vec = gold_sorted + .schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .filter(|n| !n.starts_with("_hoodie_")) + .collect(); + + for col_name in &user_cols { + let actual_idx = actual_sorted.schema().index_of(col_name); + let gold_idx = gold_sorted.schema().index_of(col_name).unwrap(); + if actual_idx.is_err() { + continue; + } + let actual_col = actual_sorted.column(actual_idx.unwrap()); + let gold_col = gold_sorted.column(gold_idx); + + // Skip timestamp columns if the types differ (us vs ns, tz vs no-tz) + // since these are representation differences, not value differences. + let is_timestamp = matches!( + actual_col.data_type(), + arrow_schema::DataType::Timestamp(_, _) + ) || matches!( + gold_col.data_type(), + arrow_schema::DataType::Timestamp(_, _) + ); + if is_timestamp && actual_col.data_type() != gold_col.data_type() { + continue; + } + + for row in 0..actual_sorted.num_rows() { + let actual_str = arrow_cast::display::array_value_to_string(actual_col, row) + .unwrap_or_else(|_| "".to_string()); + let gold_str = arrow_cast::display::array_value_to_string(gold_col, row) + .unwrap_or_else(|_| "".to_string()); + assert_eq!( + actual_str, gold_str, + "[{table_name}] mismatch at row={row} col={col_name}: actual={actual_str} gold={gold_str}", + ); + } + } +} + +// ============================================================================= +// Test 1: Log compaction — log-only, 5 log files, compacted log block +// ============================================================================= + +/// E2E: Read MOR table with compacted log block (log compaction). +/// +/// Layout: 1 file group, NO base file, 5 log files (includes compacted block). +/// Expected: 3 rows — matches gold data from SELECT * via Spark. +#[tokio::test] +async fn test_e2e_v9_mor_log_compaction() -> Result<()> { + let table_path = QuickstartTripsTable::MorLayoutLogCompaction.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutLogCompaction.path_to_mor_avro_gold(); + + let result = read_file_group( + &table_path, + "", + "", + vec![ + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030511461.log.1_0-16-23", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030518232.log.1_0-30-49", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030519923.log.1_0-44-78", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030521407.log.1_0-58-110", + ".7483a08a-02f1-4510-bc1d-1317924f4189-0_20260409030522412.log.1_0-67-128", + ], + ) + .await?; + + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&result, &gold, "table_log_compaction"); + Ok(()) +} + +// ============================================================================= +// Test 2: Log only — log-only, 3 log files (insert + update + delete) +// ============================================================================= + +/// E2E: Read MOR table with log files only (no base file). +/// +/// Layout: 1 file group, NO base file, 3 log files (data + data + delete). +/// Expected: 2 rows — k3 deleted, matches gold data. +#[tokio::test] +async fn test_e2e_v9_mor_log_only() -> Result<()> { + let table_path = QuickstartTripsTable::MorLayoutLogOnly.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutLogOnly.path_to_mor_avro_gold(); + + let result = read_file_group( + &table_path, + "", + "", + vec![ + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030525348.log.1_0-102-176", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030527298.log.1_0-116-202", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030528554.log.1_0-130-231", + ], + ) + .await?; + + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&result, &gold, "table_log_only"); + Ok(()) +} + +// ============================================================================= +// Test 3: Mixed column types — base + 2 log files (update + delete), full schema +// ============================================================================= + +/// E2E: Read MOR table with mixed column types (int, string, double, array, map, etc.). +/// +/// Layout: 1 file group, 1 base file + 2 log files. +/// Expected: 2 rows — k3 deleted, matches gold data. +#[tokio::test] +async fn test_e2e_v9_mor_mixed_column_types() -> Result<()> { + let table_path = QuickstartTripsTable::MorLayoutColumnProjection.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutColumnProjection.path_to_mor_avro_gold(); + + let result = read_file_group( + &table_path, + "", + "78076137-b2c4-410d-8473-3d6366ae0985-0_0-165-279_20260409030530945.parquet", + vec![ + ".78076137-b2c4-410d-8473-3d6366ae0985-0_20260409030532996.log.1_0-179-305", + ".78076137-b2c4-410d-8473-3d6366ae0985-0_20260409030534379.log.1_0-193-334", + ], + ) + .await?; + + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&result, &gold, "table_column_projection"); + Ok(()) +} + +// ============================================================================= +// Test 4: All data types — base + 3 log files (update + delete + update) +// ============================================================================= + +/// E2E: Read MOR table with all data types and comprehensive log blocks. +/// +/// Layout: 1 file group, 1 base file + 3 log files. +/// Expected: 4 rows — k4 deleted, matches gold data. +#[tokio::test] +async fn test_e2e_v9_mor_all_data_types() -> Result<()> { + let table_path = QuickstartTripsTable::MorLayoutAllDataTypes.path_to_mor_avro(); + let gold_dir = QuickstartTripsTable::MorLayoutAllDataTypes.path_to_mor_avro_gold(); + + let result = read_file_group( + &table_path, + "", + "c887c1e8-5fb9-475e-8171-769c5cf10c61-0_0-240-395_20260409030537482.parquet", + vec![ + ".c887c1e8-5fb9-475e-8171-769c5cf10c61-0_20260409030539332.log.1_0-254-421", + ".c887c1e8-5fb9-475e-8171-769c5cf10c61-0_20260409030540482.log.1_0-268-450", + ".c887c1e8-5fb9-475e-8171-769c5cf10c61-0_20260409030541620.log.1_0-282-482", + ], + ) + .await?; + + let gold = read_gold_parquet(&gold_dir); + assert_matches_gold(&result, &gold, "table_all_data_types"); + Ok(()) +} + +// ============================================================================= +// Test: Column projection — request subset of columns through schema handler +// ============================================================================= + +/// E2E: Column projection via `FileGroupReaderSchemaHandler` — explicit schemas. +/// +/// Uses the city=sf partition (base + log merge) from V9Mor8I4UCommitTime. +/// Requests only `id` and `name` columns. The schema handler should: +/// 1. Add `_hoodie_record_key` to required_schema (mandatory for merge) +/// 2. Read base file with only [id, name, _hoodie_record_key] +/// 3. Merge with log file (which also has _hoodie_record_key) +/// 4. Project output back to [id, name] via OutputConverter +/// +/// NOTE: This test passes `data_schema` and `requested_schema` directly to +/// `HoodieFileGroupReader::new()`. This exercises the direct construction path +/// but does NOT test the FFI/builder path where schemas live on +/// `ReaderContext.schema_handler`. See `test_e2e_v9_mor_column_projection_via_builder` +/// for the builder path test. +#[tokio::test] +async fn test_e2e_v9_mor_column_projection_via_schema_handler() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + // Request only id and name — NOT _hoodie_record_key, age, ts, city, etc. + let requested_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + ])); + + let result = read_file_group_with_projection( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + requested_schema, + ) + .await?; + + // Output should have exactly 2 columns: id and name. + let schema = result.schema(); + let output_col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!( + output_col_names, + vec!["id", "name"], + "output should contain only the requested columns" + ); + + // Verify merge results are correct (same as the non-projected test). + let ids = result + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let names = result + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let mut rows: Vec<(i32, String)> = ids + .iter() + .zip(names.iter()) + .map(|(id, name)| (id.unwrap(), name.unwrap().to_string())) + .collect(); + rows.sort_by_key(|(id, _)| *id); + + assert_eq!(rows.len(), 2, "sf partition should have 2 rows"); + assert_eq!( + rows[0], + (1, "Alice-V2".to_string()), + "id=1 should be updated" + ); + assert_eq!(rows[1], (2, "Bob".to_string()), "id=2 should be unchanged"); + + Ok(()) +} + +// ============================================================================= +// Tests: Column projection via the builder path (schemas on ReaderContext) +// +// These tests exercise the code path used by the FFI bridge, where schemas +// are placed on `ReaderContext.schema_handler` and the `CoreFileGroupReader` +// is constructed via `.builder()` WITHOUT explicit `data_schema` / +// `requested_schema`. +// +// Before FIX 1 (commit wiring ReaderContext.schema_handler), the builder path +// created an empty schema handler, so `required_schema = None`, the base file +// was read with ALL columns, and no output projection was applied. +// ============================================================================= + +use hudi_core::file_group::reader::schema_handler::FileGroupReaderSchemaHandler; + +/// Helper: Read a file group via the builder path (schemas on ReaderContext). +/// +/// Mimics the FFI bridge flow: +/// 1. Populate `ReaderContext.schema_handler` with table_schema, data_schema, +/// and requested_schema (just like `new_file_group_reader_with_context` does). +/// 2. Construct `HoodieFileGroupReader` via `.builder()` WITHOUT calling +/// `.with_data_schema()` or `.with_requested_schema()`. +/// 3. The constructor should pick up schemas from `reader_context.schema_handler`. +async fn read_file_group_via_builder( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + requested_schema: SchemaRef, +) -> Result { + let (_hudi_configs, storage) = create_configs_and_storage(table_path).await?; + + let base_path = if base_file.is_empty() { + None + } else if partition.is_empty() { + Some(base_file.to_string()) + } else { + Some(format!("{}/{}", partition, base_file)) + }; + let log_paths: Vec = log_files + .iter() + .map(|lf| { + if partition.is_empty() { + lf.to_string() + } else { + format!("{}/{}", partition, lf) + } + }) + .collect(); + + let input_split = InputSplit::new(base_path.clone(), None, log_paths, partition.to_string()); + + // Read table schema from the base file parquet metadata (like the FFI bridge does). + let table_schema: Option = if let Some(ref bp) = base_path { + storage + .get_parquet_file_schema(bp) + .await + .ok() + .map(|s| Arc::new(s) as SchemaRef) + } else { + None + }; + + // Build schema handler the same way the FFI bridge does: + // data_schema → with_table_schema + with_data_schema + // requested_schema → with_requested_schema + let schema_handler = { + let mut handler = FileGroupReaderSchemaHandler::new(); + if let Some(ts) = table_schema { + handler = handler.with_table_schema(ts.clone()).with_data_schema(ts); + } + handler = handler.with_requested_schema(requested_schema); + handler + }; + + let mut reader_context = ReaderContext::empty(); + reader_context.latest_commit_time = "99991231235959999".to_string(); + reader_context.merge_mode = "COMMIT_TIME_ORDERING".to_string(); + reader_context.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + reader_context.rebuild_record_context(partition.to_string()); + reader_context.schema_handler = schema_handler; + reader_context.has_log_files = !log_files.is_empty(); + + // Use the builder WITHOUT setting data_schema or requested_schema. + // This forces the constructor to use reader_context.schema_handler. + let mut reader = HoodieFileGroupReader::builder() + .with_reader_context(Arc::new(reader_context)) + .with_storage(storage) + .with_input_split(input_split) + .with_reader_parameters(ReaderParameters::default()) + .build()?; + + reader.read().await +} + +/// E2E: Column projection via the builder path — MOR merge with projection. +/// +/// This is the critical test that would have caught the original bug: +/// schemas are on `ReaderContext.schema_handler` (not passed as explicit args). +/// The constructor must pick them up from `reader_context.schema_handler`. +/// +/// Requests only `id` and `name` on the city=sf partition (MOR with log files). +/// Validates that: +/// - Output has exactly [id, name] — merge-internal fields stripped +/// - Merge results are correct (Alice-V2, Bob) +/// - Base file was NOT read with all columns (indirectly: if required_schema +/// was None the output would have all columns, failing the schema assert) +#[tokio::test] +async fn test_e2e_v9_mor_column_projection_via_builder() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let requested_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + ])); + + let result = read_file_group_via_builder( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + requested_schema, + ) + .await?; + + // Output should have exactly 2 columns: id and name. + let schema = result.schema(); + let output_col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!( + output_col_names, + vec!["id", "name"], + "builder path: output should contain only the requested columns, \ + not all parquet columns (was the bug before FIX 1)" + ); + + // Verify merge results are correct. + let ids = result + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let names = result + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let mut rows: Vec<(i32, String)> = ids + .iter() + .zip(names.iter()) + .map(|(id, name)| (id.unwrap(), name.unwrap().to_string())) + .collect(); + rows.sort_by_key(|(id, _)| *id); + + assert_eq!(rows.len(), 2); + assert_eq!(rows[0], (1, "Alice-V2".to_string())); + assert_eq!(rows[1], (2, "Bob".to_string())); + + Ok(()) +} + +/// E2E: COW (base-only) column projection via builder — no log files. +/// +/// Tests that column pruning works on the COW path (no merge needed). +/// Requests only `id` and `age` columns. No log files, so +/// `generate_required_schema` returns `requested_schema` as-is (COW path). +/// Output should have exactly [id, age] with correct values. +#[tokio::test] +async fn test_e2e_cow_column_projection_via_builder() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let requested_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("age", DataType::Int32, true), + ])); + + // No log files → COW path. + let result = read_file_group_via_builder( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![], + requested_schema, + ) + .await?; + + let schema = result.schema(); + let output_col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!( + output_col_names, + vec!["id", "age"], + "COW builder path: output should contain only [id, age]" + ); + + assert_eq!(result.num_rows(), 2, "sf base has 2 rows"); + + let ids = result + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let ages = result + .column_by_name("age") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let mut rows: Vec<(i32, i32)> = ids + .iter() + .zip(ages.iter()) + .map(|(id, age)| (id.unwrap(), age.unwrap())) + .collect(); + rows.sort_by_key(|(id, _)| *id); + + assert_eq!(rows[0], (1, 30), "id=1 original age=30 (no merge)"); + assert_eq!(rows[1], (2, 25), "id=2 original age=25"); + + Ok(()) +} + +/// E2E: Single column projection via builder — request only `name`. +/// +/// Validates minimal column projection: only 1 user column requested. +/// For MOR, the schema handler must still add `_hoodie_record_key` + +/// `_hoodie_is_deleted` + `_hoodie_operation` internally, but the output +/// must contain ONLY [name]. +#[tokio::test] +async fn test_e2e_mor_single_column_projection_via_builder() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + + let requested_schema: SchemaRef = + Arc::new(Schema::new(vec![Field::new("name", DataType::Utf8, true)])); + + let result = read_file_group_via_builder( + &table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + requested_schema, + ) + .await?; + + let schema = result.schema(); + let output_col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!( + output_col_names, + vec!["name"], + "single column projection: output should contain only [name]" + ); + + assert_eq!(result.num_rows(), 2); + + let names = result + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let mut values: Vec = names.iter().map(|n| n.unwrap().to_string()).collect(); + values.sort(); + + assert!( + values.contains(&"Alice-V2".to_string()), + "merged name should be Alice-V2" + ); + assert!( + values.contains(&"Bob".to_string()), + "unchanged name should be Bob" + ); + + Ok(()) +} + +// ============================================================================= +// Component tests: validate schema_handler.required_schema has the right columns +// ============================================================================= + +/// Component: Verify `required_schema` includes only the expected columns. +/// +/// Constructs a `FileGroupReaderSchemaHandler` mimicking the FFI flow, calls +/// `prepare_required_schema`, and verifies the computed `required_schema` +/// contains the requested columns PLUS mandatory merge fields, but NOT the +/// full parquet schema. +#[test] +fn test_component_required_schema_is_pruned_not_full() { + let table_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("_hoodie_commit_time", DataType::Utf8, true), + Field::new("_hoodie_commit_seqno", DataType::Utf8, true), + Field::new("_hoodie_record_key", DataType::Utf8, true), + Field::new("_hoodie_partition_path", DataType::Utf8, true), + Field::new("_hoodie_file_name", DataType::Utf8, true), + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + Field::new("age", DataType::Int32, true), + Field::new("ts", DataType::Utf8, true), + ])); + + let requested_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + ])); + + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema) + .with_requested_schema(requested_schema); + + let mut props = std::collections::HashMap::new(); + props.insert( + "hoodie.table.precombine.field".to_string(), + "ts".to_string(), + ); + props.insert( + "hoodie.table.recordkey.fields".to_string(), + "_hoodie_record_key".to_string(), + ); + + handler.prepare_required_schema( + true, // has_log_files (MOR) + &["_hoodie_record_key".to_string()], + &[], // no ordering fields for COMMIT_TIME_ORDERING + &props, + false, + "COMMIT_TIME_ORDERING", + ); + + let required = handler + .required_schema + .as_ref() + .expect("required_schema should be computed"); + let required_cols: Vec<&str> = required + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); + + // Must include user-requested columns. + assert!(required_cols.contains(&"id"), "must include requested 'id'"); + assert!( + required_cols.contains(&"name"), + "must include requested 'name'" + ); + + // Must include merge-mandatory field. + assert!( + required_cols.contains(&"_hoodie_record_key"), + "must include _hoodie_record_key for merge" + ); + + // Must NOT include columns that are neither requested nor mandatory. + assert!( + !required_cols.contains(&"age"), + "should NOT include 'age' — not requested and not mandatory" + ); + assert!( + !required_cols.contains(&"ts"), + "should NOT include 'ts' — not requested, not mandatory \ + (COMMIT_TIME_ORDERING does not require ordering fields)" + ); + assert!( + !required_cols.contains(&"_hoodie_commit_seqno"), + "should NOT include _hoodie_commit_seqno — not mandatory" + ); + assert!( + !required_cols.contains(&"_hoodie_partition_path"), + "should NOT include _hoodie_partition_path — not mandatory" + ); + assert!( + !required_cols.contains(&"_hoodie_file_name"), + "should NOT include _hoodie_file_name — not mandatory" + ); + + // output_converter should exist because required ≠ requested. + assert!( + handler.get_output_converter().is_some(), + "output_converter should be Some because required_schema has more \ + columns than requested_schema" + ); +} + +/// Component: COW path — required_schema equals requested_schema. +/// +/// For COW (no log files), `generate_required_schema` should return the +/// requested schema as-is. No mandatory fields are added. +#[test] +fn test_component_cow_required_equals_requested() { + let table_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("_hoodie_commit_time", DataType::Utf8, true), + Field::new("_hoodie_record_key", DataType::Utf8, true), + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + Field::new("age", DataType::Int32, true), + ])); + + let requested_schema: SchemaRef = + Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, true)])); + + let mut handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(table_schema.clone()) + .with_data_schema(table_schema) + .with_requested_schema(requested_schema.clone()); + + handler.prepare_required_schema( + false, // no log files (COW) + &["_hoodie_record_key".to_string()], + &[], + &std::collections::HashMap::new(), + false, + "COMMIT_TIME_ORDERING", + ); + + let required = handler.required_schema.as_ref().unwrap(); + assert_eq!( + required, &requested_schema, + "COW: required_schema should equal requested_schema exactly" + ); + + // No output converter needed (required == requested). + assert!( + handler.get_output_converter().is_none(), + "COW: no output converter when required == requested" + ); +} + +/// Component: When schemas are on ReaderContext, the builder should use them. +/// +/// This is the component-level equivalent of the E2E builder tests. +/// Constructs ReaderContext with a pre-populated schema_handler and verifies +/// that `HoodieFileGroupReader::new()` with `data_schema=None` picks it up. +#[tokio::test] +async fn test_component_builder_uses_reader_context_schema_handler() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let (_hudi_configs, storage) = create_configs_and_storage(&table_path).await?; + + let base_file = + "city=sf/fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet"; + + // Read the full parquet schema. + let full_schema: SchemaRef = Arc::new(storage.get_parquet_file_schema(base_file).await?); + let full_col_count = full_schema.fields().len(); + + // Request only 2 columns. + let requested_schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + ])); + + let schema_handler = FileGroupReaderSchemaHandler::new() + .with_table_schema(full_schema.clone()) + .with_data_schema(full_schema) + .with_requested_schema(requested_schema); + + let mut reader_context = ReaderContext::empty(); + reader_context.latest_commit_time = "99991231235959999".to_string(); + reader_context.merge_mode = "COMMIT_TIME_ORDERING".to_string(); + reader_context.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + reader_context.rebuild_record_context("city=sf".to_string()); + reader_context.schema_handler = schema_handler; + + // COW path (no log files) — simplest case. + let input_split = InputSplit::new( + Some(base_file.to_string()), + None, + Vec::new(), + "city=sf".to_string(), + ); + + // Build via builder — no explicit schemas. + let mut reader = HoodieFileGroupReader::builder() + .with_reader_context(Arc::new(reader_context)) + .with_storage(storage) + .with_input_split(input_split) + .with_reader_parameters(ReaderParameters::default()) + .build()?; + + let result = reader.read().await?; + + // The base parquet has many columns, but output should have only 2. + assert!( + full_col_count > 2, + "parquet should have more than 2 columns (has {full_col_count})" + ); + assert_eq!( + result.num_columns(), + 2, + "builder path with schema_handler: output should have only 2 columns, \ + not all {full_col_count} parquet columns" + ); + + let schema = result.schema(); + let output_col_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!(output_col_names, vec!["id", "name"]); + + Ok(()) +} + +// ============================================================================= +// Phase 3 helpers — keyFilterOpt e2e smoke test +// ============================================================================= + +/// Variant of `read_file_group` (lines 69-121 of this file) that accepts a +/// `key_filter_opt`. Mirrors the existing helper, with the addition of +/// `reader_context.key_filter_opt = key_filter_opt;` on the ctx before the +/// reader is constructed. +async fn read_file_group_with_key_filter( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + key_filter_opt: Option>, +) -> Result { + let (_hudi_configs, storage) = create_configs_and_storage(table_path).await?; + + let base_path = if base_file.is_empty() { + None + } else if partition.is_empty() { + Some(base_file.to_string()) + } else { + Some(format!("{}/{}", partition, base_file)) + }; + let log_paths: Vec = log_files + .iter() + .map(|lf| { + if partition.is_empty() { + lf.to_string() + } else { + format!("{}/{}", partition, lf) + } + }) + .collect(); + + let input_split = InputSplit::new(base_path, None, log_paths, partition.to_string()); + + let mut reader_context = ReaderContext::empty(); + reader_context.latest_commit_time = "99991231235959999".to_string(); + reader_context.merge_mode = "COMMIT_TIME_ORDERING".to_string(); + reader_context.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + reader_context.rebuild_record_context(partition.to_string()); + reader_context.key_filter_opt = key_filter_opt; // ← the only addition + + let mut reader = HoodieFileGroupReader::new( + Arc::new(reader_context), + storage, + input_split, + ReaderParameters::default(), + None, + None, + ); + + reader.read().await +} + +/// Look up the `_hoodie_record_key` string for a given `id` in the baseline +/// batch. Used to derive the literal value for the test filter without +/// hard-coding the keygen encoding. +fn lookup_record_key(batch: &arrow_array::RecordBatch, id: i32) -> String { + let id_col_idx = batch.schema().index_of("id").expect("id column missing"); + let key_col_idx = batch + .schema() + .index_of("_hoodie_record_key") + .expect("_hoodie_record_key column missing"); + + let id_col = batch + .column(id_col_idx) + .as_any() + .downcast_ref::() + .expect("id column should be Int32"); + let key_col = batch + .column(key_col_idx) + .as_any() + .downcast_ref::() + .expect("_hoodie_record_key column should be StringArray"); + + for i in 0..batch.num_rows() { + if id_col.value(i) == id { + return key_col.value(i).to_string(); + } + } + panic!("id {id} not found in baseline batch"); +} + +/// Pull the row tuple `(id, name, age)` for the given id, or None. +fn extract_row_with_id_opt( + batch: &arrow_array::RecordBatch, + id: i32, +) -> Option<(i32, String, i32)> { + let id_col = batch + .column_by_name("id")? + .as_any() + .downcast_ref::()?; + let name_col = batch + .column_by_name("name")? + .as_any() + .downcast_ref::()?; + let age_col = batch + .column_by_name("age")? + .as_any() + .downcast_ref::()?; + + for i in 0..batch.num_rows() { + if id_col.value(i) == id { + return Some(( + id_col.value(i), + name_col.value(i).to_string(), + age_col.value(i), + )); + } + } + None +} + +// ============================================================================= +// Phase 3 e2e smoke test — keyFilterOpt actually filters rows +// ============================================================================= + +/// Test 1: filter on a key updated in log → 1 row, value from log. +/// +/// Validates: filter applies during BOTH log scan and base read. id=1 is +/// updated in log → filtered output should have the log value (Alice-V2). +#[tokio::test] +async fn fg_filter_in_log_updated_key() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + // First read no-filter to discover id=1's record key dynamically. + let baseline_for_key_lookup = + read_file_group_with_key_filter(&table_path, partition, base_file, log_files.clone(), None) + .await?; + let key_for_id1 = lookup_record_key(&baseline_for_key_lookup, 1); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id1)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + ab.assert_filtered_ids_eq(&[1]); // exactly id=1 + + // Cross-validate: filtered id=1 row equals baseline id=1 row. + let expected = extract_row_with_id_opt(&ab.baseline, 1).expect("id=1 in baseline"); + let actual = extract_row_with_id_opt(&ab.filtered, 1).expect("id=1 in filtered"); + assert_eq!( + expected, actual, + "filtered id=1 must equal baseline id=1 (Alice-V2)" + ); + + Ok(()) +} + +// ============================================================================= +// keyFilterOpt e2e coverage — shared helpers (FilterAbResult + ab_read_with_filter) +// +// Each test in this section ABs a no-filter read against a filtered read of +// the same file group, then asserts via FilterAbResult that the filter is +// real (or, for unsupported predicates, that it's a no-op). +// ============================================================================= + +/// Result of an AB read: same file group read once with no filter and once +/// with `Some(filter)`. +struct FilterAbResult { + baseline: arrow_array::RecordBatch, + filtered: arrow_array::RecordBatch, +} + +impl FilterAbResult { + /// Assert filter is real, not a no-op: filtered must have STRICTLY fewer + /// rows than baseline. + fn assert_filter_narrowed(&self) { + assert!( + self.filtered.num_rows() < self.baseline.num_rows(), + "filter did not narrow rows: baseline={}, filtered={} \ + (this means the filter was a no-op — bug)", + self.baseline.num_rows(), + self.filtered.num_rows(), + ); + } + + /// Assert filter is a no-op: filtered.num_rows == baseline.num_rows AND + /// every row in filtered also appears in baseline (compared by sorted + /// id-only since row order may differ between two reads). + fn assert_filter_was_noop(&self) { + assert_eq!( + self.filtered.num_rows(), + self.baseline.num_rows(), + "expected filter to be no-op (predicate not In/StringStartsWithAny), \ + but row counts differ: baseline={}, filtered={}", + self.baseline.num_rows(), + self.filtered.num_rows(), + ); + let baseline_ids: std::collections::BTreeSet = ids_in_batch(&self.baseline); + let filtered_ids: std::collections::BTreeSet = ids_in_batch(&self.filtered); + assert_eq!( + baseline_ids, filtered_ids, + "expected same id set after no-op filter" + ); + } + + /// Assert that filtered.num_rows == 0 AND baseline has rows (filter + /// narrowed everything away). + fn assert_filtered_empty(&self) { + assert!( + self.baseline.num_rows() > 0, + "baseline must have rows for the empty-filter assertion to be meaningful" + ); + assert_eq!( + self.filtered.num_rows(), + 0, + "expected filter to drop all rows, got {}", + self.filtered.num_rows() + ); + } + + /// Assert the exact set of `id`s present in filtered (sorted). + fn assert_filtered_ids_eq(&self, expected: &[i32]) { + let actual: Vec = ids_in_batch(&self.filtered).into_iter().collect(); + let mut expected_sorted = expected.to_vec(); + expected_sorted.sort(); + assert_eq!(actual, expected_sorted, "filtered id set mismatch"); + } +} + +/// Helper: collect `id` column values from a RecordBatch as a sorted set. +fn ids_in_batch(batch: &arrow_array::RecordBatch) -> std::collections::BTreeSet { + let id_col = batch + .column_by_name("id") + .expect("id column missing from batch") + .as_any() + .downcast_ref::() + .expect("id column should be Int32"); + (0..batch.num_rows()).map(|i| id_col.value(i)).collect() +} + +/// Shared driver: read once with None, once with Some(filter), return both. +async fn ab_read_with_filter( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + filter: StdArc, +) -> Result { + let baseline = + read_file_group_with_key_filter(table_path, partition, base_file, log_files.clone(), None) + .await?; + let filtered = + read_file_group_with_key_filter(table_path, partition, base_file, log_files, Some(filter)) + .await?; + Ok(FilterAbResult { baseline, filtered }) +} + +// ============================================================================= +// Fixture locators +// ============================================================================= + +/// Fixture locator: V9Mor8I4UCommitTime city=sf file group. +/// Same filenames as test_e2e_v9_mor_commit_time_sf_merge. +fn sf_file_group() -> (String, &'static str, &'static str, Vec<&'static str>) { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + ( + table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + ) +} + +/// Fixture locator: V9MorNonpart3Commits — non-partitioned, single file group, +/// 1 base + 2 log files (delete block + update block). +/// Schema: id INT, name STRING, price DOUBLE, ts LONG. +fn nonpart_3commits_file_group() -> (String, &'static str, &'static str, Vec<&'static str>) { + let table_path = QuickstartTripsTable::V9MorNonpart3Commits.path_to_mor_avro(); + ( + table_path, + "", + "960a29a0-0f78-401d-85b1-1cbc44b34121-0_0-846-1597_20260409002001492.parquet", + vec![ + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002002957.log.1_0-868-1644", + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002003963.log.1_0-890-1691", + ], + ) +} + +/// Fixture locator: MorLayoutLogOnly — no base file, 3 log files. +fn log_only_file_group() -> (String, &'static str, &'static str, Vec<&'static str>) { + let table_path = QuickstartTripsTable::MorLayoutLogOnly.path_to_mor_avro(); + ( + table_path, + "", + "", + vec![ + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030525348.log.1_0-102-176", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030527298.log.1_0-116-202", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030528554.log.1_0-130-231", + ], + ) +} + +/// Test 2: filter on a key that exists ONLY in the base file (no log update). +/// +/// Validates: ISOLATES the base-file filter (apply_key_filter_to_batch). +/// id=2 in city=sf has no log update — log scan finds no records for it, +/// only the base-file filter narrows the output. +#[tokio::test] +async fn fg_filter_in_base_only_key() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + let baseline_for_key_lookup = + read_file_group_with_key_filter(&table_path, partition, base_file, log_files.clone(), None) + .await?; + let key_for_id2 = lookup_record_key(&baseline_for_key_lookup, 2); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id2)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + ab.assert_filtered_ids_eq(&[2]); // exactly id=2 + + // Cross-validate: filtered id=2 row equals baseline id=2 row (Bob, base value). + let expected = extract_row_with_id_opt(&ab.baseline, 2).expect("id=2 in baseline"); + let actual = extract_row_with_id_opt(&ab.filtered, 2).expect("id=2 in filtered"); + assert_eq!( + expected, actual, + "filtered id=2 must equal baseline id=2 (Bob)" + ); + + Ok(()) +} + +/// Test 3: filter on a non-existent key → 0 rows. +/// +/// Validates: filter actually drops everything when nothing matches. +/// Catches a bug where filter is silently a no-op. +#[tokio::test] +async fn fg_filter_in_no_match() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string("definitely-not-a-real-record-key")) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filtered_empty(); // baseline > 0, filtered == 0 + + Ok(()) +} + +/// Test 4: filter via StringStartsWithAny → PrefixKeys path. +/// +/// Validates: exercises the PrefixKeys KeySpec branch of create_key_spec +/// (Predicates::StringStartsWithAny → KeySpec::PrefixKeys). Uses id=1's +/// full record-key string as the prefix; this yields a single-row match +/// (a prefix that equals one key matches exactly that key). +#[tokio::test] +async fn fg_filter_starts_with_any_prefix() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + let baseline_for_key_lookup = + read_file_group_with_key_filter(&table_path, partition, base_file, log_files.clone(), None) + .await?; + let prefix_for_id1 = lookup_record_key(&baseline_for_key_lookup, 1); + + let filter: StdArc = StdArc::new(predicates_factory::starts_with_any( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(prefix_for_id1)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + ab.assert_filtered_ids_eq(&[1]); + + Ok(()) +} + +/// Test 5: unsupported predicate (eq) → filter is a no-op. +/// +/// Validates: regression guard. create_key_spec only handles In and +/// StringStartsWithAny — anything else returns None, which means the +/// filter must NOT drop rows. This catches a bug where an unsupported +/// predicate accidentally drops rows. +#[tokio::test] +async fn fg_filter_unsupported_predicate_is_noop() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + // BinaryComparison (eq) is not In/StringStartsWithAny → create_key_spec returns None. + let filter: StdArc = StdArc::new(predicates_factory::eq( + Box::new(NameReference::new("_hoodie_record_key")), + Box::new(Literal::string("anything")), + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_was_noop(); // baseline.num_rows == filtered.num_rows; same id set + + Ok(()) +} + +/// Schema-specific extractor for V9MorNonpart3Commits. +/// Schema: id INT, name STRING, price DOUBLE, ts LONG. +fn extract_row_with_id_opt_v9nonpart( + batch: &arrow_array::RecordBatch, + id: i32, +) -> Option<(i32, String, f64, i64)> { + let id_col = batch + .column_by_name("id")? + .as_any() + .downcast_ref::()?; + let name_col = batch + .column_by_name("name")? + .as_any() + .downcast_ref::()?; + let price_col = batch + .column_by_name("price")? + .as_any() + .downcast_ref::()?; + let ts_col = batch + .column_by_name("ts")? + .as_any() + .downcast_ref::()?; + + for i in 0..batch.num_rows() { + if id_col.value(i) == id { + return Some(( + id_col.value(i), + name_col.value(i).to_string(), + price_col.value(i), + ts_col.value(i), + )); + } + } + None +} + +/// Test 6: filter on a deleted-by-log key → 0 rows in filtered output. +/// +/// Validates: filter passes through process_delete_block correctly. +/// V9MorNonpart3Commits: id=0 is deleted by log block. To look up id=0's +/// record key, read the base file alone (no logs) — that unmerged view +/// still contains id=0. +#[tokio::test] +async fn fg_filter_in_with_delete_block() -> Result<()> { + let (table_path, partition, base_file, log_files) = nonpart_3commits_file_group(); + + // Read base-only (no log files) to recover id=0's record key. + let base_only = + read_file_group_with_key_filter(&table_path, partition, base_file, vec![], None).await?; + let key_for_id0 = lookup_record_key(&base_only, 0); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id0)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + // Baseline already excludes id=0 (deleted by log). + assert!( + extract_row_with_id_opt_v9nonpart(&ab.baseline, 0).is_none(), + "baseline must NOT contain id=0 (deleted by log)" + ); + + // Filtered must also exclude id=0 (filter applied to deleted key → 0 rows). + assert!( + extract_row_with_id_opt_v9nonpart(&ab.filtered, 0).is_none(), + "filtered must NOT contain id=0" + ); + + // Stronger: filtered should be empty since the only key in the filter is id=0, + // which is deleted, so no rows survive the filter. + ab.assert_filtered_empty(); + + Ok(()) +} + +/// Test 7: filter on a log-updated key on the non-partitioned fixture. +/// +/// Validates: same as Test 1 but on V9MorNonpart3Commits (non-partitioned, +/// 2 log files). id=5 is updated by log 2 → filtered output should reflect +/// the log update (E2/55.0). +#[tokio::test] +async fn fg_filter_in_log_updated_key_nonpart() -> Result<()> { + let (table_path, partition, base_file, log_files) = nonpart_3commits_file_group(); + + let baseline_for_key_lookup = + read_file_group_with_key_filter(&table_path, partition, base_file, log_files.clone(), None) + .await?; + let key_for_id5 = lookup_record_key(&baseline_for_key_lookup, 5); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id5)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 4 + ab.assert_filtered_ids_eq(&[5]); + + // Cross-validate: filtered id=5 row should be the log-updated value (E2, 55.0). + let expected = extract_row_with_id_opt_v9nonpart(&ab.baseline, 5).expect("id=5 in baseline"); + let actual = extract_row_with_id_opt_v9nonpart(&ab.filtered, 5).expect("id=5 in filtered"); + assert_eq!( + expected, actual, + "filtered id=5 must equal baseline id=5 (log-updated)" + ); + // Sanity: name should be "E2" (the log update value). + assert_eq!( + expected.1, "E2", + "id=5 baseline value should be log update (E2)" + ); + + Ok(()) +} + +// ============================================================================= +// Test 8: log-only filter — isolates log-scan filter +// ============================================================================= + +/// Test 8: filter on a log-only file group (no base file). +/// +/// Validates: ISOLATES the log-scan filter (KeyBasedFileGroupRecordBuffer +/// process_data_block / process_delete_block). All output flows through +/// the log-scan path because there is no base file. +/// +/// MorLayoutLogOnly schema: key STRING, ts INT64, level STRING, severity INT32, +/// partition STRING, round INT32. _hoodie_record_key values are "k1", "k2" (k3 deleted). +/// +/// Picks one key from the baseline batch ("k1") and filters on it; expects the +/// filtered output to contain exactly that key's row (and no others). +#[tokio::test] +async fn fg_filter_in_log_only_filegroup() -> Result<()> { + let (table_path, partition, base_file, log_files) = log_only_file_group(); + + // Read baseline first to discover one valid record key. + let baseline_for_key_lookup = + read_file_group_with_key_filter(&table_path, partition, base_file, log_files.clone(), None) + .await?; + assert!( + baseline_for_key_lookup.num_rows() >= 2, + "log-only baseline must have >=2 rows for the filter to be a real narrow" + ); + + // Use the _hoodie_record_key of the first row. + let hoodie_key_col = baseline_for_key_lookup + .column_by_name("_hoodie_record_key") + .expect("MorLayoutLogOnly should expose _hoodie_record_key in merged output") + .as_any() + .downcast_ref::() + .expect("_hoodie_record_key should be StringArray"); + let target_key = hoodie_key_col.value(0).to_string(); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(target_key.clone())) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + + // Verify exactly the target key remains. + let filtered_keys: Vec = { + let key_col = ab + .filtered + .column_by_name("_hoodie_record_key") + .expect("_hoodie_record_key missing from filtered batch") + .as_any() + .downcast_ref::() + .expect("_hoodie_record_key should be StringArray"); + (0..ab.filtered.num_rows()) + .map(|i| key_col.value(i).to_string()) + .collect() + }; + assert_eq!( + filtered_keys, + vec![target_key.clone()], + "filtered keys should be exactly [target_key]" + ); + + Ok(()) +} diff --git a/crates/core/tests/table_read_tests.rs b/crates/core/tests/table_read_tests.rs index e3d424f9..74d3da81 100644 --- a/crates/core/tests/table_read_tests.rs +++ b/crates/core/tests/table_read_tests.rs @@ -45,28 +45,6 @@ mod v6_tables { Ok(()) } - #[tokio::test] - async fn test_non_partitioned() -> Result<()> { - for base_url in SampleTable::V6Nonpartitioned.urls() { - let hudi_table = Table::new(base_url.path()).await?; - let records = hudi_table.read_snapshot(empty_filters()).await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![ - (1, "Alice", false), - (2, "Bob", false), - (3, "Carol", true), - (4, "Diana", true), - ] - ); - } - Ok(()) - } - #[tokio::test] async fn test_non_partitioned_read_optimized() -> Result<()> { let base_url = SampleTable::V6Nonpartitioned.url_to_mor_parquet(); @@ -101,46 +79,6 @@ mod v6_tables { Ok(()) } - #[tokio::test] - async fn test_non_partitioned_rollback() -> Result<()> { - let base_url = SampleTable::V6NonpartitionedRollback.url_to_mor_parquet(); - let hudi_table = Table::new(base_url.path()).await?; - let records = hudi_table.read_snapshot(empty_filters()).await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![ - (1, "Alice", true), // this was updated to false then rolled back to true - (2, "Bob", true), // this was updated to true after rollback - (3, "Carol", true), - ] - ); - Ok(()) - } - - #[tokio::test] - async fn test_complex_keygen_hive_style_with_filters() -> Result<()> { - for base_url in SampleTable::V6ComplexkeygenHivestyle.urls() { - let hudi_table = Table::new(base_url.path()).await?; - - let filters = vec![ - ("byteField", ">=", "10"), - ("byteField", "<", "20"), - ("shortField", "!=", "100"), - ]; - let records = hudi_table.read_snapshot(filters).await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!(sample_data, vec![(1, "Alice", false), (3, "Carol", true),]); - } - Ok(()) - } - #[tokio::test] async fn test_simple_keygen_hivestyle_no_metafields() -> Result<()> { for base_url in SampleTable::V6SimplekeygenHivestyleNoMetafields.urls() { @@ -194,109 +132,6 @@ mod v6_tables { } } - mod mor_log_file_queries { - use super::*; - - #[tokio::test] - async fn test_quickstart_trips_inserts_updates() -> Result<()> { - let base_url = QuickstartTripsTable::V6Trips8I1U.url_to_mor_avro(); - let hudi_table = Table::new(base_url.path()).await?; - - let updated_rider = "rider-D"; - - // verify updated record as of the latest commit - let records = hudi_table.read_snapshot(empty_filters()).await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let uuid_rider_and_fare = QuickstartTripsTable::uuid_rider_and_fare(&records) - .into_iter() - .filter(|(_, rider, _)| rider == updated_rider) - .collect::>(); - assert_eq!(uuid_rider_and_fare.len(), 1); - assert_eq!( - uuid_rider_and_fare[0].0, - "9909a8b1-2d15-4d3d-8ec9-efc48c536a00" - ); - assert_eq!(uuid_rider_and_fare[0].2, 25.0); - - // verify updated record as of the first commit - let commit_timestamps = hudi_table - .timeline - .completed_commits - .iter() - .map(|i| i.timestamp.as_str()) - .collect::>(); - let first_commit = commit_timestamps[0]; - let records = hudi_table - .read_snapshot_as_of(first_commit, empty_filters()) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let uuid_rider_and_fare = QuickstartTripsTable::uuid_rider_and_fare(&records) - .into_iter() - .filter(|(_, rider, _)| rider == updated_rider) - .collect::>(); - assert_eq!(uuid_rider_and_fare.len(), 1); - assert_eq!( - uuid_rider_and_fare[0].0, - "9909a8b1-2d15-4d3d-8ec9-efc48c536a00" - ); - assert_eq!(uuid_rider_and_fare[0].2, 33.9); - - Ok(()) - } - - #[tokio::test] - async fn test_quickstart_trips_inserts_deletes() -> Result<()> { - let base_url = QuickstartTripsTable::V6Trips8I3D.url_to_mor_avro(); - let hudi_table = Table::new(base_url.path()).await?; - - let deleted_riders = ["rider-A", "rider-C", "rider-D"]; - - // verify deleted record as of the latest commit - let records = hudi_table.read_snapshot(empty_filters()).await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let riders = QuickstartTripsTable::uuid_rider_and_fare(&records) - .into_iter() - .map(|(_, rider, _)| rider) - .collect::>(); - assert!( - riders - .iter() - .all(|rider| { !deleted_riders.contains(&rider.as_str()) }) - ); - - // verify deleted record as of the first commit - let commit_timestamps = hudi_table - .timeline - .completed_commits - .iter() - .map(|i| i.timestamp.as_str()) - .collect::>(); - let first_commit = commit_timestamps[0]; - let records = hudi_table - .read_snapshot_as_of(first_commit, empty_filters()) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let mut uuid_rider_and_fare = QuickstartTripsTable::uuid_rider_and_fare(&records) - .into_iter() - .filter(|(_, rider, _)| deleted_riders.contains(&rider.as_str())) - .collect::>(); - uuid_rider_and_fare.sort_unstable_by_key(|(_, rider, _)| rider.to_string()); - assert_eq!(uuid_rider_and_fare.len(), 3); - assert_eq!(uuid_rider_and_fare[0].1, "rider-A"); - assert_eq!(uuid_rider_and_fare[0].2, 19.10); - assert_eq!(uuid_rider_and_fare[1].1, "rider-C"); - assert_eq!(uuid_rider_and_fare[1].2, 27.70); - assert_eq!(uuid_rider_and_fare[2].1, "rider-D"); - assert_eq!(uuid_rider_and_fare[2].2, 33.90); - - Ok(()) - } - } - mod incremental_queries { use super::*; @@ -309,85 +144,6 @@ mod v6_tables { } Ok(()) } - - #[tokio::test] - async fn test_simplekeygen_nonhivestyle_overwritetable() -> Result<()> { - for base_url in SampleTable::V6SimplekeygenNonhivestyleOverwritetable.urls() { - let hudi_table = Table::new(base_url.path()).await?; - let commit_timestamps = hudi_table - .timeline - .completed_commits - .iter() - .map(|i| i.timestamp.as_str()) - .collect::>(); - assert_eq!(commit_timestamps.len(), 3); - let first_commit = commit_timestamps[0]; - let second_commit = commit_timestamps[1]; - let third_commit = commit_timestamps[2]; - - // read records changed from the beginning to the 1st commit - let records = hudi_table - .read_incremental_records("19700101000000", Some(first_commit)) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![(1, "Alice", true), (2, "Bob", false), (3, "Carol", true),], - "Should return 3 records inserted in the 1st commit" - ); - - // read records changed from the 1st to the 2nd commit - let records = hudi_table - .read_incremental_records(first_commit, Some(second_commit)) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![(1, "Alice", false), (4, "Diana", true),], - "Should return 2 records inserted or updated in the 2nd commit" - ); - - // read records changed from the 2nd to the 3rd commit - let records = hudi_table - .read_incremental_records(second_commit, Some(third_commit)) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![(4, "Diana", false),], - "Should return 1 record insert-overwritten in the 3rd commit" - ); - - // read records changed from the 1st commit - let records = hudi_table - .read_incremental_records(first_commit, None) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![(4, "Diana", false),], - "Should return 1 record insert-overwritten in the 3rd commit" - ); - - // read records changed from the 3rd commit - let records = hudi_table - .read_incremental_records(third_commit, None) - .await?; - assert!( - records.is_empty(), - "Should return 0 record as it's the latest commit" - ); - } - Ok(()) - } } } @@ -498,76 +254,6 @@ mod v8_tables { /// MOR log file tests for v8 tables mod mor_log_file_queries { use super::*; - - #[tokio::test] - async fn test_quickstart_trips_inserts_updates_deletes() -> Result<()> { - // V8Trips8I3U1D: 8 inserts, 3 updates (A, J, G fare=0), 2 deletes (F, J) - let base_url = QuickstartTripsTable::V8Trips8I3U1D.url_to_mor_avro(); - let hudi_table = Table::new(base_url.path()).await?; - - let deleted_riders = ["rider-F", "rider-J"]; - - // verify deleted records are not present in latest snapshot - let records = hudi_table.read_snapshot(empty_filters()).await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let uuid_rider_and_fare = QuickstartTripsTable::uuid_rider_and_fare(&records); - let riders: Vec<_> = uuid_rider_and_fare - .iter() - .map(|(_, rider, _)| rider.as_str()) - .collect(); - - // Deleted riders should not be present - assert!( - riders - .iter() - .all(|rider| { !deleted_riders.contains(rider) }) - ); - - // Should have 6 active riders (8 - 2 deleted) - assert_eq!(riders.len(), 6); - - // Verify updated fares (rider-A and rider-G have fare=0) - let rider_a = uuid_rider_and_fare - .iter() - .find(|(_, r, _)| r == "rider-A") - .expect("rider-A should exist"); - assert_eq!(rider_a.2, 0.0, "rider-A fare should be updated to 0"); - - let rider_g = uuid_rider_and_fare - .iter() - .find(|(_, r, _)| r == "rider-G") - .expect("rider-G should exist"); - assert_eq!(rider_g.2, 0.0, "rider-G fare should be updated to 0"); - - // verify deleted records were present in first commit (before updates/deletes) - let commit_timestamps = hudi_table - .timeline - .completed_commits - .iter() - .map(|i| i.timestamp.as_str()) - .collect::>(); - let first_commit = commit_timestamps[0]; - let records = hudi_table - .read_snapshot_as_of(first_commit, empty_filters()) - .await?; - let schema = &records[0].schema(); - let records = concat_batches(schema, &records)?; - let mut uuid_rider_and_fare = QuickstartTripsTable::uuid_rider_and_fare(&records) - .into_iter() - .filter(|(_, rider, _)| deleted_riders.contains(&rider.as_str())) - .collect::>(); - uuid_rider_and_fare.sort_unstable_by_key(|(_, rider, _)| rider.to_string()); - - // Both deleted riders should be present before delete - assert_eq!(uuid_rider_and_fare.len(), 2); - assert_eq!(uuid_rider_and_fare[0].1, "rider-F"); - assert_eq!(uuid_rider_and_fare[0].2, 34.15); - assert_eq!(uuid_rider_and_fare[1].1, "rider-J"); - assert_eq!(uuid_rider_and_fare[1].2, 17.85); - - Ok(()) - } } /// Streaming query tests for v8 tables @@ -799,39 +485,6 @@ mod streaming_queries { Ok(()) } - #[tokio::test] - async fn test_read_snapshot_stream_basic() -> Result<()> { - for base_url in SampleTable::V6Nonpartitioned.urls() { - let hudi_table = Table::new(base_url.path()).await?; - let options = ReadOptions::new(); - let mut stream = hudi_table.read_snapshot_stream(&options).await?; - - // Collect all batches from stream - let mut batches = Vec::new(); - while let Some(result) = stream.next().await { - batches.push(result?); - } - - assert!(!batches.is_empty(), "Should produce at least one batch"); - - // Concatenate batches and verify data - let schema = &batches[0].schema(); - let records = concat_batches(schema, &batches)?; - - let sample_data = SampleTable::sample_data_order_by_id(&records); - assert_eq!( - sample_data, - vec![ - (1, "Alice", false), - (2, "Bob", false), - (3, "Carol", true), - (4, "Diana", true), - ] - ); - } - Ok(()) - } - #[tokio::test] async fn test_read_snapshot_stream_with_batch_size() -> Result<()> { let base_url = SampleTable::V6Nonpartitioned.url_to_cow(); diff --git a/crates/datafusion/src/lib.rs b/crates/datafusion/src/lib.rs index b7e3200e..5f0de435 100644 --- a/crates/datafusion/src/lib.rs +++ b/crates/datafusion/src/lib.rs @@ -329,14 +329,22 @@ impl TableProvider for HudiDataSource { for file_slice_vec in file_slices { let mut parquet_file_group_vec = Vec::new(); for f in file_slice_vec { - let relative_path = f.base_file_relative_path().map_err(|e| { - Execution(format!( - "Failed to get base file relative path for {f:?} due to {e:?}" - )) - })?; + let relative_path = match f.base_file_relative_path() { + Ok(Some(path)) => path, + Ok(None) => continue, // Log-only slice: skip for DataFusion COW path + Err(e) => { + return Err(Execution(format!( + "Failed to get base file relative path for {f:?} due to {e:?}" + ))); + } + }; let url = join_url_segments(&base_url, &[relative_path.as_str()]) .map_err(|e| Execution(format!("Failed to join URL segments: {e:?}")))?; - let size = f.base_file.file_metadata.as_ref().map_or(0, |m| m.size); + let size = f + .base_file + .as_ref() + .and_then(|bf| bf.file_metadata.as_ref()) + .map_or(0, |m| m.size); let partitioned_file = PartitionedFile::new(url.path(), size); parquet_file_group_vec.push(partitioned_file); } diff --git a/crates/test/data/quickstart_trips_table/mor/avro/table_all_data_types.zip b/crates/test/data/quickstart_trips_table/mor/avro/table_all_data_types.zip new file mode 100644 index 00000000..49cc4c08 Binary files /dev/null and b/crates/test/data/quickstart_trips_table/mor/avro/table_all_data_types.zip differ diff --git a/crates/test/data/quickstart_trips_table/mor/avro/table_column_projection.zip b/crates/test/data/quickstart_trips_table/mor/avro/table_column_projection.zip new file mode 100644 index 00000000..11824e5e Binary files /dev/null and b/crates/test/data/quickstart_trips_table/mor/avro/table_column_projection.zip differ diff --git a/crates/test/data/quickstart_trips_table/mor/avro/table_log_compaction.zip b/crates/test/data/quickstart_trips_table/mor/avro/table_log_compaction.zip new file mode 100644 index 00000000..2874fb7b Binary files /dev/null and b/crates/test/data/quickstart_trips_table/mor/avro/table_log_compaction.zip differ diff --git a/crates/test/data/quickstart_trips_table/mor/avro/table_log_only.zip b/crates/test/data/quickstart_trips_table/mor/avro/table_log_only.zip new file mode 100644 index 00000000..4e9d8cc3 Binary files /dev/null and b/crates/test/data/quickstart_trips_table/mor/avro/table_log_only.zip differ diff --git a/crates/test/data/quickstart_trips_table/mor/avro/v9_mor_8i4u_commit_time.zip b/crates/test/data/quickstart_trips_table/mor/avro/v9_mor_8i4u_commit_time.zip new file mode 100644 index 00000000..18a41449 Binary files /dev/null and b/crates/test/data/quickstart_trips_table/mor/avro/v9_mor_8i4u_commit_time.zip differ diff --git a/crates/test/data/quickstart_trips_table/mor/avro/v9_mor_nonpart_3commits.zip b/crates/test/data/quickstart_trips_table/mor/avro/v9_mor_nonpart_3commits.zip new file mode 100644 index 00000000..0b8923a4 Binary files /dev/null and b/crates/test/data/quickstart_trips_table/mor/avro/v9_mor_nonpart_3commits.zip differ diff --git a/crates/test/src/lib.rs b/crates/test/src/lib.rs index 5d65edb5..1058a7c8 100644 --- a/crates/test/src/lib.rs +++ b/crates/test/src/lib.rs @@ -50,6 +50,31 @@ pub enum QuickstartTripsTable { V6Trips8I3D, #[strum(serialize = "v8_trips_8i3u1d")] V8Trips8I3U1D, + /// v9 MOR table, 8 inserts + 4 updates, COMMIT_TIME_ORDERING. + /// Schema: id INT, name STRING, age INT, ts STRING, city STRING (partitioned by city) + /// Commit 1: INSERT 8 rows → base .parquet per partition + /// Commit 2: UPSERT 4 rows (ids 1,3,5,7) → .log files + #[strum(serialize = "v9_mor_8i4u_commit_time")] + V9Mor8I4UCommitTime, + /// v9 MOR non-partitioned table, 3 commits (insert + merge-delete + merge-update). + /// Schema: id INT, name STRING, price DOUBLE, ts LONG (non-partitioned) + /// Commit 1: INSERT 7 rows (ids 0-6) → base .parquet + /// Commit 2: MERGE INTO DELETE 3 rows (ids 0,1,2) → .log file 1 (delete block) + /// Commit 3: MERGE INTO UPDATE 3 rows (ids 4,5,6) → .log file 2 (avro data block) + #[strum(serialize = "v9_mor_nonpart_3commits")] + V9MorNonpart3Commits, + /// v9 MOR non-partitioned, log-only with compacted log block (5 log files). + #[strum(serialize = "table_log_compaction")] + MorLayoutLogCompaction, + /// v9 MOR non-partitioned, log-only (3 log files: insert + update + delete). + #[strum(serialize = "table_log_only")] + MorLayoutLogOnly, + /// v9 MOR non-partitioned, base + 2 log files (update + delete), all column types. + #[strum(serialize = "table_column_projection")] + MorLayoutColumnProjection, + /// v9 MOR non-partitioned, base + 3 log files (update + delete + update), all data types. + #[strum(serialize = "table_all_data_types")] + MorLayoutAllDataTypes, } impl QuickstartTripsTable { @@ -102,10 +127,75 @@ impl QuickstartTripsTable { path_buf.to_str().unwrap().to_string() } + pub fn path_to_mor_avro_gold(&self) -> String { + let table = self.path_to_mor_avro(); + format!("{table}/gold_data") + } + pub fn url_to_mor_avro(&self) -> Url { let path = self.path_to_mor_avro(); Url::from_file_path(path).unwrap() } + + /// Extract (id, name, price) tuples from a RecordBatch (for v9_mor_nonpart tables), sorted by id. + pub fn id_name_price(record_batch: &RecordBatch) -> Vec<(i32, String, f64)> { + let ids = record_batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let names = record_batch + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let prices = record_batch + .column_by_name("price") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let mut result: Vec<(i32, String, f64)> = ids + .iter() + .zip(names.iter()) + .zip(prices.iter()) + .map(|((id, name), price)| (id.unwrap(), name.unwrap().to_string(), price.unwrap())) + .collect(); + result.sort_by_key(|(id, _, _)| *id); + result + } + + /// Extract (id, name, age) tuples from a RecordBatch (for v9_mor tables), sorted by id. + pub fn id_name_age(record_batch: &RecordBatch) -> Vec<(i32, String, i32)> { + let ids = record_batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let names = record_batch + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let ages = record_batch + .column_by_name("age") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let mut result: Vec<(i32, String, i32)> = ids + .iter() + .zip(names.iter()) + .zip(ages.iter()) + .map(|((id, name), age)| (id.unwrap(), name.unwrap().to_string(), age.unwrap())) + .collect(); + result.sort_by_key(|(id, _, _)| *id); + result + } } #[allow(dead_code)] @@ -225,17 +315,17 @@ mod tests { fn quickstart_trips_table_zip_file_should_exist() { for t in QuickstartTripsTable::iter() { match t { - QuickstartTripsTable::V6Trips8I1U => { - let path = t.zip_path("mor", Some("avro")); - assert!(path.exists()); - } - QuickstartTripsTable::V6Trips8I3D => { - let path = t.zip_path("mor", Some("avro")); - assert!(path.exists()); - } - QuickstartTripsTable::V8Trips8I3U1D => { + QuickstartTripsTable::V6Trips8I1U + | QuickstartTripsTable::V6Trips8I3D + | QuickstartTripsTable::V8Trips8I3U1D + | QuickstartTripsTable::V9Mor8I4UCommitTime + | QuickstartTripsTable::V9MorNonpart3Commits + | QuickstartTripsTable::MorLayoutLogCompaction + | QuickstartTripsTable::MorLayoutLogOnly + | QuickstartTripsTable::MorLayoutColumnProjection + | QuickstartTripsTable::MorLayoutAllDataTypes => { let path = t.zip_path("mor", Some("avro")); - assert!(path.exists()); + assert!(path.exists(), "Missing zip for {:?}: {:?}", t, path); } } } diff --git a/derby.log b/derby.log new file mode 100644 index 00000000..280e6222 --- /dev/null +++ b/derby.log @@ -0,0 +1,13 @@ +---------------------------------------------------------------- +Wed Apr 08 05:30:26 UTC 2026: +Booting Derby version The Apache Software Foundation - Apache Derby - 10.14.2.0 - (1828579): instance a816c00e-019d-6b92-0aa7-000007717440 +on database directory /home/ubuntu/ws3/hudi-rs/metastore_db with class loader jdk.internal.loader.ClassLoaders$AppClassLoader@5ffd2b27 +Loaded from file:/home/ubuntu/operations/sparks/spark-3.5.2-bin-spark-a95cfdae94/jars/derby-10.14.2.0.jar +java.vendor=Ubuntu +java.runtime.version=17.0.18+8-Ubuntu-122.04.1 +user.dir=/home/ubuntu/ws3/hudi-rs +os.name=Linux +os.arch=amd64 +os.version=6.8.0-1050-aws +derby.system.home=null +Database Class Loader started - derby.database.classpath='' diff --git a/docs/superpowers/plans/2026-05-07-keyfilteropt-e2e-coverage-implementation.md b/docs/superpowers/plans/2026-05-07-keyfilteropt-e2e-coverage-implementation.md new file mode 100644 index 00000000..4bbe9964 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-keyfilteropt-e2e-coverage-implementation.md @@ -0,0 +1,897 @@ +# keyFilterOpt e2e coverage expansion — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add 8 new e2e tests across 3 fixtures so each KeySpec variant, each filter call site, and the no-op corner case all have explicit AB-pattern validation. + +**Architecture:** All tests live in the existing `crates/core/tests/file_group_reader_tests.rs` (append-only). Each test calls a shared `ab_read_with_filter` driver that reads twice (no-filter baseline + filtered) and validates via `FilterAbResult` assertion methods. Schema-specific row extractors handle the three different fixture schemas. + +**Tech Stack:** Rust 2024, Tokio, Arrow `RecordBatch`, `hudi_test::QuickstartTripsTable`. + +**Spec reference:** `docs/superpowers/specs/2026-05-07-keyfilteropt-e2e-coverage-design.md` + +**Working from:** `/home/ubuntu/ws3/hudi-rs` on branch `davis/phase3migrateFilterOptCode`. + +**Key fixture facts (verified before plan):** +- `V9Mor8I4UCommitTime` city=sf base file: `fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet`; log: `.fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73`. Schema: `id INT, name STRING, age INT, ts STRING, city STRING`. +- `V9MorNonpart3Commits` base file: `960a29a0-0f78-401d-85b1-1cbc44b34121-0_0-846-1597_20260409002001492.parquet`; logs: `.960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002002957.log.1_0-868-1644` (delete block), `.960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002003963.log.1_0-890-1691` (update block). Schema: `id INT, name STRING, price DOUBLE, ts LONG`. Baseline merged content: 4 rows — `(3,"C",30.0)`, `(4,"D2",45.0)`, `(5,"E2",55.0)`, `(6,"F2",65.0)`. Ids 0/1/2 are deleted by log; ids 4/5/6 are updated to D2/E2/F2. +- `MorLayoutLogOnly` (no base file) logs: `.7787bafe-f674-4382-85f7-a94177194136-0_20260409030525348.log.1_0-102-176`, `.7787bafe-f674-4382-85f7-a94177194136-0_20260409030527298.log.1_0-116-202`, `.7787bafe-f674-4382-85f7-a94177194136-0_20260409030528554.log.1_0-130-231`. Schema uses a `key` column (string), not `id`. + +--- + +## File structure + +Single file modified throughout (append only): + +``` +crates/core/tests/file_group_reader_tests.rs +└── (NEW append-only sections, in order) + ├── FilterAbResult struct + impl ← Task A.1 + ├── ab_read_with_filter driver ← Task A.1 + ├── 3 fixture locators ← Task A.1 + │ sf_file_group, nonpart_3commits_file_group, log_only_file_group + ├── 2 schema-specific extractors ← Tasks C.1 / D.1 + │ extract_row_with_id_opt_v9nonpart, extract_row_with_id_opt_log_only + ├── Test 1: fg_filter_in_log_updated_key ← Task A.1 (renamed from existing fg_reader_with_key_filter_filters_rows; refactored to use AB helper) + ├── Test 2: fg_filter_in_base_only_key ← Task B.1 + ├── Test 3: fg_filter_in_no_match ← Task B.2 + ├── Test 4: fg_filter_starts_with_any_prefix ← Task B.3 + ├── Test 5: fg_filter_unsupported_predicate_is_noop ← Task B.4 + ├── Test 6: fg_filter_in_with_delete_block ← Task C.1 + ├── Test 7: fg_filter_in_log_updated_key_nonpart ← Task C.2 + └── Test 8: fg_filter_in_log_only_filegroup ← Task D.1 +``` + +--- + +## Phase A — Shared helpers + refactor existing test + +### Task A.1: Add FilterAbResult, ab_read_with_filter, fixture locators; refactor existing test + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append helpers + locators; rename and refactor the existing keyFilterOpt test) + +This task replaces the existing `fg_reader_with_key_filter_filters_rows` test with `fg_filter_in_log_updated_key` that uses the new `FilterAbResult` helper. The behavior tested is identical; only the structure changes. + +- [ ] **Step 1: Append helpers + locators** at the bottom of `crates/core/tests/file_group_reader_tests.rs`. + +```rust +// ============================================================================= +// keyFilterOpt e2e coverage — shared helpers (FilterAbResult + ab_read_with_filter) +// +// Each test in this section ABs a no-filter read against a filtered read of +// the same file group, then asserts via FilterAbResult that the filter is +// real (or, for unsupported predicates, that it's a no-op). +// ============================================================================= + +/// Result of an AB read: same file group read once with no filter and once +/// with `Some(filter)`. +struct FilterAbResult { + baseline: arrow_array::RecordBatch, + filtered: arrow_array::RecordBatch, +} + +impl FilterAbResult { + /// Assert filter is real, not a no-op: filtered must have STRICTLY fewer + /// rows than baseline. + fn assert_filter_narrowed(&self) { + assert!( + self.filtered.num_rows() < self.baseline.num_rows(), + "filter did not narrow rows: baseline={}, filtered={} \ + (this means the filter was a no-op — bug)", + self.baseline.num_rows(), + self.filtered.num_rows(), + ); + } + + /// Assert filter is a no-op: filtered.num_rows == baseline.num_rows AND + /// every row in filtered also appears in baseline (compared by sorted + /// id-only since row order may differ between two reads). + fn assert_filter_was_noop(&self) { + assert_eq!( + self.filtered.num_rows(), + self.baseline.num_rows(), + "expected filter to be no-op (predicate not In/StringStartsWithAny), \ + but row counts differ: baseline={}, filtered={}", + self.baseline.num_rows(), + self.filtered.num_rows(), + ); + let baseline_ids: std::collections::BTreeSet = ids_in_batch(&self.baseline); + let filtered_ids: std::collections::BTreeSet = ids_in_batch(&self.filtered); + assert_eq!( + baseline_ids, filtered_ids, + "expected same id set after no-op filter" + ); + } + + /// Assert that filtered.num_rows == 0 AND baseline has rows (filter + /// narrowed everything away). + fn assert_filtered_empty(&self) { + assert!( + self.baseline.num_rows() > 0, + "baseline must have rows for the empty-filter assertion to be meaningful" + ); + assert_eq!( + self.filtered.num_rows(), + 0, + "expected filter to drop all rows, got {}", + self.filtered.num_rows() + ); + } + + /// Assert the exact set of `id`s present in filtered (sorted). + fn assert_filtered_ids_eq(&self, expected: &[i32]) { + let actual: Vec = ids_in_batch(&self.filtered).into_iter().collect(); + let mut expected_sorted = expected.to_vec(); + expected_sorted.sort(); + assert_eq!(actual, expected_sorted, "filtered id set mismatch"); + } +} + +/// Helper: collect `id` column values from a RecordBatch as a sorted set. +fn ids_in_batch(batch: &arrow_array::RecordBatch) -> std::collections::BTreeSet { + let id_col = batch + .column_by_name("id") + .expect("id column missing from batch") + .as_any() + .downcast_ref::() + .expect("id column should be Int32"); + (0..batch.num_rows()).map(|i| id_col.value(i)).collect() +} + +/// Shared driver: read once with None, once with Some(filter), return both. +async fn ab_read_with_filter( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + filter: StdArc, +) -> Result { + let baseline = read_file_group_with_key_filter( + table_path, + partition, + base_file, + log_files.clone(), + None, + ) + .await?; + let filtered = read_file_group_with_key_filter( + table_path, + partition, + base_file, + log_files, + Some(filter), + ) + .await?; + Ok(FilterAbResult { baseline, filtered }) +} + +// ============================================================================= +// Fixture locators +// ============================================================================= + +/// Fixture locator: V9Mor8I4UCommitTime city=sf file group. +/// Same filenames as test_e2e_v9_mor_commit_time_sf_merge. +fn sf_file_group() -> (String, &'static str, &'static str, Vec<&'static str>) { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + ( + table_path, + "city=sf", + "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet", + vec![".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73"], + ) +} + +/// Fixture locator: V9MorNonpart3Commits — non-partitioned, single file group, +/// 1 base + 2 log files (delete block + update block). +/// Schema: id INT, name STRING, price DOUBLE, ts LONG. +fn nonpart_3commits_file_group() -> (String, &'static str, &'static str, Vec<&'static str>) { + let table_path = QuickstartTripsTable::V9MorNonpart3Commits.path_to_mor_avro(); + ( + table_path, + "", + "960a29a0-0f78-401d-85b1-1cbc44b34121-0_0-846-1597_20260409002001492.parquet", + vec![ + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002002957.log.1_0-868-1644", + ".960a29a0-0f78-401d-85b1-1cbc44b34121-0_20260409002003963.log.1_0-890-1691", + ], + ) +} + +/// Fixture locator: MorLayoutLogOnly — no base file, 3 log files. +fn log_only_file_group() -> (String, &'static str, &'static str, Vec<&'static str>) { + let table_path = QuickstartTripsTable::MorLayoutLogOnly.path_to_mor_avro(); + ( + table_path, + "", + "", + vec![ + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030525348.log.1_0-102-176", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030527298.log.1_0-116-202", + ".7787bafe-f674-4382-85f7-a94177194136-0_20260409030528554.log.1_0-130-231", + ], + ) +} +``` + +- [ ] **Step 2: Replace the existing test** `fg_reader_with_key_filter_filters_rows`. + +Find the existing test (around the section labeled "Phase 3 e2e smoke test — keyFilterOpt actually filters rows"). Replace its body with the AB-helper version below. Keep the `#[tokio::test]` attribute; rename the function to `fg_filter_in_log_updated_key`. + +```rust +/// Test 1: filter on a key updated in log → 1 row, value from log. +/// +/// Validates: filter applies during BOTH log scan and base read. id=1 is +/// updated in log → filtered output should have the log value (Alice-V2). +#[tokio::test] +async fn fg_filter_in_log_updated_key() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + // First read no-filter to discover id=1's record key dynamically. + let baseline_for_key_lookup = read_file_group_with_key_filter( + &table_path, partition, base_file, log_files.clone(), None, + ).await?; + let key_for_id1 = lookup_record_key(&baseline_for_key_lookup, 1); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id1)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + ab.assert_filtered_ids_eq(&[1]); // exactly id=1 + + // Cross-validate: filtered id=1 row equals baseline id=1 row. + let expected = extract_row_with_id_opt(&ab.baseline, 1).expect("id=1 in baseline"); + let actual = extract_row_with_id_opt(&ab.filtered, 1).expect("id=1 in filtered"); + assert_eq!(expected, actual, "filtered id=1 must equal baseline id=1 (Alice-V2)"); + + Ok(()) +} +``` + +- [ ] **Step 3: Run the renamed test + the two existing keyFilterOpt-related references** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_in_log_updated_key 2>&1 | tail -10` +Expected: 1 test passes. + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests 2>&1 | tail -10` +Expected: all integration tests pass; the renamed test runs and passes; no compile errors. + +- [ ] **Step 4: Commit** + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add FilterAbResult helper + refactor keyFilterOpt e2e test 1 + +Adds: +- FilterAbResult struct + 4 assertion methods (narrowed/noop/empty/ids_eq) +- ab_read_with_filter shared driver +- 3 fixture locators: sf_file_group, nonpart_3commits_file_group, log_only_file_group +- ids_in_batch helper + +Renames fg_reader_with_key_filter_filters_rows → fg_filter_in_log_updated_key, +refactored to use the AB helper. Behavior unchanged. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Phase B — Tests 2-5 on V9Mor8I4UCommitTime + +### Task B.1: Test 2 — `fg_filter_in_base_only_key` + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append one test) + +Filter on `id=2`'s key. id=2 has no log update for `city=sf` (only id=1 is updated). The log scan finds no records matching this key, so the base-file filter is the path doing the work. Validates the base-file filter (`apply_key_filter_to_batch`) in isolation. + +- [ ] **Step 1: Append the test** at the bottom of the file. + +```rust +/// Test 2: filter on a key that exists ONLY in the base file (no log update). +/// +/// Validates: ISOLATES the base-file filter (apply_key_filter_to_batch). +/// id=2 in city=sf has no log update — log scan finds no records for it, +/// only the base-file filter narrows the output. +#[tokio::test] +async fn fg_filter_in_base_only_key() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + let baseline_for_key_lookup = read_file_group_with_key_filter( + &table_path, partition, base_file, log_files.clone(), None, + ).await?; + let key_for_id2 = lookup_record_key(&baseline_for_key_lookup, 2); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id2)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + ab.assert_filtered_ids_eq(&[2]); // exactly id=2 + + // Cross-validate: filtered id=2 row equals baseline id=2 row (Bob, base value). + let expected = extract_row_with_id_opt(&ab.baseline, 2).expect("id=2 in baseline"); + let actual = extract_row_with_id_opt(&ab.filtered, 2).expect("id=2 in filtered"); + assert_eq!(expected, actual, "filtered id=2 must equal baseline id=2 (Bob)"); + + Ok(()) +} +``` + +- [ ] **Step 2: Run the test** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_in_base_only_key 2>&1 | tail -10` +Expected: 1 test passes. + +- [ ] **Step 3: Commit** + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_in_base_only_key (isolates base-file filter) + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task B.2: Test 3 — `fg_filter_in_no_match` + +Filter on a record key that doesn't exist in the data. Validates the filter actually drops everything. + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append one test) + +- [ ] **Step 1: Append the test** + +```rust +/// Test 3: filter on a non-existent key → 0 rows. +/// +/// Validates: filter actually drops everything when nothing matches. +/// Catches a bug where filter is silently a no-op. +#[tokio::test] +async fn fg_filter_in_no_match() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string("definitely-not-a-real-record-key")) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filtered_empty(); // baseline > 0, filtered == 0 + + Ok(()) +} +``` + +- [ ] **Step 2: Run + commit** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_in_no_match 2>&1 | tail -10` +Expected: 1 test passes. + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_in_no_match (filter drops everything) + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task B.3: Test 4 — `fg_filter_starts_with_any_prefix` + +Filter on a prefix that matches id=1's record key but not id=2's. Validates the `PrefixKeys` KeySpec path (currently 0 e2e coverage). + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append one test) + +This test derives a useful prefix from the actual id=1 key. Most realistic case: if the keys look like `"id:1"` and `"id:2"`, the prefix `"id:1"` matches exactly id=1. If the keys are simply `"1"` and `"2"`, the prefix `"1"` matches id=1 but not id=2. Either way, taking the **full key string** of id=1 as the prefix is a safe choice — a prefix that equals one specific key matches exactly that key. + +- [ ] **Step 1: Append the test** + +```rust +/// Test 4: filter via StringStartsWithAny → PrefixKeys path. +/// +/// Validates: exercises the PrefixKeys KeySpec branch of create_key_spec +/// (Predicates::StringStartsWithAny → KeySpec::PrefixKeys). Uses id=1's +/// full record-key string as the prefix; this yields a single-row match +/// (a prefix that equals one key matches exactly that key). +#[tokio::test] +async fn fg_filter_starts_with_any_prefix() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + let baseline_for_key_lookup = read_file_group_with_key_filter( + &table_path, partition, base_file, log_files.clone(), None, + ).await?; + let prefix_for_id1 = lookup_record_key(&baseline_for_key_lookup, 1); + + let filter: StdArc = StdArc::new(predicates_factory::starts_with_any( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(prefix_for_id1)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 2 + ab.assert_filtered_ids_eq(&[1]); + + Ok(()) +} +``` + +- [ ] **Step 2: Run + commit** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_starts_with_any_prefix 2>&1 | tail -10` +Expected: 1 test passes. + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_starts_with_any_prefix (PrefixKeys path) + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task B.4: Test 5 — `fg_filter_unsupported_predicate_is_noop` + +Filter with `Predicates::eq` (BinaryComparison). Since `create_key_spec` only matches `In` and `StringStartsWithAny`, this falls through to `None` and the filter must be a no-op (no rows dropped). + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append one test) + +- [ ] **Step 1: Append the test** + +```rust +/// Test 5: unsupported predicate (eq) → filter is a no-op. +/// +/// Validates: regression guard. create_key_spec only handles In and +/// StringStartsWithAny — anything else returns None, which means the +/// filter must NOT drop rows. This catches a bug where an unsupported +/// predicate accidentally drops rows. +#[tokio::test] +async fn fg_filter_unsupported_predicate_is_noop() -> Result<()> { + let (table_path, partition, base_file, log_files) = sf_file_group(); + + // BinaryComparison (eq) is not In/StringStartsWithAny → create_key_spec returns None. + let filter: StdArc = StdArc::new(predicates_factory::eq( + Box::new(NameReference::new("_hoodie_record_key")), + Box::new(Literal::string("anything")), + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_was_noop(); // baseline.num_rows == filtered.num_rows; same id set + + Ok(()) +} +``` + +- [ ] **Step 2: Run + commit** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_unsupported_predicate_is_noop 2>&1 | tail -10` +Expected: 1 test passes. + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_unsupported_predicate_is_noop (regression guard) + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Phase C — Tests 6-7 on V9MorNonpart3Commits + +This fixture has a **different schema**: `id INT, name STRING, price DOUBLE, ts LONG` (no `age`, no `city`). The existing `extract_row_with_id_opt` returns `(id, name, age)` — wrong shape here. Add a fixture-specific extractor. + +The fixture has 7 base rows (ids 0-6); log 1 deletes ids 0/1/2; log 2 updates ids 4/5/6. Baseline merged = 4 rows: `(3,C,30.0)`, `(4,D2,45.0)`, `(5,E2,55.0)`, `(6,F2,65.0)`. + +### Task C.1: Add fixture-specific extractor + Test 6 (`fg_filter_in_with_delete_block`) + +Filter on id=0's record key. id=0 is deleted by the log delete block, so it's not in the baseline. We need to look up id=0's record key BEFORE the log merge — read the base file alone (no log files) to get an unmerged view that still contains id=0. + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append extractor + test) + +- [ ] **Step 1: Append the extractor** + +```rust +/// Schema-specific extractor for V9MorNonpart3Commits. +/// Schema: id INT, name STRING, price DOUBLE, ts LONG. +fn extract_row_with_id_opt_v9nonpart( + batch: &arrow_array::RecordBatch, + id: i32, +) -> Option<(i32, String, f64, i64)> { + let id_col = batch.column_by_name("id")? + .as_any().downcast_ref::()?; + let name_col = batch.column_by_name("name")? + .as_any().downcast_ref::()?; + let price_col = batch.column_by_name("price")? + .as_any().downcast_ref::()?; + let ts_col = batch.column_by_name("ts")? + .as_any().downcast_ref::()?; + + for i in 0..batch.num_rows() { + if id_col.value(i) == id { + return Some(( + id_col.value(i), + name_col.value(i).to_string(), + price_col.value(i), + ts_col.value(i), + )); + } + } + None +} +``` + +- [ ] **Step 2: Append Test 6** + +```rust +/// Test 6: filter on a deleted-by-log key → 0 rows in filtered output. +/// +/// Validates: filter passes through process_delete_block correctly. +/// V9MorNonpart3Commits: id=0 is deleted by log block. To look up id=0's +/// record key, read the base file alone (no logs) — that unmerged view +/// still contains id=0. +#[tokio::test] +async fn fg_filter_in_with_delete_block() -> Result<()> { + let (table_path, partition, base_file, log_files) = nonpart_3commits_file_group(); + + // Read base-only (no log files) to recover id=0's record key. + let base_only = read_file_group_with_key_filter( + &table_path, partition, base_file, vec![], None, + ).await?; + let key_for_id0 = lookup_record_key(&base_only, 0); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id0)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + // Baseline already excludes id=0 (deleted by log). + assert!(extract_row_with_id_opt_v9nonpart(&ab.baseline, 0).is_none(), + "baseline must NOT contain id=0 (deleted by log)"); + + // Filtered must also exclude id=0 (filter applied to deleted key → 0 rows). + assert!(extract_row_with_id_opt_v9nonpart(&ab.filtered, 0).is_none(), + "filtered must NOT contain id=0"); + + // Stronger: filtered should be empty since the only key in the filter is id=0, + // which is deleted, so no rows survive the filter. + ab.assert_filtered_empty(); + + Ok(()) +} +``` + +- [ ] **Step 3: Run + commit** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_in_with_delete_block 2>&1 | tail -15` +Expected: 1 test passes. + +If FAILS with "id=0 not found in baseline batch" inside `lookup_record_key` — confirm that `read_file_group_with_key_filter(..., vec![], None)` returns the base file unmerged. If `lookup_record_key`'s implementation panics because `_hoodie_record_key` column is missing, replace `key_for_id0 = lookup_record_key(&base_only, 0)` with the literal key value `"0"` (the SimpleKeyGenerator default for an `id` field is the integer-as-string). Re-run and re-validate. + +If FAILS with `assert_filtered_empty` — investigate: maybe id=0 wasn't actually deleted, or the filter didn't apply. Inspect `ab.baseline` and `ab.filtered` row content via `dbg!`. + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_in_with_delete_block (delete-block path) + +extract_row_with_id_opt_v9nonpart helper + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task C.2: Test 7 — `fg_filter_in_log_updated_key_nonpart` + +Filter on id=5 (updated in log). Baseline = 4 rows. Filtered = 1 row, value updated to E2. + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append one test) + +- [ ] **Step 1: Append the test** + +```rust +/// Test 7: filter on a log-updated key on the non-partitioned fixture. +/// +/// Validates: same as Test 1 but on V9MorNonpart3Commits (non-partitioned, +/// 2 log files). id=5 is updated by log 2 → filtered output should reflect +/// the log update (E2/55.0). +#[tokio::test] +async fn fg_filter_in_log_updated_key_nonpart() -> Result<()> { + let (table_path, partition, base_file, log_files) = nonpart_3commits_file_group(); + + let baseline_for_key_lookup = read_file_group_with_key_filter( + &table_path, partition, base_file, log_files.clone(), None, + ).await?; + let key_for_id5 = lookup_record_key(&baseline_for_key_lookup, 5); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id5)) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < 4 + ab.assert_filtered_ids_eq(&[5]); + + // Cross-validate: filtered id=5 row should be the log-updated value (E2, 55.0). + let expected = extract_row_with_id_opt_v9nonpart(&ab.baseline, 5) + .expect("id=5 in baseline"); + let actual = extract_row_with_id_opt_v9nonpart(&ab.filtered, 5) + .expect("id=5 in filtered"); + assert_eq!(expected, actual, "filtered id=5 must equal baseline id=5 (log-updated)"); + // Sanity: name should be "E2" (the log update value). + assert_eq!(expected.1, "E2", "id=5 baseline value should be log update (E2)"); + + Ok(()) +} +``` + +- [ ] **Step 2: Run + commit** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_in_log_updated_key_nonpart 2>&1 | tail -10` +Expected: 1 test passes. + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_in_log_updated_key_nonpart + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Phase D — Test 8 on MorLayoutLogOnly + +This fixture has **no base file** and uses a `key` column (string) instead of `id`. The existing test (`test_e2e_v9_mor_log_only`) compares against a gold parquet and explicitly filters out `_hoodie_*` columns. We need to verify whether `_hoodie_record_key` is in the merged output before writing assertions. + +### Task D.1: Verify schema, add log-only extractor + Test 8 + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (append a probe test, then the real test) + +- [ ] **Step 1: Verify the fixture schema and key column** + +Add a temporary probe test (will be deleted in Step 4 below): + +```rust +/// Temporary probe — DO NOT COMMIT. Used during plan implementation only. +#[tokio::test] +#[ignore = "probe-only — used during impl to inspect fixture schema"] +async fn probe_log_only_schema() -> Result<()> { + let (table_path, partition, base_file, log_files) = log_only_file_group(); + let batch = read_file_group_with_key_filter( + &table_path, partition, base_file, log_files, None, + ).await?; + + eprintln!("MorLayoutLogOnly schema:"); + for f in batch.schema().fields() { + eprintln!(" {} ({:?})", f.name(), f.data_type()); + } + eprintln!("Row count: {}", batch.num_rows()); + if batch.num_rows() > 0 { + // Print first row of each column. + for col_idx in 0..batch.num_columns() { + let name = batch.schema().field(col_idx).name(); + eprintln!(" col '{}'[0] = {:?}", name, batch.column(col_idx).slice(0, 1)); + } + } + Ok(()) +} +``` + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests probe_log_only_schema -- --ignored --nocapture 2>&1 | tail -50` + +Read the printed schema. Note: +- Whether `_hoodie_record_key` is present. +- Whether there's a `key` column or some other unique-id-like column. +- The data type of the `key`-equivalent column. +- The baseline row count. + +- [ ] **Step 2: Append `extract_row_with_id_opt_log_only` based on findings** + +If schema has `key` (String) as the unique-id column, append: + +```rust +/// Schema-specific row lookup for MorLayoutLogOnly. +/// The fixture uses a `key` column (String) as the row identity. +/// Returns the row's (key, _full_row_index) for a given key, or None. +fn extract_row_index_by_key_log_only( + batch: &arrow_array::RecordBatch, + key: &str, +) -> Option { + let key_col = batch.column_by_name("key")? + .as_any().downcast_ref::()?; + for i in 0..batch.num_rows() { + if key_col.value(i) == key { + return Some(i); + } + } + None +} + +/// Variant of ids_in_batch for the log-only fixture's `key` column. +fn keys_in_batch(batch: &arrow_array::RecordBatch) -> std::collections::BTreeSet { + let key_col = batch + .column_by_name("key") + .expect("key column missing") + .as_any() + .downcast_ref::() + .expect("key column should be String"); + (0..batch.num_rows()).map(|i| key_col.value(i).to_string()).collect() +} +``` + +If the schema has a different unique-id column, adapt accordingly — replace `key` with the actual column name throughout. + +- [ ] **Step 3: Append Test 8** + +```rust +/// Test 8: filter on a log-only file group (no base file). +/// +/// Validates: ISOLATES the log-scan filter (KeyBasedFileGroupRecordBuffer +/// process_data_block / process_delete_block). All output flows through +/// the log-scan path because there is no base file. +/// +/// Picks one key from the baseline batch and filters on it; expects the +/// filtered output to contain exactly that key's row (and no others). +#[tokio::test] +async fn fg_filter_in_log_only_filegroup() -> Result<()> { + let (table_path, partition, base_file, log_files) = log_only_file_group(); + + // Read baseline first to discover one valid record key. + let baseline_for_key_lookup = read_file_group_with_key_filter( + &table_path, partition, base_file, log_files.clone(), None, + ).await?; + assert!(baseline_for_key_lookup.num_rows() >= 2, + "log-only baseline must have ≥2 rows for the filter to be a real narrow"); + + // Use the _hoodie_record_key of the first row (or a different identity column + // if the probe revealed _hoodie_record_key is missing — adjust based on Step 1). + let hoodie_key_col = baseline_for_key_lookup + .column_by_name("_hoodie_record_key") + .expect("MorLayoutLogOnly should expose _hoodie_record_key in merged output") + .as_any() + .downcast_ref::() + .expect("_hoodie_record_key should be String"); + let target_key = hoodie_key_col.value(0).to_string(); + + let filter: StdArc = StdArc::new(predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(target_key.clone())) as Box], + )); + + let ab = ab_read_with_filter(&table_path, partition, base_file, log_files, filter).await?; + + ab.assert_filter_narrowed(); // 1 < N + + // Verify exactly the target key remains. + let filtered_keys: Vec = (0..ab.filtered.num_rows()) + .map(|i| { + ab.filtered + .column_by_name("_hoodie_record_key").unwrap() + .as_any().downcast_ref::().unwrap() + .value(i).to_string() + }) + .collect(); + assert_eq!(filtered_keys, vec![target_key.clone()], + "filtered keys should be exactly [target_key]"); + + Ok(()) +} +``` + +If the probe in Step 1 revealed that `_hoodie_record_key` is NOT present in the merged output for this fixture, mark the test `#[ignore = "MorLayoutLogOnly does not expose _hoodie_record_key — TODO: add support or change fixture"]` and document the gap in the commit message. + +- [ ] **Step 4: Remove the temporary probe test** + +Delete the `probe_log_only_schema` test added in Step 1. It is not committed. + +- [ ] **Step 5: Run + commit** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter_in_log_only_filegroup 2>&1 | tail -10` +Expected: 1 test passes (or `ignored` with TODO if the schema doesn't support the assertion). + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +add e2e test fg_filter_in_log_only_filegroup (isolates log-scan filter) + +Reads MorLayoutLogOnly with and without an In filter on +_hoodie_record_key. Since the fixture has no base file, ALL output +flows through KeyBasedFileGroupRecordBuffer's process_data_block / +process_delete_block — this test isolates the log-scan filter. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Final verification + +### Task E.1: Run the full test suite + +**Files:** None modified — verification only. + +- [ ] **Step 1: Run all keyFilterOpt tests** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_filter 2>&1 | tail -15` +Expected: 8 tests pass (or 7 + 1 ignored if Test 8 hit a fixture gap). + +- [ ] **Step 2: Run full hudi-core** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core 2>&1 | tail -10` +Expected: all unit + integration tests pass; no regressions. + +- [ ] **Step 3: Run clippy on the test file** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo clippy -p hudi-core --test file_group_reader_tests 2>&1 | grep -E "(error|warning).*file_group_reader" | head -10` + +Address any new warnings introduced by this work (the existing test file has pre-existing warnings — only fix what this work added). + +- [ ] **Step 4: Optional final cleanup commit** + +If clippy fixes are needed: + +```bash +cd /home/ubuntu/ws3/hudi-rs && git add crates/core/tests/file_group_reader_tests.rs && git commit -m "$(cat <<'EOF' +keyFilterOpt e2e coverage: clippy cleanup + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +If no cleanup needed, no commit. + +--- + +## Test catalog summary (from spec §3 — for cross-reference) + +| # | Test | Fixture | Filter | Expected | Validates | +|---|------|---------|--------|----------|-----------| +| 1 | `fg_filter_in_log_updated_key` | V9Mor8I4UCommitTime sf | In([key_for_id1]) | 1 row, Alice-V2 | log+base together; log-updated path | +| 2 | `fg_filter_in_base_only_key` | V9Mor8I4UCommitTime sf | In([key_for_id2]) | 1 row, Bob | **isolates base-file filter** | +| 3 | `fg_filter_in_no_match` | V9Mor8I4UCommitTime sf | In([nonexistent]) | 0 rows | filter actually drops everything | +| 4 | `fg_filter_starts_with_any_prefix` | V9Mor8I4UCommitTime sf | StringStartsWithAny([key_for_id1]) | 1 row | **PrefixKeys** path | +| 5 | `fg_filter_unsupported_predicate_is_noop` | V9Mor8I4UCommitTime sf | eq(...) | == baseline | regression: unsupported predicate must be no-op | +| 6 | `fg_filter_in_with_delete_block` | V9MorNonpart3Commits | In([key_for_id0]) | 0 rows | **delete-block** path through buffer | +| 7 | `fg_filter_in_log_updated_key_nonpart` | V9MorNonpart3Commits | In([key_for_id5]) | 1 row, E2/55.0 | log update on non-partitioned fixture | +| 8 | `fg_filter_in_log_only_filegroup` | MorLayoutLogOnly | In([first key]) | 1 row | **isolates log-scan filter** (no base file) | + +## Out-of-plan future work (from spec §8) + +- Combined-predicate coverage (`And(In, In)`, `Or(In, In)`) once `create_key_spec` is extended. +- Bootstrap merge filter coverage (currently rejected at construction time). +- COW-table filter coverage when COW write path lands. +- FFI-bridge filter pushdown (out of scope per the original port spec). diff --git a/docs/superpowers/plans/2026-05-07-keyfilteropt-port-implementation.md b/docs/superpowers/plans/2026-05-07-keyfilteropt-port-implementation.md new file mode 100644 index 00000000..848385e2 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-keyfilteropt-port-implementation.md @@ -0,0 +1,3903 @@ +# keyFilterOpt port — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bring hudi-rs's `ReaderContext` to parity with Java's `HoodieReaderContext.keyFilterOpt` by porting `org.apache.hudi.expression` (Predicate hierarchy) plus the `org.apache.hudi.internal.schema.{Type,Types}` subset it depends on, then wiring `key_filter_opt: Option>` through the file-group reader so an `In`-predicate on `_hoodie_record_key` actually filters output rows on a v9 MOR + COMMIT_TIME_ORDERING table. + +**Architecture:** Two new top-level modules: `internal_schema` (Type/Types port) and `expression` (full Predicate/Expression hierarchy with `kind() -> {Predicate,Expression}Kind<'_>` accessors for idiomatic Rust pattern matching). Phase 2 adds a new field on `ReaderContext`, a new `key_spec.rs` file with `create_key_spec()`, and modifies `BaseHoodieLogRecordReader::scan_internal` to accept and apply `Option`. Phase 3 adds one e2e integration test. + +**Tech Stack:** Rust 2024 edition, Tokio async runtime, Arrow `RecordBatch`, existing hudi-rs crates (`hudi_core`, `hudi_test`). + +**Spec reference:** `docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md` + +**Source-of-truth Java files** (read these alongside the plan): +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java` (189 lines) +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java` (955 lines) +- All 14 files under `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/` +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java` (the keyFilterOpt declaration + getter) +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java` (createKeySpec) + +**File structure created/modified:** + +``` +crates/core/src/ +├── lib.rs ← MODIFY: add `pub mod expression;` + `pub mod internal_schema;` +│ +├── internal_schema/ ← NEW +│ ├── mod.rs +│ ├── type_.rs ← Type trait + TypeID enum + PrimitiveType/NestedType markers +│ ├── types.rs ← all concrete types (Boolean, Int, …, Decimal, Record, Array, Map) +│ └── tests.rs ← integration tests for type system +│ +├── expression/ ← NEW +│ ├── mod.rs +│ ├── expression.rs ← Expression trait + ExpressionKind + Operator +│ ├── struct_like.rs ← StructLike trait +│ ├── array_data.rs ← ArrayData +│ ├── leaf_expression.rs ← LeafExpression trait +│ ├── binary_expression.rs ← BinaryExpression +│ ├── literal.rs ← Literal + LiteralValue enum +│ ├── name_reference.rs ← NameReference +│ ├── bound_reference.rs ← BoundReference +│ ├── predicate.rs ← Predicate trait + PredicateKind enum +│ ├── predicates.rs ← Predicates module (12 inner classes + factories) +│ ├── comparators.rs ← Comparators::for_type +│ ├── expression_visitor.rs ← ExpressionVisitor trait +│ ├── bind_visitor.rs ← BindVisitor +│ ├── partial_bind_visitor.rs ← PartialBindVisitor +│ └── tests.rs ← integration tests for expression hierarchy +│ +└── file_group/reader/ + ├── reader_context.rs ← MODIFY: add `key_filter_opt` field + getter + ├── key_spec.rs ← NEW: KeySpec enum + create_key_spec + string_literals + ├── log_record_reader.rs ← MODIFY: scan_internal takes Option; apply filter + ├── merged_log_record_reader.rs ← MODIFY: perform_scan calls create_key_spec + └── mod.rs ← MODIFY: HoodieFileGroupReader::make_base_file_batches applies row filter + +crates/core/tests/ +└── file_group_reader_tests.rs ← MODIFY: add fg_reader_with_key_filter_filters_rows test +``` + +--- + +## Phase 1 — Port the Predicate hierarchy + Type system + +Phase 1 produces a self-contained, fully unit-tested port. **No reader integration in this phase.** The phase ends with one commit and `cargo test -p hudi-core` passing. + +### Task 1.1: Module skeleton + lib.rs re-exports + +**Files:** +- Create: `crates/core/src/internal_schema/mod.rs` +- Create: `crates/core/src/expression/mod.rs` +- Modify: `crates/core/src/lib.rs:49-60` (insert two new module declarations alphabetically) + +- [ ] **Step 1: Create empty `internal_schema/mod.rs`** + +```rust +// crates/core/src/internal_schema/mod.rs +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +//! Mirrors `org.apache.hudi.internal.schema.{Type, Types}`. +//! +//! Schema-evolution code (`InternalSchema`, `AvroInternalSchemaConverter`, +//! schema-change actions/visitors, `SchemaChangeUtils`) is intentionally +//! out of scope — see the keyFilterOpt design spec §2. + +pub mod type_; +pub mod types; + +pub use type_::{Type, TypeID}; +``` + +- [ ] **Step 2: Create empty `expression/mod.rs`** + +```rust +// crates/core/src/expression/mod.rs +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * ... + */ + +//! Mirrors `org.apache.hudi.expression`. +//! +//! Full port of the Predicate / Expression hierarchy. Includes a +//! `kind() -> {Predicate,Expression}Kind<'_>` accessor on each trait +//! to give downstream Rust code idiomatic pattern-matching instead +//! of `Any::downcast_ref`. See the keyFilterOpt design spec §4.1. + +pub mod array_data; +pub mod bind_visitor; +pub mod binary_expression; +pub mod bound_reference; +pub mod comparators; +pub mod expression; +pub mod expression_visitor; +pub mod leaf_expression; +pub mod literal; +pub mod name_reference; +pub mod partial_bind_visitor; +pub mod predicate; +pub mod predicates; +pub mod struct_like; + +pub use expression::{Expression, ExpressionKind, Operator}; +pub use literal::{Literal, LiteralValue}; +pub use name_reference::NameReference; +pub use predicate::{Predicate, PredicateKind}; +``` + +(The submodule files don't exist yet — they will be created in subsequent tasks. The `mod` declarations will fail to compile until each task adds its file. We could stub each one as empty modules first; do that in Step 3.) + +- [ ] **Step 3: Stub all `expression/*.rs` and `internal_schema/*.rs` files as empty placeholders** + +Create each file with just the Apache license header + one-line module doc comment. Example: + +```rust +// crates/core/src/internal_schema/type_.rs +/* … license header … */ + +//! Stub — populated in Task 1.2. +``` + +Files to stub: +- `crates/core/src/internal_schema/type_.rs` +- `crates/core/src/internal_schema/types.rs` +- `crates/core/src/expression/array_data.rs` +- `crates/core/src/expression/bind_visitor.rs` +- `crates/core/src/expression/binary_expression.rs` +- `crates/core/src/expression/bound_reference.rs` +- `crates/core/src/expression/comparators.rs` +- `crates/core/src/expression/expression.rs` +- `crates/core/src/expression/expression_visitor.rs` +- `crates/core/src/expression/leaf_expression.rs` +- `crates/core/src/expression/literal.rs` +- `crates/core/src/expression/name_reference.rs` +- `crates/core/src/expression/partial_bind_visitor.rs` +- `crates/core/src/expression/predicate.rs` +- `crates/core/src/expression/predicates.rs` +- `crates/core/src/expression/struct_like.rs` + +The two `mod.rs` files reference items (`Type`, `TypeID`, `Expression`, etc.) that don't exist yet, so temporarily comment out the `pub use` statements in both `mod.rs` files. They'll be uncommented as the items are introduced in later tasks. + +- [ ] **Step 4: Update `crates/core/src/lib.rs`** + +In `crates/core/src/lib.rs:49-60`, insert the two module declarations alphabetically: + +```rust +pub mod avro_to_arrow; +pub mod config; +pub mod error; +pub mod expr; +pub mod expression; // ← NEW +pub mod file_group; +pub mod hfile; +pub mod internal_schema; // ← NEW +pub mod keygen; +pub mod merge; +pub mod metadata; +mod record; +pub mod schema; +pub mod statistics; +``` + +- [ ] **Step 5: Verify the crate compiles** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo check -p hudi-core 2>&1 | tail -20` +Expected: clean compile (no errors). Possible warnings about unused empty modules are OK. + +- [ ] **Step 6: Commit the skeleton** + +```bash +git add crates/core/src/lib.rs crates/core/src/internal_schema crates/core/src/expression +git commit -m "$(cat <<'EOF' +add empty module skeletons for expression + internal_schema + +Stub modules to be populated in subsequent tasks of the keyFilterOpt +port. See docs/superpowers/plans/2026-05-07-keyfilteropt-port-implementation.md + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 1.2: Port `Type` trait + `TypeID` enum + PrimitiveType/NestedType markers + +**Files:** +- Modify: `crates/core/src/internal_schema/type_.rs` +- Modify: `crates/core/src/internal_schema/mod.rs:14` (uncomment `pub use type_::{Type, TypeID};`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java` (189 lines, full file) + +- [ ] **Step 1: Write failing tests** for `crates/core/src/internal_schema/type_.rs` (append at end of file) + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn type_id_get_name_returns_lowercase() { + assert_eq!(TypeID::Int.get_name(), "int"); + assert_eq!(TypeID::TimestampMillis.get_name(), "timestamp_millis"); + assert_eq!(TypeID::LocalTimestampMicros.get_name(), "local_timestamp_micros"); + } + + #[test] + fn type_id_from_value_round_trips() { + for variant in TypeID::all() { + let s = variant.get_name(); + assert_eq!(TypeID::from_value(s).unwrap(), variant); + } + } + + #[test] + fn type_id_from_value_rejects_unknown() { + assert!(TypeID::from_value("nope").is_err()); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib internal_schema::type_ 2>&1 | tail -10` +Expected: FAIL — `TypeID` does not exist. + +- [ ] **Step 3: Implement `Type` trait + `TypeID` enum** + +Replace the file body with: + +```rust +// crates/core/src/internal_schema/type_.rs +/* … license header … */ + +//! Mirrors `org.apache.hudi.internal.schema.Type`. + +use std::fmt::Debug; + +/// Type identifier — mirrors Java's `Type.TypeID` enum. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TypeID { + Record, + Array, + Map, + Fixed, + String, + Binary, + Int, + Long, + Float, + Double, + Date, + Boolean, + Time, + Timestamp, + Decimal, + UUID, + DecimalBytes, + DecimalFixed, + TimeMillis, + TimestampMillis, + LocalTimestampMillis, + LocalTimestampMicros, +} + +impl TypeID { + /// Mirrors Java `TypeID.getName()` — lowercase snake_case of the enum name. + pub fn get_name(&self) -> &'static str { + match self { + TypeID::Record => "record", + TypeID::Array => "array", + TypeID::Map => "map", + TypeID::Fixed => "fixed", + TypeID::String => "string", + TypeID::Binary => "binary", + TypeID::Int => "int", + TypeID::Long => "long", + TypeID::Float => "float", + TypeID::Double => "double", + TypeID::Date => "date", + TypeID::Boolean => "boolean", + TypeID::Time => "time", + TypeID::Timestamp => "timestamp", + TypeID::Decimal => "decimal", + TypeID::UUID => "uuid", + TypeID::DecimalBytes => "decimal_bytes", + TypeID::DecimalFixed => "decimal_fixed", + TypeID::TimeMillis => "time_millis", + TypeID::TimestampMillis => "timestamp_millis", + TypeID::LocalTimestampMillis => "local_timestamp_millis", + TypeID::LocalTimestampMicros => "local_timestamp_micros", + } + } + + /// Mirrors Java `Type.fromValue(String)` — looks up a TypeID by its + /// uppercase Java enum name (case-insensitive). Returns Err for unknown. + pub fn from_value(value: &str) -> Result { + match value.to_ascii_uppercase().as_str() { + "RECORD" => Ok(TypeID::Record), + "ARRAY" => Ok(TypeID::Array), + "MAP" => Ok(TypeID::Map), + "FIXED" => Ok(TypeID::Fixed), + "STRING" => Ok(TypeID::String), + "BINARY" => Ok(TypeID::Binary), + "INT" => Ok(TypeID::Int), + "LONG" => Ok(TypeID::Long), + "FLOAT" => Ok(TypeID::Float), + "DOUBLE" => Ok(TypeID::Double), + "DATE" => Ok(TypeID::Date), + "BOOLEAN" => Ok(TypeID::Boolean), + "TIME" => Ok(TypeID::Time), + "TIMESTAMP" => Ok(TypeID::Timestamp), + "DECIMAL" => Ok(TypeID::Decimal), + "UUID" => Ok(TypeID::UUID), + "DECIMAL_BYTES" => Ok(TypeID::DecimalBytes), + "DECIMAL_FIXED" => Ok(TypeID::DecimalFixed), + "TIME_MILLIS" => Ok(TypeID::TimeMillis), + "TIMESTAMP_MILLIS" => Ok(TypeID::TimestampMillis), + "LOCAL_TIMESTAMP_MILLIS" => Ok(TypeID::LocalTimestampMillis), + "LOCAL_TIMESTAMP_MICROS" => Ok(TypeID::LocalTimestampMicros), + _ => Err(format!("Invalid value of Type: {value}")), + } + } + + /// Test helper — list all variants for round-trip testing. + #[cfg(test)] + pub fn all() -> Vec { + vec![ + TypeID::Record, TypeID::Array, TypeID::Map, TypeID::Fixed, + TypeID::String, TypeID::Binary, TypeID::Int, TypeID::Long, + TypeID::Float, TypeID::Double, TypeID::Date, TypeID::Boolean, + TypeID::Time, TypeID::Timestamp, TypeID::Decimal, TypeID::UUID, + TypeID::DecimalBytes, TypeID::DecimalFixed, TypeID::TimeMillis, + TypeID::TimestampMillis, TypeID::LocalTimestampMillis, + TypeID::LocalTimestampMicros, + ] + } +} + +/// Mirrors Java `Type` interface. All concrete types implement this trait. +/// +/// Java has `extends Serializable` — Rust trait objects can derive `Debug` +/// and the concrete impls in `types.rs` derive `Clone, PartialEq, Eq, Hash`. +pub trait Type: Debug + Send + Sync { + /// Mirrors Java `Type.typeId()`. + fn type_id(&self) -> TypeID; + + /// Mirrors Java default `Type.isNestedType()`. Default false; nested + /// types override. + fn is_nested_type(&self) -> bool { + false + } +} +``` + +- [ ] **Step 4: Uncomment the re-export in `internal_schema/mod.rs`** + +Edit `crates/core/src/internal_schema/mod.rs` to uncomment: + +```rust +pub use type_::{Type, TypeID}; +``` + +- [ ] **Step 5: Run tests and verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib internal_schema::type_ 2>&1 | tail -10` +Expected: 3 tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/internal_schema/type_.rs crates/core/src/internal_schema/mod.rs +git commit -m "port Type trait + TypeID enum from internal.schema.Type.java + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.3: Port primitive types from `Types.java` + +**Files:** +- Modify: `crates/core/src/internal_schema/types.rs` + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java` lines 36-560 (covering BooleanType through UUIDType — i.e., all classes that extend `PrimitiveType`). + +This task ports the 17 primitive types: BooleanType, IntType, LongType, FloatType, DoubleType, DateType, TimeType, TimestampType, TimeMillisType, TimestampMillisType, LocalTimestampMillisType, LocalTimestampMicrosType, StringType, BinaryType, FixedType, DecimalType, UUIDType. + +- [ ] **Step 1: Write failing tests** at the bottom of `crates/core/src/internal_schema/types.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::internal_schema::TypeID; + + #[test] + fn primitive_type_ids() { + assert_eq!(BooleanType::get().type_id(), TypeID::Boolean); + assert_eq!(IntType::get().type_id(), TypeID::Int); + assert_eq!(LongType::get().type_id(), TypeID::Long); + assert_eq!(FloatType::get().type_id(), TypeID::Float); + assert_eq!(DoubleType::get().type_id(), TypeID::Double); + assert_eq!(DateType::get().type_id(), TypeID::Date); + assert_eq!(TimeType::get().type_id(), TypeID::Time); + assert_eq!(TimestampType::get().type_id(), TypeID::Timestamp); + assert_eq!(TimeMillisType::get().type_id(), TypeID::TimeMillis); + assert_eq!(TimestampMillisType::get().type_id(), TypeID::TimestampMillis); + assert_eq!(LocalTimestampMillisType::get().type_id(), TypeID::LocalTimestampMillis); + assert_eq!(LocalTimestampMicrosType::get().type_id(), TypeID::LocalTimestampMicros); + assert_eq!(StringType::get().type_id(), TypeID::String); + assert_eq!(BinaryType::get().type_id(), TypeID::Binary); + assert_eq!(UUIDType::get().type_id(), TypeID::UUID); + } + + #[test] + fn primitive_type_to_string() { + assert_eq!(BooleanType::get().to_string(), "boolean"); + assert_eq!(IntType::get().to_string(), "int"); + assert_eq!(StringType::get().to_string(), "string"); + } + + #[test] + fn primitive_types_are_not_nested() { + assert!(!BooleanType::get().is_nested_type()); + assert!(!IntType::get().is_nested_type()); + assert!(!StringType::get().is_nested_type()); + } + + #[test] + fn primitive_types_equal_their_singletons() { + assert_eq!(IntType::get(), IntType::get()); + // Different types are never equal even if both are PrimitiveType. + // (PrimitiveType equality in Java compares typeId only.) + } + + #[test] + fn fixed_type_carries_size() { + let f = FixedType::new(16); + assert_eq!(f.type_id(), TypeID::Fixed); + assert_eq!(f.size(), 16); + assert_eq!(f.to_string(), "fixed(16)"); + } + + #[test] + fn decimal_type_carries_precision_and_scale() { + let d = DecimalType::new(10, 2); + assert_eq!(d.type_id(), TypeID::Decimal); + assert_eq!(d.precision(), 10); + assert_eq!(d.scale(), 2); + assert_eq!(d.to_string(), "decimal(10, 2)"); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure (none of the types exist yet)** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib internal_schema::types 2>&1 | tail -20` +Expected: FAIL — `BooleanType`, `IntType`, etc. do not exist. + +- [ ] **Step 3: Implement all 17 primitive types** + +Replace `crates/core/src/internal_schema/types.rs` body with: + +```rust +// crates/core/src/internal_schema/types.rs +/* … license header … */ + +//! Mirrors `org.apache.hudi.internal.schema.Types`. + +use crate::internal_schema::{Type, TypeID}; +use std::fmt; + +// ========================================================================= +// Singleton primitive types — all map to a single `Self` instance via `get()`. +// ========================================================================= + +macro_rules! singleton_primitive { + ($name:ident, $id:ident, $disp:literal) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct $name; + + impl $name { + pub fn get() -> Self { $name } + } + + impl Type for $name { + fn type_id(&self) -> TypeID { TypeID::$id } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str($disp) + } + } + }; +} + +singleton_primitive!(BooleanType, Boolean, "boolean"); +singleton_primitive!(IntType, Int, "int"); +singleton_primitive!(LongType, Long, "long"); +singleton_primitive!(FloatType, Float, "float"); +singleton_primitive!(DoubleType, Double, "double"); +singleton_primitive!(DateType, Date, "date"); +singleton_primitive!(TimeType, Time, "time"); +singleton_primitive!(TimestampType, Timestamp, "timestamp"); +singleton_primitive!(TimeMillisType, TimeMillis, "time_millis"); +singleton_primitive!(TimestampMillisType, TimestampMillis, "timestamp_millis"); +singleton_primitive!(LocalTimestampMillisType, LocalTimestampMillis, "local_timestamp_millis"); +singleton_primitive!(LocalTimestampMicrosType, LocalTimestampMicros, "local_timestamp_micros"); +singleton_primitive!(StringType, String, "string"); +singleton_primitive!(BinaryType, Binary, "binary"); +singleton_primitive!(UUIDType, UUID, "uuid"); + +// ========================================================================= +// Parameterized primitive types +// ========================================================================= + +/// Mirrors Java `Types.FixedType(int size)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FixedType { + size: u32, +} + +impl FixedType { + pub fn new(size: u32) -> Self { Self { size } } + pub fn size(&self) -> u32 { self.size } +} + +impl Type for FixedType { + fn type_id(&self) -> TypeID { TypeID::Fixed } +} + +impl fmt::Display for FixedType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "fixed({})", self.size) + } +} + +/// Mirrors Java `Types.DecimalType(int precision, int scale)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DecimalType { + precision: u8, + scale: u8, +} + +impl DecimalType { + pub fn new(precision: u8, scale: u8) -> Self { Self { precision, scale } } + pub fn precision(&self) -> u8 { self.precision } + pub fn scale(&self) -> u8 { self.scale } +} + +impl Type for DecimalType { + fn type_id(&self) -> TypeID { TypeID::Decimal } +} + +impl fmt::Display for DecimalType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "decimal({}, {})", self.precision, self.scale) + } +} +``` + +- [ ] **Step 4: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib internal_schema::types 2>&1 | tail -10` +Expected: 6 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/core/src/internal_schema/types.rs +git commit -m "port primitive Types from internal.schema.Types.java + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.4: Port nested types (Field, RecordType, ArrayType, MapType) + +**Files:** +- Modify: `crates/core/src/internal_schema/types.rs` (append nested types after primitive types) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java` lines 562-955 (Field, RecordType, ArrayType, MapType inner classes). + +- [ ] **Step 1: Write failing tests** (append to existing `tests` mod in `types.rs`) + +```rust + #[test] + fn field_carries_id_name_type_nullable() { + let f = Field::new(1, "id", Box::new(IntType::get()), false); + assert_eq!(f.id(), 1); + assert_eq!(f.name(), "id"); + assert!(!f.is_optional()); + assert_eq!(f.field_type().type_id(), TypeID::Int); + } + + #[test] + fn record_type_lookup_field_by_name() { + let fields = vec![ + Field::new(1, "id", Box::new(IntType::get()), false), + Field::new(2, "name", Box::new(StringType::get()), true), + ]; + let rec = RecordType::new(fields); + assert_eq!(rec.type_id(), TypeID::Record); + assert!(rec.is_nested_type()); + assert_eq!(rec.field_by_name("id").unwrap().id(), 1); + assert_eq!(rec.field_by_name("name").unwrap().id(), 2); + assert!(rec.field_by_name("missing").is_none()); + } + + #[test] + fn array_type_carries_element_type() { + let arr = ArrayType::new(2, true, Box::new(IntType::get())); + assert_eq!(arr.type_id(), TypeID::Array); + assert!(arr.is_nested_type()); + assert!(arr.is_element_optional()); + assert_eq!(arr.element_type().type_id(), TypeID::Int); + } + + #[test] + fn map_type_carries_key_value_types() { + let m = MapType::new( + 3, 4, true, + Box::new(StringType::get()), + Box::new(LongType::get()), + ); + assert_eq!(m.type_id(), TypeID::Map); + assert!(m.is_nested_type()); + assert_eq!(m.key_type().type_id(), TypeID::String); + assert_eq!(m.value_type().type_id(), TypeID::Long); + assert!(m.is_value_optional()); + } +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib internal_schema::types 2>&1 | tail -10` +Expected: FAIL — `Field`, `RecordType`, etc. don't exist. + +- [ ] **Step 3: Implement Field + RecordType + ArrayType + MapType** + +Append to `crates/core/src/internal_schema/types.rs` (before the `#[cfg(test)]` mod): + +```rust +// ========================================================================= +// Nested types +// ========================================================================= + +/// Mirrors Java `Types.Field`. +/// +/// Java fields: `int id`, `boolean isOptional`, `String name`, `Type type`, `String doc`. +/// Default Java constructor sets `doc = null`. +#[derive(Debug, Clone)] +pub struct Field { + id: i32, + is_optional: bool, + name: String, + field_type: Box, + doc: Option, +} + +impl Field { + pub fn new(id: i32, name: impl Into, field_type: Box, is_optional: bool) -> Self { + Self { id, name: name.into(), field_type, is_optional, doc: None } + } + + pub fn with_doc(mut self, doc: impl Into) -> Self { + self.doc = Some(doc.into()); + self + } + + pub fn id(&self) -> i32 { self.id } + pub fn name(&self) -> &str { &self.name } + pub fn field_type(&self) -> &dyn Type { self.field_type.as_ref() } + pub fn is_optional(&self) -> bool { self.is_optional } + pub fn doc(&self) -> Option<&str> { self.doc.as_deref() } +} + +impl PartialEq for Field { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.is_optional == other.is_optional + && self.name == other.name + && self.field_type.type_id() == other.field_type.type_id() + && self.doc == other.doc + } +} + +impl Eq for Field {} + +/// Mirrors Java `Types.RecordType`. +#[derive(Debug, Clone)] +pub struct RecordType { + fields: Vec, +} + +impl RecordType { + pub fn new(fields: Vec) -> Self { + Self { fields } + } + + pub fn fields(&self) -> &[Field] { &self.fields } + + /// Mirrors Java `RecordType.fieldType(String name)`. + pub fn field_type_by_name(&self, name: &str) -> Option<&dyn Type> { + self.fields.iter().find(|f| f.name == name).map(|f| f.field_type()) + } + + /// Mirrors Java `RecordType.field(int id)` — find by field id. + pub fn field_by_id(&self, id: i32) -> Option<&Field> { + self.fields.iter().find(|f| f.id == id) + } + + /// Find a field by name (Rust convenience accessor). + pub fn field_by_name(&self, name: &str) -> Option<&Field> { + self.fields.iter().find(|f| f.name == name) + } +} + +impl Type for RecordType { + fn type_id(&self) -> TypeID { TypeID::Record } + fn is_nested_type(&self) -> bool { true } +} + +impl fmt::Display for RecordType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "record<{}>", self.fields.len()) + } +} + +/// Mirrors Java `Types.ArrayType`. +#[derive(Debug, Clone)] +pub struct ArrayType { + element_id: i32, + element_optional: bool, + element_type: Box, +} + +impl ArrayType { + pub fn new(element_id: i32, element_optional: bool, element_type: Box) -> Self { + Self { element_id, element_optional, element_type } + } + + pub fn element_id(&self) -> i32 { self.element_id } + pub fn is_element_optional(&self) -> bool { self.element_optional } + pub fn element_type(&self) -> &dyn Type { self.element_type.as_ref() } +} + +impl Type for ArrayType { + fn type_id(&self) -> TypeID { TypeID::Array } + fn is_nested_type(&self) -> bool { true } +} + +impl fmt::Display for ArrayType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "array<{}>", self.element_type.type_id().get_name()) + } +} + +/// Mirrors Java `Types.MapType`. +#[derive(Debug, Clone)] +pub struct MapType { + key_id: i32, + value_id: i32, + value_optional: bool, + key_type: Box, + value_type: Box, +} + +impl MapType { + pub fn new( + key_id: i32, + value_id: i32, + value_optional: bool, + key_type: Box, + value_type: Box, + ) -> Self { + Self { key_id, value_id, value_optional, key_type, value_type } + } + + pub fn key_id(&self) -> i32 { self.key_id } + pub fn value_id(&self) -> i32 { self.value_id } + pub fn is_value_optional(&self) -> bool { self.value_optional } + pub fn key_type(&self) -> &dyn Type { self.key_type.as_ref() } + pub fn value_type(&self) -> &dyn Type { self.value_type.as_ref() } +} + +impl Type for MapType { + fn type_id(&self) -> TypeID { TypeID::Map } + fn is_nested_type(&self) -> bool { true } +} + +impl fmt::Display for MapType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "map<{},{}>", self.key_type.type_id().get_name(), self.value_type.type_id().get_name()) + } +} +``` + +- [ ] **Step 4: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib internal_schema::types 2>&1 | tail -10` +Expected: 10 tests pass (6 from Task 1.3 + 4 new). + +- [ ] **Step 5: Commit** + +```bash +git add crates/core/src/internal_schema/types.rs +git commit -m "port nested Types (Field, RecordType, ArrayType, MapType) + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.5: Port `StructLike` trait + `ArrayData` + +**Files:** +- Modify: `crates/core/src/expression/struct_like.rs` +- Modify: `crates/core/src/expression/array_data.rs` + +**Java sources:** +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/StructLike.java` (26 lines, full) +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/ArrayData.java` (40 lines, full) + +- [ ] **Step 1: Write failing tests** at the bottom of `struct_like.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + + struct MockStruct(Vec>>); + + impl StructLike for MockStruct { + fn num_fields(&self) -> usize { self.0.len() } + fn get(&self, pos: usize) -> Option<&dyn std::any::Any> { + self.0[pos].as_deref() + } + } + + #[test] + fn struct_like_basic() { + let s = MockStruct(vec![ + Some(Box::new(42i32)), + Some(Box::new("hello".to_string())), + None, + ]); + assert_eq!(s.num_fields(), 3); + assert!(s.get(2).is_none()); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::struct_like 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `StructLike`** + +Replace `crates/core/src/expression/struct_like.rs` body: + +```rust +// crates/core/src/expression/struct_like.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.StructLike`. + +use std::any::Any; + +/// Read-only view over a record-like value. Mirrors Java's `StructLike`. +/// +/// Java has `int numFields()` + ` T get(int pos, Class javaClass)`. +/// Rust uses `&dyn Any` so callers can downcast. +pub trait StructLike: Send + Sync { + fn num_fields(&self) -> usize; + fn get(&self, pos: usize) -> Option<&dyn Any>; +} +``` + +- [ ] **Step 4: Implement `ArrayData`** + +Replace `crates/core/src/expression/array_data.rs` body: + +```rust +// crates/core/src/expression/array_data.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.ArrayData`. + +use std::any::Any; + +/// Read-only view over an array-like value. Mirrors Java's `ArrayData`. +pub trait ArrayData: Send + Sync { + fn num_elements(&self) -> usize; + fn get(&self, pos: usize) -> Option<&dyn Any>; +} +``` + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::struct_like 2>&1 | tail -10` +Expected: 1 test passes. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression/struct_like.rs crates/core/src/expression/array_data.rs +git commit -m "port StructLike + ArrayData traits + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.6: Port `Operator` enum + `Expression` trait + `ExpressionKind` + +**Files:** +- Modify: `crates/core/src/expression/expression.rs` +- Modify: `crates/core/src/expression/mod.rs:18-20` (uncomment `pub use expression::{Expression, ExpressionKind, Operator};`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Expression.java` (69 lines, full) + +This is the most central piece of Phase 1. It defines: +1. `Operator` enum mirroring Java `Expression.Operator`. +2. `Expression` trait with the `kind()` accessor (deviation #2 from spec §6). +3. `ExpressionKind<'a>` borrowed enum view. + +The concrete variants `Literal`, `NameReference`, `BoundReference`, and `Predicate` referenced in `ExpressionKind` don't exist yet — we forward-declare with `#[allow(dead_code)]` or use type aliases that resolve when those tasks land. Plan: declare `ExpressionKind` as enum with placeholder unit-variant signatures referring to types defined in later tasks, and use `pub mod` re-exports to make them visible. Alternative: define `ExpressionKind` as a marker trait now and convert to enum later. Easier: define the enum but use `&dyn Expression` for variants we haven't created concrete types for, then refine in subsequent tasks. + +For this task: define `ExpressionKind<'a>` with **only the variants whose target types exist** at this point (none — Literal/NameReference/BoundReference/Predicate are all later). So the initial `ExpressionKind<'a>` is empty-but-extensible — we'll add variants in Tasks 1.7, 1.8, 1.9, 1.11. Use a single `Other(&'a dyn Expression)` placeholder variant that is removed once all real variants exist. + +Actually, simpler: define `ExpressionKind<'a>` as enum **with all 4 variants up-front** but each variant references a concrete struct that we'll build in next tasks. Since the structs don't exist, use a **forward-declared module path** that compiles only after those modules are populated. Concretely: temporarily comment out the variants, add them as each subsequent task introduces the type. + +Cleanest approach: define `ExpressionKind` minimally now (no variants), grow it in subsequent tasks. Each later task adds: +- the concrete struct, +- the `ExpressionKind` variant pointing at it, +- the `kind()` impl returning that variant. + +Implement that pattern. + +- [ ] **Step 1: Write failing tests** at the bottom of `expression.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn operator_symbol_matches_java() { + assert_eq!(Operator::And.symbol(), "AND"); + assert_eq!(Operator::Or.symbol(), "OR"); + assert_eq!(Operator::Not.symbol(), "NOT"); + assert_eq!(Operator::Eq.symbol(), "="); + assert_eq!(Operator::Gt.symbol(), ">"); + assert_eq!(Operator::Lt.symbol(), "<"); + assert_eq!(Operator::GtEq.symbol(), ">="); + assert_eq!(Operator::LtEq.symbol(), "<="); + assert_eq!(Operator::In.symbol(), "IN"); + assert_eq!(Operator::IsNull.symbol(), "IS NULL"); + assert_eq!(Operator::IsNotNull.symbol(), "IS NOT NULL"); + assert_eq!(Operator::StartsWith.symbol(), "starts_with"); + assert_eq!(Operator::Contains.symbol(), "contains"); + assert_eq!(Operator::True.symbol(), "TRUE"); + assert_eq!(Operator::False.symbol(), "FALSE"); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::expression 2>&1 | tail -10` +Expected: FAIL — `Operator` does not exist. + +- [ ] **Step 3: Implement `Operator` + `Expression` + `ExpressionKind`** + +Replace `crates/core/src/expression/expression.rs` body: + +```rust +// crates/core/src/expression/expression.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.Expression`. +//! +//! Defines the `Expression` trait and the `Operator` enum. Adds an +//! `ExpressionKind<'_>` accessor for idiomatic Rust pattern matching +//! (see keyFilterOpt design spec §4.1, deviation #2). + +use crate::expression::struct_like::StructLike; +use crate::internal_schema::Type; +use std::any::Any; +use std::fmt::Debug; + +/// Mirrors Java `Expression.Operator`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Operator { + And, + Or, + Not, + Eq, + Gt, + Lt, + GtEq, + LtEq, + In, + IsNull, + IsNotNull, + StartsWith, + Contains, + True, + False, +} + +impl Operator { + /// Mirrors Java `Operator.symbol`. + pub fn symbol(&self) -> &'static str { + match self { + Operator::And => "AND", + Operator::Or => "OR", + Operator::Not => "NOT", + Operator::Eq => "=", + Operator::Gt => ">", + Operator::Lt => "<", + Operator::GtEq => ">=", + Operator::LtEq => "<=", + Operator::In => "IN", + Operator::IsNull => "IS NULL", + Operator::IsNotNull => "IS NOT NULL", + Operator::StartsWith => "starts_with", + Operator::Contains => "contains", + Operator::True => "TRUE", + Operator::False => "FALSE", + } + } +} + +/// Mirrors Java `org.apache.hudi.expression.Expression`. +/// +/// Trait must be `Send + Sync + Debug` so trait objects can be `Arc`'d +/// across the reader pipeline. +pub trait Expression: Debug + Send + Sync + Any { + /// Mirrors Java `Expression.getDataType()`. + fn data_type(&self) -> &dyn Type; + + /// Mirrors Java `Expression.getChildren()`. + fn children(&self) -> Vec<&dyn Expression> { Vec::new() } + + /// Mirrors Java `Expression.getOperator()`. + fn operator(&self) -> Operator; + + /// Mirrors Java `Expression.eval(StructLike data)`. Default is `None` + /// — concrete types override. + /// + /// Returns `Box` so callers can downcast to the expected type. + /// Not used on the keyFilterOpt reader path; ported for unit-test parity. + fn eval(&self, _data: Option<&dyn StructLike>) -> Option> { + None + } + + /// Borrowed enum view for idiomatic Rust pattern matching. + /// + /// Variants are added incrementally as each concrete Expression type is + /// introduced in subsequent tasks (Literal — Task 1.8, NameReference — + /// Task 1.9, BoundReference — Task 1.10, Predicate — Task 1.11). + fn kind(&self) -> ExpressionKind<'_>; +} + +/// Borrowed enum view of `Expression` for pattern matching. Variants are +/// added as concrete types come online — see comment on `Expression::kind`. +pub enum ExpressionKind<'a> { + // Variants added by subsequent tasks: + // Literal(&'a Literal), (Task 1.8) + // NameReference(&'a NameReference), (Task 1.9) + // BoundReference(&'a BoundReference), (Task 1.10) + // Predicate(&'a dyn crate::expression::Predicate), (Task 1.11) + /// Placeholder so the empty enum compiles. Removed in Task 1.11. + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +- [ ] **Step 4: Uncomment the re-export in `expression/mod.rs`** + +```rust +pub use expression::{Expression, ExpressionKind, Operator}; +``` + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::expression 2>&1 | tail -10` +Expected: 1 test passes. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression/expression.rs crates/core/src/expression/mod.rs +git commit -m "port Expression trait + Operator enum + empty ExpressionKind + +ExpressionKind variants land in subsequent tasks as concrete types are +introduced (Literal, NameReference, BoundReference, Predicate). + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.7: Port `LeafExpression` and `BinaryExpression` + +**Files:** +- Modify: `crates/core/src/expression/leaf_expression.rs` +- Modify: `crates/core/src/expression/binary_expression.rs` + +**Java sources:** +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/LeafExpression.java` (32 lines, full) +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/BinaryExpression.java` (60 lines, full) + +These are abstract base classes in Java. In Rust they become traits. Concrete subclasses (Literal, NameReference for LeafExpression; And, Or, BinaryComparison for BinaryExpression) implement them in later tasks. + +- [ ] **Step 1: Implement `LeafExpression`** (no separate test — exercised by concrete subclasses in later tasks) + +Replace `crates/core/src/expression/leaf_expression.rs`: + +```rust +// crates/core/src/expression/leaf_expression.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.LeafExpression`. + +use crate::expression::Expression; + +/// Marker trait for expressions with no children. Mirrors Java's +/// `LeafExpression` abstract class. +pub trait LeafExpression: Expression {} +``` + +- [ ] **Step 2: Implement `BinaryExpression`** + +Replace `crates/core/src/expression/binary_expression.rs`: + +```rust +// crates/core/src/expression/binary_expression.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.BinaryExpression`. + +use crate::expression::{Expression, Operator}; + +/// Mirrors Java `BinaryExpression`. Fields: left, operator, right. +/// +/// Java has `protected final` fields and getters. Rust struct mirrors +/// that with public accessors. +#[derive(Debug)] +pub struct BinaryExpression { + pub left: Box, + pub operator: Operator, + pub right: Box, +} + +impl BinaryExpression { + pub fn new(left: Box, operator: Operator, right: Box) -> Self { + Self { left, operator, right } + } + + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right(&self) -> &dyn Expression { self.right.as_ref() } + pub fn operator(&self) -> Operator { self.operator } +} +``` + +(Note: `BinaryExpression` is a *helper* struct that concrete predicate impls compose, not a trait. This deviates slightly from Java where `BinaryExpression` is an abstract class — in Rust we can't naturally express "concrete struct with virtual methods" without trait objects. The composition approach keeps the data shape identical.) + +- [ ] **Step 3: Run tests; verify still pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression 2>&1 | tail -10` +Expected: previous tests still pass; nothing new since these have no behavior of their own. + +- [ ] **Step 4: Commit** + +```bash +git add crates/core/src/expression/leaf_expression.rs crates/core/src/expression/binary_expression.rs +git commit -m "port LeafExpression trait + BinaryExpression helper struct + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.8: Port `LiteralValue` enum + `Literal` struct + add `ExpressionKind::Literal` + +**Files:** +- Modify: `crates/core/src/expression/literal.rs` +- Modify: `crates/core/src/expression/expression.rs` (add `Literal` variant to `ExpressionKind`) +- Modify: `crates/core/src/expression/mod.rs` (uncomment `pub use literal::{Literal, LiteralValue};`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Literal.java` (122 lines, full) + +This task introduces the **first concrete Expression variant**. After this task, `ExpressionKind::_Placeholder` shrinks (we add `Literal` alongside it). The placeholder gets removed in Task 1.11 once all four variants exist. + +- [ ] **Step 1: Write failing tests** at the bottom of `literal.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, ExpressionKind, Operator}; + use crate::internal_schema::TypeID; + + #[test] + fn literal_string_value_extraction() { + let lit = Literal::string("hello"); + assert_eq!(lit.data_type().type_id(), TypeID::String); + match lit.value() { + LiteralValue::String(s) => assert_eq!(s, "hello"), + _ => panic!("expected String literal"), + } + } + + #[test] + fn literal_int_value_extraction() { + let lit = Literal::int(42); + assert_eq!(lit.data_type().type_id(), TypeID::Int); + match lit.value() { + LiteralValue::Int(n) => assert_eq!(*n, 42), + _ => panic!("expected Int literal"), + } + } + + #[test] + fn literal_kind_pattern_match() { + let lit = Literal::string("k"); + match lit.kind() { + ExpressionKind::Literal(_) => {} // expected + _ => panic!("expected ExpressionKind::Literal"), + } + } + + #[test] + fn literal_operator_is_true() { + // Java's Literal has no specific operator semantics — getOperator() + // returns Operator.TRUE by default since LeafExpression. Rust mirrors. + let lit = Literal::int(1); + assert_eq!(lit.operator(), Operator::True); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::literal 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `LiteralValue` + `Literal`** + +Replace `crates/core/src/expression/literal.rs`: + +```rust +// crates/core/src/expression/literal.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.Literal`. +//! +//! Diverges from Java in one place: Java `Literal` is generic over T; +//! Rust uses a closed `LiteralValue` enum so callers extracting values +//! via `kind()` can pattern-match without a second downcast. See the +//! keyFilterOpt design spec §6 deviation #1. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::types::*; +use crate::internal_schema::Type; + +/// Closed enum of literal-value variants. Mirrors the runtime values +/// Java's `Literal` can carry. Variants are 1:1 with `Types.java` primitives +/// (we store the unscaled+precision+scale tuple for Decimal rather than a +/// `BigDecimal`). +#[derive(Debug, Clone, PartialEq)] +pub enum LiteralValue { + Bool(bool), + Int(i32), + Long(i64), + Float(f32), + Double(f64), + Date(i32), // days since epoch + Time(i64), // micros since midnight + TimeMillis(i32), // millis since midnight + Timestamp(i64), // micros since epoch + TimestampMillis(i64), // millis since epoch + LocalTimestampMillis(i64), + LocalTimestampMicros(i64), + Decimal { unscaled: i128, precision: u8, scale: u8 }, + String(String), + Binary(Vec), + Fixed(Vec), + UUID([u8; 16]), + Null, +} + +/// Mirrors Java `Literal(T value, Type type)` — non-generic in Rust. +#[derive(Debug, Clone)] +pub struct Literal { + value: LiteralValue, + data_type: Box, +} + +impl Literal { + pub fn new(value: LiteralValue, data_type: Box) -> Self { + Self { value, data_type } + } + + /// Mirrors Java `Literal.getValue()` (returns the carried value enum). + pub fn value(&self) -> &LiteralValue { &self.value } + + // Convenience constructors — mirror Java's `Literal.from(value)` static. + pub fn string(s: impl Into) -> Self { + Self::new(LiteralValue::String(s.into()), Box::new(StringType::get())) + } + pub fn int(n: i32) -> Self { + Self::new(LiteralValue::Int(n), Box::new(IntType::get())) + } + pub fn long(n: i64) -> Self { + Self::new(LiteralValue::Long(n), Box::new(LongType::get())) + } + pub fn boolean(b: bool) -> Self { + Self::new(LiteralValue::Bool(b), Box::new(BooleanType::get())) + } + pub fn double(d: f64) -> Self { + Self::new(LiteralValue::Double(d), Box::new(DoubleType::get())) + } + pub fn float(d: f32) -> Self { + Self::new(LiteralValue::Float(d), Box::new(FloatType::get())) + } + pub fn null() -> Self { + Self::new(LiteralValue::Null, Box::new(StringType::get())) + // ^ Java's Literal.from(null) defaults to STRING type. Rust mirrors. + } +} + +impl Expression for Literal { + fn data_type(&self) -> &dyn Type { self.data_type.as_ref() } + + /// Java `Literal.getOperator()` is inherited from LeafExpression which + /// in turn returns `Operator.TRUE` (Java has no specific literal op). + fn operator(&self) -> Operator { Operator::True } + + fn eval(&self, _data: Option<&dyn crate::expression::struct_like::StructLike>) + -> Option> { + // Mirrors Java: `Literal.eval(data)` returns the carried value + // ignoring `data`. We box-clone the LiteralValue. + Some(Box::new(self.value.clone())) + } + + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::Literal(self) + } +} + +impl LeafExpression for Literal {} +``` + +- [ ] **Step 4: Add `Literal` variant to `ExpressionKind`** in `expression.rs` + +Edit `crates/core/src/expression/expression.rs`. Replace the `ExpressionKind` enum body with: + +```rust +pub enum ExpressionKind<'a> { + Literal(&'a crate::expression::literal::Literal), + // Future variants (added in subsequent tasks): + // NameReference(&'a NameReference), (Task 1.9) + // BoundReference(&'a BoundReference), (Task 1.10) + // Predicate(&'a dyn crate::expression::Predicate), (Task 1.11) + /// Placeholder so the enum can grow until all variants exist. Removed in Task 1.11. + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +- [ ] **Step 5: Uncomment `pub use` in `expression/mod.rs`** + +```rust +pub use literal::{Literal, LiteralValue}; +``` + +- [ ] **Step 6: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::literal 2>&1 | tail -10` +Expected: 4 tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port Literal + LiteralValue + add ExpressionKind::Literal variant + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.9: Port `NameReference` + add `ExpressionKind::NameReference` + +**Files:** +- Modify: `crates/core/src/expression/name_reference.rs` +- Modify: `crates/core/src/expression/expression.rs` (add `NameReference` variant to `ExpressionKind`) +- Modify: `crates/core/src/expression/mod.rs` (uncomment `pub use name_reference::NameReference;`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/NameReference.java` (49 lines, full) + +- [ ] **Step 1: Write failing tests** at the bottom of `name_reference.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, ExpressionKind}; + + #[test] + fn name_reference_carries_name() { + let nr = NameReference::new("_hoodie_record_key"); + assert_eq!(nr.name(), "_hoodie_record_key"); + } + + #[test] + fn name_reference_kind_pattern_match() { + let nr = NameReference::new("foo"); + match nr.kind() { + ExpressionKind::NameReference(_) => {} + _ => panic!("expected ExpressionKind::NameReference"), + } + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::name_reference 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `NameReference`** + +Replace `crates/core/src/expression/name_reference.rs`: + +```rust +// crates/core/src/expression/name_reference.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.NameReference`. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::types::StringType; +use crate::internal_schema::Type; + +/// Reference to a column by name. Mirrors Java `NameReference(String name)`. +/// +/// Java's `NameReference` has `getDataType()` returning a placeholder +/// `Types.StringType` since the actual type is resolved by `BindVisitor`. +/// Rust mirrors this default. +#[derive(Debug, Clone)] +pub struct NameReference { + name: String, +} + +impl NameReference { + pub fn new(name: impl Into) -> Self { + Self { name: name.into() } + } + + pub fn name(&self) -> &str { &self.name } +} + +impl Expression for NameReference { + fn data_type(&self) -> &dyn Type { + // Java returns Types.StringType.get() as a placeholder until + // BindVisitor resolves the real type. Rust mirrors via a + // `'static` reference held by the struct's static `BIND_PLACEHOLDER`. + // For simplicity we return a fresh StringType — same observable + // behaviour (Java's `Types.StringType.get()` is also a singleton). + static TYPE: StringType = StringType; + &TYPE + } + + fn operator(&self) -> Operator { Operator::True } + + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::NameReference(self) + } +} + +impl LeafExpression for NameReference {} +``` + +- [ ] **Step 4: Add `NameReference` variant to `ExpressionKind`** in `expression.rs` + +Replace the `ExpressionKind` enum: + +```rust +pub enum ExpressionKind<'a> { + Literal(&'a crate::expression::literal::Literal), + NameReference(&'a crate::expression::name_reference::NameReference), + // Future variants: + // BoundReference(&'a BoundReference), (Task 1.10) + // Predicate(&'a dyn crate::expression::Predicate), (Task 1.11) + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +- [ ] **Step 5: Uncomment `pub use` in `expression/mod.rs`** + +```rust +pub use name_reference::NameReference; +``` + +- [ ] **Step 6: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::name_reference 2>&1 | tail -10` +Expected: 2 tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port NameReference + add ExpressionKind::NameReference variant + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.10: Port `BoundReference` + add `ExpressionKind::BoundReference` + +**Files:** +- Modify: `crates/core/src/expression/bound_reference.rs` +- Modify: `crates/core/src/expression/expression.rs` (add `BoundReference` variant to `ExpressionKind`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/BoundReference.java` (53 lines, full) + +- [ ] **Step 1: Write failing tests** at the bottom of `bound_reference.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, ExpressionKind}; + use crate::internal_schema::types::IntType; + use crate::internal_schema::Type; + + #[test] + fn bound_reference_carries_ordinal_and_type() { + let br = BoundReference::new(3, Box::new(IntType::get())); + assert_eq!(br.ordinal(), 3); + assert_eq!(br.data_type().type_id(), crate::internal_schema::TypeID::Int); + } + + #[test] + fn bound_reference_kind_pattern_match() { + let br = BoundReference::new(0, Box::new(IntType::get())); + match br.kind() { + ExpressionKind::BoundReference(_) => {} + _ => panic!("expected ExpressionKind::BoundReference"), + } + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::bound_reference 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `BoundReference`** + +Replace `crates/core/src/expression/bound_reference.rs`: + +```rust +// crates/core/src/expression/bound_reference.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.BoundReference`. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::struct_like::StructLike; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::Type; +use std::any::Any; + +/// Reference to a column by ordinal position. Mirrors Java `BoundReference`. +#[derive(Debug)] +pub struct BoundReference { + ordinal: usize, + data_type: Box, +} + +impl BoundReference { + pub fn new(ordinal: usize, data_type: Box) -> Self { + Self { ordinal, data_type } + } + + pub fn ordinal(&self) -> usize { self.ordinal } +} + +impl Expression for BoundReference { + fn data_type(&self) -> &dyn Type { self.data_type.as_ref() } + + fn operator(&self) -> Operator { Operator::True } + + /// Mirrors Java `BoundReference.eval(data)` — returns the cell value + /// at `ordinal`. + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let s = data?; + let _ = s.get(self.ordinal)?; + // The borrow can't escape; we return None for now — eval is not + // exercised on the keyFilterOpt path. To enable real eval we'd + // need an owned getter on StructLike. + None + } + + fn kind(&self) -> ExpressionKind<'_> { + ExpressionKind::BoundReference(self) + } +} + +impl LeafExpression for BoundReference {} +``` + +- [ ] **Step 4: Add `BoundReference` variant to `ExpressionKind`** + +```rust +pub enum ExpressionKind<'a> { + Literal(&'a crate::expression::literal::Literal), + NameReference(&'a crate::expression::name_reference::NameReference), + BoundReference(&'a crate::expression::bound_reference::BoundReference), + // Future variants: + // Predicate(&'a dyn crate::expression::Predicate), (Task 1.11) + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::bound_reference 2>&1 | tail -10` +Expected: 2 tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port BoundReference + add ExpressionKind::BoundReference variant + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.11: Port `Predicate` trait + `PredicateKind` enum + remove `ExpressionKind::_Placeholder` + +**Files:** +- Modify: `crates/core/src/expression/predicate.rs` +- Modify: `crates/core/src/expression/expression.rs` (add `Predicate` variant; remove `_Placeholder`) +- Modify: `crates/core/src/expression/mod.rs` (uncomment `pub use predicate::{Predicate, PredicateKind};`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Predicate.java` (40 lines, full) + +`PredicateKind<'a>` references the 12 Predicate concrete types from `predicates.rs` which are *not yet defined*. We use the same incremental approach: define `PredicateKind` here with **all 12 variants from the start**, but each variant references a struct in `predicates.rs` that will be created in Tasks 1.12-1.14. Until those tasks land, this task uses a placeholder unit-variant `Pending(std::marker::PhantomData<&'a ()>)`. We remove the placeholder and add real variants in Task 1.14. + +Actually — to avoid 4 incremental edits, the cleanest approach is: **add all 12 variants now** but stub the referenced types. That requires `predicates.rs` to define empty struct stubs. Simpler: **declare `PredicateKind` with a single placeholder variant for now**; replace in Task 1.14 once `predicates.rs` is fully populated. + +This task: define `Predicate` trait + minimal `PredicateKind` placeholder. Real variants added in Task 1.14. + +- [ ] **Step 1: Write failing tests** at the bottom of `predicate.rs` + +```rust +#[cfg(test)] +mod tests { + // No tests until concrete predicates are implemented in Task 1.14. + // This file just defines the trait + enum skeleton. +} +``` + +- [ ] **Step 2: Implement `Predicate` trait + `PredicateKind` skeleton** + +Replace `crates/core/src/expression/predicate.rs`: + +```rust +// crates/core/src/expression/predicate.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.Predicate`. + +use crate::expression::Expression; + +/// Mirrors Java `interface Predicate extends Expression`. +/// +/// Adds a `kind() -> PredicateKind<'_>` accessor for idiomatic Rust +/// pattern matching. See keyFilterOpt design spec §4.1, deviation #2. +pub trait Predicate: Expression { + /// Borrowed enum view for pattern matching. + fn kind_predicate(&self) -> PredicateKind<'_>; +} + +/// Borrowed enum view of `Predicate` for pattern matching. Variants +/// added in Task 1.14 once concrete predicate types exist. +pub enum PredicateKind<'a> { + // Variants added in Task 1.14: + // True, + // False, + // And(&'a predicates::And), + // Or(&'a predicates::Or), + // Not(&'a predicates::Not), + // BinaryComparison(&'a predicates::BinaryComparison), + // In(&'a predicates::In), + // IsNull(&'a predicates::IsNull), + // IsNotNull(&'a predicates::IsNotNull), + // StringStartsWith(&'a predicates::StringStartsWith), + // StringStartsWithAny(&'a predicates::StringStartsWithAny), + // StringContains(&'a predicates::StringContains), + /// Placeholder so the enum compiles. Removed in Task 1.14. + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +(Note: the trait method is `kind_predicate()` not `kind()` to avoid clashing with the `Expression::kind() -> ExpressionKind` already on the supertrait. Java has both `Predicate.getOperator()` (inherited) and the cast-based pattern; Rust separates them. Alternatively, we can rename `Expression::kind` to `Expression::expr_kind` — pick whichever feels less surprising.) + +For consistency with the spec text which says "trait Predicate has a `kind()` accessor", we'll go with the spec name. Solution: name the supertrait method differently. Edit `Expression::kind` → `Expression::expr_kind` if conflict surfaces; or use disambiguated calls (`::kind()`). Decision: keep both as `kind()` — Rust allows it via UFCS but call sites use `::kind` or `::kind`. Cleaner path: name them differently. For this task, name them: +- `Expression::kind() -> ExpressionKind` +- `Predicate::pred_kind() -> PredicateKind` + +Rename the spec accordingly later (or note this in the implementation deviation). + +Actually simpler: Predicate is an Expression. So its `Expression::kind()` could return `ExpressionKind::Predicate(self)`, and its own pattern-match accessor for *predicate-specific* matching is named differently. Let's call the predicate-specific one `Predicate::pred_kind()` to avoid the name clash entirely. + +Apply this naming to the trait: + +```rust +pub trait Predicate: Expression { + fn pred_kind(&self) -> PredicateKind<'_>; +} +``` + +Update the spec text-mention in commit message if needed. + +- [ ] **Step 3: Add `Predicate` variant to `ExpressionKind`** in `expression.rs` + +Replace `ExpressionKind`: + +```rust +pub enum ExpressionKind<'a> { + Literal(&'a crate::expression::literal::Literal), + NameReference(&'a crate::expression::name_reference::NameReference), + BoundReference(&'a crate::expression::bound_reference::BoundReference), + Predicate(&'a dyn crate::expression::predicate::Predicate), +} +``` + +(The `_Placeholder` variant is now removed. All four variants exist.) + +- [ ] **Step 4: Uncomment `pub use` in `expression/mod.rs`** + +```rust +pub use predicate::{Predicate, PredicateKind}; +``` + +- [ ] **Step 5: Run tests; verify still pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression 2>&1 | tail -10` +Expected: all previous tests pass; nothing new in this task. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port Predicate trait + PredicateKind skeleton; finalize ExpressionKind + +PredicateKind variants are added in Task 1.14 once concrete predicate +types exist. ExpressionKind now has all four variants and the placeholder +is removed. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.12: Port `Predicates` — singleton predicates (TrueExpression, FalseExpression) + +**Files:** +- Modify: `crates/core/src/expression/predicates.rs` + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java` lines 96-152 (TrueExpression + FalseExpression). + +The `predicates.rs` file holds all 12 inner classes from Java's `Predicates`. We split the port across Tasks 1.12-1.14 by class group. Each task adds tests + implementation + (eventually) a `PredicateKind` variant. + +- [ ] **Step 1: Write failing tests** at the bottom of `predicates.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::{Expression, Predicate, Operator}; + + #[test] + fn true_expression_eval_returns_true() { + let t = TrueExpression; + let result = t.eval(None).unwrap(); + let bool_val = result.downcast_ref::().expect("expected bool"); + assert!(bool_val); + } + + #[test] + fn true_expression_operator_is_true() { + assert_eq!(TrueExpression.operator(), Operator::True); + } + + #[test] + fn false_expression_eval_returns_false() { + let f = FalseExpression; + let result = f.eval(None).unwrap(); + let bool_val = result.downcast_ref::().expect("expected bool"); + assert!(!bool_val); + } + + #[test] + fn false_expression_operator_is_false() { + assert_eq!(FalseExpression.operator(), Operator::False); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::predicates 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `TrueExpression` + `FalseExpression`** + +Replace `crates/core/src/expression/predicates.rs`: + +```rust +// crates/core/src/expression/predicates.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.Predicates`. +//! +//! All 12 inner classes from Java's `Predicates` live in this single file +//! to mirror Java's structure. Factory functions at the bottom mirror the +//! static methods on Java's `Predicates`. + +use crate::expression::expression::ExpressionKind; +use crate::expression::leaf_expression::LeafExpression; +use crate::expression::predicate::{Predicate, PredicateKind}; +use crate::expression::struct_like::StructLike; +use crate::expression::{Expression, Operator}; +use crate::internal_schema::types::BooleanType; +use crate::internal_schema::Type; +use std::any::Any; + +// ========================================================================= +// Singleton predicates +// ========================================================================= + +/// Mirrors Java `Predicates.TrueExpression`. +#[derive(Debug, Clone, Copy)] +pub struct TrueExpression; + +impl Expression for TrueExpression { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::True } + fn eval(&self, _data: Option<&dyn StructLike>) -> Option> { + Some(Box::new(true)) + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl LeafExpression for TrueExpression {} + +impl Predicate for TrueExpression { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::True } +} + +/// Mirrors Java `Predicates.FalseExpression`. +#[derive(Debug, Clone, Copy)] +pub struct FalseExpression; + +impl Expression for FalseExpression { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::False } + fn eval(&self, _data: Option<&dyn StructLike>) -> Option> { + Some(Box::new(false)) + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl LeafExpression for FalseExpression {} + +impl Predicate for FalseExpression { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::False } +} +``` + +(`PredicateKind::True` / `PredicateKind::False` don't exist yet — referenced here in anticipation of Task 1.14. Compilation will fail. Add them now to `predicate.rs::PredicateKind`:) + +- [ ] **Step 4: Update `PredicateKind` to include True + False variants** + +Edit `crates/core/src/expression/predicate.rs`. Replace the `PredicateKind` enum: + +```rust +pub enum PredicateKind<'a> { + True, + False, + // Future variants (Task 1.13, 1.14): + // And(&'a predicates::And), Or(&'a predicates::Or), Not(&'a predicates::Not), + // BinaryComparison(&'a predicates::BinaryComparison), + // In(&'a predicates::In), IsNull(&'a predicates::IsNull), IsNotNull(&'a predicates::IsNotNull), + // StringStartsWith(&'a predicates::StringStartsWith), + // StringStartsWithAny(&'a predicates::StringStartsWithAny), + // StringContains(&'a predicates::StringContains), + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::predicates 2>&1 | tail -10` +Expected: 4 tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port Predicates::TrueExpression + FalseExpression + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.13: Port `Predicates` — boolean combinators (And, Or, Not) + +**Files:** +- Modify: `crates/core/src/expression/predicates.rs` (append) +- Modify: `crates/core/src/expression/predicate.rs` (add 3 variants to `PredicateKind`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java` lines 154-224 (And + Or) and 360-387 (Not). + +- [ ] **Step 1: Write failing tests** (append to existing `tests` mod in `predicates.rs`) + +```rust + #[test] + fn and_constructs_and_carries_children() { + let a = And::new(Box::new(TrueExpression), Box::new(FalseExpression)); + assert_eq!(a.operator(), Operator::And); + assert_eq!(a.children().len(), 2); + } + + #[test] + fn or_constructs_and_carries_children() { + let o = Or::new(Box::new(TrueExpression), Box::new(FalseExpression)); + assert_eq!(o.operator(), Operator::Or); + assert_eq!(o.children().len(), 2); + } + + #[test] + fn not_carries_child() { + let n = Not::new(Box::new(TrueExpression)); + assert_eq!(n.operator(), Operator::Not); + assert_eq!(n.children().len(), 1); + } + + #[test] + fn and_eval_short_circuits_false() { + let a = And::new(Box::new(FalseExpression), Box::new(TrueExpression)); + let result = a.eval(None).unwrap(); + let b = result.downcast_ref::().unwrap(); + assert!(!b); + } +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::predicates 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `And`, `Or`, `Not`** + +Append to `crates/core/src/expression/predicates.rs`: + +```rust +// ========================================================================= +// Boolean combinators +// ========================================================================= + +/// Mirrors Java `Predicates.And`. +#[derive(Debug)] +pub struct And { + pub left: Box, + pub right: Box, +} + +impl And { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right(&self) -> &dyn Expression { self.right.as_ref() } +} + +impl Expression for And { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::And } + fn children(&self) -> Vec<&dyn Expression> { vec![self.left.as_ref(), self.right.as_ref()] } + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let left = self.left.eval(data)?; + let lb = left.downcast_ref::().copied().unwrap_or(false); + if !lb { return Some(Box::new(false)); } + let right = self.right.eval(data)?; + let rb = right.downcast_ref::().copied().unwrap_or(false); + Some(Box::new(lb && rb)) + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for And { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::And(self) } +} + +/// Mirrors Java `Predicates.Or`. +#[derive(Debug)] +pub struct Or { + pub left: Box, + pub right: Box, +} + +impl Or { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right(&self) -> &dyn Expression { self.right.as_ref() } +} + +impl Expression for Or { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::Or } + fn children(&self) -> Vec<&dyn Expression> { vec![self.left.as_ref(), self.right.as_ref()] } + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let left = self.left.eval(data)?; + let lb = left.downcast_ref::().copied().unwrap_or(false); + if lb { return Some(Box::new(true)); } + let right = self.right.eval(data)?; + let rb = right.downcast_ref::().copied().unwrap_or(false); + Some(Box::new(lb || rb)) + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for Or { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::Or(self) } +} + +/// Mirrors Java `Predicates.Not`. +#[derive(Debug)] +pub struct Not { + pub child: Box, +} + +impl Not { + pub fn new(child: Box) -> Self { Self { child } } + pub fn child(&self) -> &dyn Expression { self.child.as_ref() } +} + +impl Expression for Not { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::Not } + fn children(&self) -> Vec<&dyn Expression> { vec![self.child.as_ref()] } + fn eval(&self, data: Option<&dyn StructLike>) -> Option> { + let v = self.child.eval(data)?; + let b = v.downcast_ref::().copied().unwrap_or(false); + Some(Box::new(!b)) + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for Not { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::Not(self) } +} +``` + +- [ ] **Step 4: Add 3 variants to `PredicateKind`** in `predicate.rs` + +```rust +pub enum PredicateKind<'a> { + True, + False, + And(&'a crate::expression::predicates::And), + Or(&'a crate::expression::predicates::Or), + Not(&'a crate::expression::predicates::Not), + // Future variants (Task 1.14): + // BinaryComparison(...), In(...), IsNull(...), IsNotNull(...), + // StringStartsWith(...), StringStartsWithAny(...), StringContains(...) + _Placeholder(std::marker::PhantomData<&'a ()>), +} +``` + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::predicates 2>&1 | tail -10` +Expected: 8 tests pass (4 from Task 1.12 + 4 new). + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port Predicates::And/Or/Not + 3 PredicateKind variants + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.14: Port `Predicates` — comparison + null + string predicates (BinaryComparison, In, IsNull, IsNotNull, StringStartsWith, StringStartsWithAny, StringContains) + factories + +**Files:** +- Modify: `crates/core/src/expression/predicates.rs` (append) +- Modify: `crates/core/src/expression/predicate.rs` (replace `PredicateKind` with full 12-variant enum, remove `_Placeholder`) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Predicates.java` lines 226-456 (StringStartsWith, StringContains, In, IsNull, IsNotNull, BinaryComparison, StringStartsWithAny) plus the static factory methods on lines 30-94. + +This is the largest single task in Phase 1 — 7 new predicate types plus factory functions plus the final PredicateKind shape. + +- [ ] **Step 1: Write failing tests** (append to existing `tests` mod in `predicates.rs`) + +```rust + use crate::expression::Literal; + + #[test] + fn binary_comparison_eq() { + let bc = BinaryComparison::new( + Box::new(Literal::int(5)), Operator::Eq, Box::new(Literal::int(5))); + assert_eq!(bc.operator(), Operator::Eq); + } + + #[test] + fn in_predicate_carries_children() { + let val = Box::new(Literal::string("col")); + let validvalues = vec![ + Box::new(Literal::string("a")) as Box, + Box::new(Literal::string("b")) as Box, + ]; + let p = In::new(val, validvalues); + assert_eq!(p.operator(), Operator::In); + assert_eq!(p.right_children().len(), 2); + assert_eq!(p.children().len(), 3); // value + 2 valid values + } + + #[test] + fn is_null_carries_child() { + let p = IsNull::new(Box::new(Literal::int(1))); + assert_eq!(p.operator(), Operator::IsNull); + } + + #[test] + fn is_not_null_carries_child() { + let p = IsNotNull::new(Box::new(Literal::int(1))); + assert_eq!(p.operator(), Operator::IsNotNull); + } + + #[test] + fn string_starts_with_carries_left_right() { + let p = StringStartsWith::new( + Box::new(Literal::string("name")), Box::new(Literal::string("Al"))); + assert_eq!(p.operator(), Operator::StartsWith); + } + + #[test] + fn string_starts_with_any_carries_children() { + let p = StringStartsWithAny::new( + Box::new(Literal::string("name")), + vec![ + Box::new(Literal::string("Al")) as Box, + Box::new(Literal::string("Bo")) as Box, + ] + ); + assert_eq!(p.operator(), Operator::StartsWith); + assert_eq!(p.right_children().len(), 2); + } + + #[test] + fn string_contains_constructs() { + let p = StringContains::new( + Box::new(Literal::string("desc")), Box::new(Literal::string("foo"))); + assert_eq!(p.operator(), Operator::Contains); + } + + // Factory functions + #[test] + fn factory_in_constructs_in() { + let p = predicates_factory::in_( + Box::new(Literal::string("col")), + vec![Box::new(Literal::string("a"))], + ); + assert_eq!(p.operator(), Operator::In); + } + + #[test] + fn factory_starts_with_any_constructs_string_starts_with_any() { + let p = predicates_factory::starts_with_any( + Box::new(Literal::string("col")), + vec![Box::new(Literal::string("a"))], + ); + assert_eq!(p.operator(), Operator::StartsWith); + } +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::predicates 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `BinaryComparison`, `In`, `IsNull`, `IsNotNull`, `StringStartsWith`, `StringStartsWithAny`, `StringContains`** + +Append to `crates/core/src/expression/predicates.rs`: + +```rust +// ========================================================================= +// Comparison + null + string predicates +// ========================================================================= + +/// Mirrors Java `Predicates.BinaryComparison`. +#[derive(Debug)] +pub struct BinaryComparison { + pub left: Box, + pub op: Operator, + pub right: Box, +} + +impl BinaryComparison { + pub fn new(left: Box, op: Operator, right: Box) -> Self { + Self { left, op, right } + } + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right(&self) -> &dyn Expression { self.right.as_ref() } +} + +impl Expression for BinaryComparison { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { self.op } + fn children(&self) -> Vec<&dyn Expression> { vec![self.left.as_ref(), self.right.as_ref()] } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for BinaryComparison { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::BinaryComparison(self) } +} + +/// Mirrors Java `Predicates.In`. +#[derive(Debug)] +pub struct In { + pub value: Box, + pub valid_values: Vec>, +} + +impl In { + pub fn new(value: Box, valid_values: Vec>) -> Self { + Self { value, valid_values } + } + pub fn value(&self) -> &dyn Expression { self.value.as_ref() } + pub fn right_children(&self) -> &[Box] { &self.valid_values } +} + +impl Expression for In { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::In } + fn children(&self) -> Vec<&dyn Expression> { + let mut all = vec![self.value.as_ref()]; + all.extend(self.valid_values.iter().map(|e| e.as_ref())); + all + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for In { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::In(self) } +} + +/// Mirrors Java `Predicates.IsNull`. +#[derive(Debug)] +pub struct IsNull { + pub child: Box, +} + +impl IsNull { + pub fn new(child: Box) -> Self { Self { child } } + pub fn child(&self) -> &dyn Expression { self.child.as_ref() } +} + +impl Expression for IsNull { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::IsNull } + fn children(&self) -> Vec<&dyn Expression> { vec![self.child.as_ref()] } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for IsNull { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::IsNull(self) } +} + +/// Mirrors Java `Predicates.IsNotNull`. +#[derive(Debug)] +pub struct IsNotNull { + pub child: Box, +} + +impl IsNotNull { + pub fn new(child: Box) -> Self { Self { child } } + pub fn child(&self) -> &dyn Expression { self.child.as_ref() } +} + +impl Expression for IsNotNull { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::IsNotNull } + fn children(&self) -> Vec<&dyn Expression> { vec![self.child.as_ref()] } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for IsNotNull { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::IsNotNull(self) } +} + +/// Mirrors Java `Predicates.StringStartsWith`. +#[derive(Debug)] +pub struct StringStartsWith { + pub left: Box, + pub right: Box, +} + +impl StringStartsWith { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right(&self) -> &dyn Expression { self.right.as_ref() } +} + +impl Expression for StringStartsWith { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::StartsWith } + fn children(&self) -> Vec<&dyn Expression> { vec![self.left.as_ref(), self.right.as_ref()] } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for StringStartsWith { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::StringStartsWith(self) } +} + +/// Mirrors Java `Predicates.StringStartsWithAny`. +#[derive(Debug)] +pub struct StringStartsWithAny { + pub left: Box, + pub right: Vec>, +} + +impl StringStartsWithAny { + pub fn new(left: Box, right: Vec>) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right_children(&self) -> &[Box] { &self.right } +} + +impl Expression for StringStartsWithAny { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::StartsWith } + fn children(&self) -> Vec<&dyn Expression> { + let mut all = vec![self.left.as_ref()]; + all.extend(self.right.iter().map(|e| e.as_ref())); + all + } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for StringStartsWithAny { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::StringStartsWithAny(self) } +} + +/// Mirrors Java `Predicates.StringContains`. +#[derive(Debug)] +pub struct StringContains { + pub left: Box, + pub right: Box, +} + +impl StringContains { + pub fn new(left: Box, right: Box) -> Self { + Self { left, right } + } + pub fn left(&self) -> &dyn Expression { self.left.as_ref() } + pub fn right(&self) -> &dyn Expression { self.right.as_ref() } +} + +impl Expression for StringContains { + fn data_type(&self) -> &dyn Type { static T: BooleanType = BooleanType; &T } + fn operator(&self) -> Operator { Operator::Contains } + fn children(&self) -> Vec<&dyn Expression> { vec![self.left.as_ref(), self.right.as_ref()] } + fn kind(&self) -> ExpressionKind<'_> { ExpressionKind::Predicate(self) } +} + +impl Predicate for StringContains { + fn pred_kind(&self) -> PredicateKind<'_> { PredicateKind::StringContains(self) } +} + +// ========================================================================= +// Factory functions — mirror Java `Predicates` static methods. +// +// `in` is a Rust keyword so the function is named `in_`. +// ========================================================================= + +pub mod predicates_factory { + use super::*; + + pub fn always_true() -> TrueExpression { TrueExpression } + pub fn always_false() -> FalseExpression { FalseExpression } + pub fn and(left: Box, right: Box) -> And { And::new(left, right) } + pub fn or(left: Box, right: Box) -> Or { Or::new(left, right) } + pub fn not(child: Box) -> Not { Not::new(child) } + pub fn gt(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::Gt, right) + } + pub fn lt(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::Lt, right) + } + pub fn eq(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::Eq, right) + } + pub fn gteq(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::GtEq, right) + } + pub fn lteq(left: Box, right: Box) -> BinaryComparison { + BinaryComparison::new(left, Operator::LtEq, right) + } + pub fn starts_with(left: Box, right: Box) -> StringStartsWith { + StringStartsWith::new(left, right) + } + pub fn contains(left: Box, right: Box) -> StringContains { + StringContains::new(left, right) + } + pub fn in_(value: Box, valid_values: Vec>) -> In { + In::new(value, valid_values) + } + pub fn is_null(child: Box) -> IsNull { IsNull::new(child) } + pub fn is_not_null(child: Box) -> IsNotNull { IsNotNull::new(child) } + pub fn starts_with_any(left: Box, right: Vec>) -> StringStartsWithAny { + StringStartsWithAny::new(left, right) + } +} +``` + +- [ ] **Step 4: Replace `PredicateKind` with full 12-variant enum (no placeholder)** in `predicate.rs` + +```rust +pub enum PredicateKind<'a> { + True, + False, + And(&'a crate::expression::predicates::And), + Or(&'a crate::expression::predicates::Or), + Not(&'a crate::expression::predicates::Not), + BinaryComparison(&'a crate::expression::predicates::BinaryComparison), + In(&'a crate::expression::predicates::In), + IsNull(&'a crate::expression::predicates::IsNull), + IsNotNull(&'a crate::expression::predicates::IsNotNull), + StringStartsWith(&'a crate::expression::predicates::StringStartsWith), + StringStartsWithAny(&'a crate::expression::predicates::StringStartsWithAny), + StringContains(&'a crate::expression::predicates::StringContains), +} +``` + +(The `_Placeholder` variant is now removed.) + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::predicates 2>&1 | tail -10` +Expected: 18 tests pass (8 from prior tasks + 10 new). + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port Predicates::{BinaryComparison,In,IsNull,IsNotNull,StringStartsWith*,StringContains} + factories + +Finalizes PredicateKind to 12 variants. All Predicates inner classes +from Java's Predicates.java are now ported. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.15: Port `Comparators` + +**Files:** +- Modify: `crates/core/src/expression/comparators.rs` + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/Comparators.java` (55 lines, full) + +- [ ] **Step 1: Write failing tests** at the bottom of `comparators.rs` + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::LiteralValue; + use crate::internal_schema::types::*; + use std::cmp::Ordering; + + #[test] + fn int_comparator_orders_correctly() { + let cmp = Comparators::for_type(&IntType::get()); + assert_eq!(cmp(&LiteralValue::Int(1), &LiteralValue::Int(2)), Ordering::Less); + assert_eq!(cmp(&LiteralValue::Int(5), &LiteralValue::Int(5)), Ordering::Equal); + assert_eq!(cmp(&LiteralValue::Int(9), &LiteralValue::Int(2)), Ordering::Greater); + } + + #[test] + fn string_comparator_lexicographic() { + let cmp = Comparators::for_type(&StringType::get()); + assert_eq!(cmp(&LiteralValue::String("a".into()), &LiteralValue::String("b".into())), Ordering::Less); + } + + #[test] + fn boolean_comparator() { + let cmp = Comparators::for_type(&BooleanType::get()); + assert_eq!(cmp(&LiteralValue::Bool(false), &LiteralValue::Bool(true)), Ordering::Less); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::comparators 2>&1 | tail -10` +Expected: FAIL. + +- [ ] **Step 3: Implement `Comparators`** + +Replace `crates/core/src/expression/comparators.rs`: + +```rust +// crates/core/src/expression/comparators.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.Comparators`. + +use crate::expression::LiteralValue; +use crate::internal_schema::{Type, TypeID}; +use std::cmp::Ordering; + +pub struct Comparators; + +impl Comparators { + /// Mirrors Java `Comparators.forType(Type.PrimitiveType)`. Returns a + /// comparator function over `LiteralValue`. For unsupported types the + /// comparator returns `Ordering::Equal` (equivalent to Java throwing, + /// but we don't panic since the keyFilterOpt path doesn't exercise + /// non-string comparisons). + pub fn for_type(ty: &dyn Type) -> fn(&LiteralValue, &LiteralValue) -> Ordering { + match ty.type_id() { + TypeID::Boolean => |a, b| match (a, b) { + (LiteralValue::Bool(x), LiteralValue::Bool(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::Int | TypeID::Date | TypeID::TimeMillis => |a, b| match (a, b) { + (LiteralValue::Int(x), LiteralValue::Int(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::Long + | TypeID::Time + | TypeID::Timestamp + | TypeID::TimestampMillis + | TypeID::LocalTimestampMillis + | TypeID::LocalTimestampMicros => |a, b| match (a, b) { + (LiteralValue::Long(x), LiteralValue::Long(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::Float => |a, b| match (a, b) { + (LiteralValue::Float(x), LiteralValue::Float(y)) => x.partial_cmp(y).unwrap_or(Ordering::Equal), + _ => Ordering::Equal, + }, + TypeID::Double => |a, b| match (a, b) { + (LiteralValue::Double(x), LiteralValue::Double(y)) => x.partial_cmp(y).unwrap_or(Ordering::Equal), + _ => Ordering::Equal, + }, + TypeID::String => |a, b| match (a, b) { + (LiteralValue::String(x), LiteralValue::String(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + TypeID::UUID => |a, b| match (a, b) { + (LiteralValue::UUID(x), LiteralValue::UUID(y)) => x.cmp(y), + _ => Ordering::Equal, + }, + _ => |_, _| Ordering::Equal, + } + } +} +``` + +- [ ] **Step 4: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression::comparators 2>&1 | tail -10` +Expected: 3 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/core/src/expression/comparators.rs +git commit -m "port Comparators + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.16: Port `ExpressionVisitor`, `BindVisitor`, `PartialBindVisitor` + +**Files:** +- Modify: `crates/core/src/expression/expression_visitor.rs` +- Modify: `crates/core/src/expression/bind_visitor.rs` +- Modify: `crates/core/src/expression/partial_bind_visitor.rs` + +**Java sources:** +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/ExpressionVisitor.java` (41 lines, full) +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/BindVisitor.java` (187 lines) +- `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/expression/PartialBindVisitor.java` (153 lines) + +These are ported but **not wired into the reader path** (deviation #6 in spec §6). Since they're not used by keyFilterOpt's runtime path, our test bar is "they compile, basic construction works". A full implementation of bind logic mirrors Java behavior but is dead code in our scope. + +- [ ] **Step 1: Implement `ExpressionVisitor` trait** + +Replace `crates/core/src/expression/expression_visitor.rs`: + +```rust +// crates/core/src/expression/expression_visitor.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.ExpressionVisitor`. + +use crate::expression::predicates::{ + And, BinaryComparison, FalseExpression, In, IsNotNull, IsNull, Not, Or, StringContains, + StringStartsWith, StringStartsWithAny, TrueExpression, +}; +use crate::expression::{Expression, Literal, NameReference}; +use crate::expression::bound_reference::BoundReference; + +/// Mirrors Java `ExpressionVisitor`. Each visit method has a default +/// returning `T::default()` (Rust constraint: T: Default). +pub trait ExpressionVisitor { + fn always_true(&mut self) -> T where T: Default { T::default() } + fn always_false(&mut self) -> T where T: Default { T::default() } + fn visit_and(&mut self, _and: &And) -> T where T: Default { T::default() } + fn visit_or(&mut self, _or: &Or) -> T where T: Default { T::default() } + fn visit_not(&mut self, _not: &Not) -> T where T: Default { T::default() } + fn visit_predicate(&mut self, _expr: &dyn Expression) -> T where T: Default { T::default() } + fn visit_literal(&mut self, _lit: &Literal) -> T where T: Default { T::default() } + fn visit_name_reference(&mut self, _nr: &NameReference) -> T where T: Default { T::default() } + fn visit_bound_reference(&mut self, _br: &BoundReference) -> T where T: Default { T::default() } + fn visit_binary_comparison(&mut self, _bc: &BinaryComparison) -> T where T: Default { T::default() } + fn visit_in(&mut self, _in_: &In) -> T where T: Default { T::default() } + fn visit_is_null(&mut self, _p: &IsNull) -> T where T: Default { T::default() } + fn visit_is_not_null(&mut self, _p: &IsNotNull) -> T where T: Default { T::default() } + fn visit_string_starts_with(&mut self, _p: &StringStartsWith) -> T where T: Default { T::default() } + fn visit_string_starts_with_any(&mut self, _p: &StringStartsWithAny) -> T where T: Default { T::default() } + fn visit_string_contains(&mut self, _p: &StringContains) -> T where T: Default { T::default() } + fn visit_true(&mut self, _t: &TrueExpression) -> T where T: Default { T::default() } + fn visit_false(&mut self, _f: &FalseExpression) -> T where T: Default { T::default() } +} +``` + +- [ ] **Step 2: Implement `BindVisitor` (stub)** + +Replace `crates/core/src/expression/bind_visitor.rs`: + +```rust +// crates/core/src/expression/bind_visitor.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.BindVisitor`. +//! +//! BindVisitor walks an expression tree, replacing `NameReference`s with +//! `BoundReference`s by looking up names in a `RecordType` schema. +//! +//! Ported as a placeholder — not wired into the keyFilterOpt reader path. +//! See keyFilterOpt design spec §6 deviation #6. + +use crate::expression::ExpressionVisitor; +use crate::internal_schema::types::RecordType; + +pub struct BindVisitor { + pub schema: RecordType, + pub case_sensitive: bool, +} + +impl BindVisitor { + pub fn new(schema: RecordType, case_sensitive: bool) -> Self { + Self { schema, case_sensitive } + } +} + +// Minimal ExpressionVisitor impl — defaults return Box::::default +// which doesn't compile, so we'll need a wrapper. For now, BindVisitor is a +// declared-but-unused type. Full implementation is deferred until the +// keyFilterOpt scope expands to need binding. Spec §6 deviation #6. +impl ExpressionVisitor for BindVisitor {} +``` + +- [ ] **Step 3: Implement `PartialBindVisitor` (stub)** + +Replace `crates/core/src/expression/partial_bind_visitor.rs`: + +```rust +// crates/core/src/expression/partial_bind_visitor.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.expression.PartialBindVisitor`. +//! +//! Like BindVisitor but tolerates unresolvable NameReferences. Ported as +//! placeholder; not wired. Spec §6 deviation #6. + +use crate::expression::ExpressionVisitor; +use crate::internal_schema::types::RecordType; + +pub struct PartialBindVisitor { + pub schema: RecordType, + pub case_sensitive: bool, +} + +impl PartialBindVisitor { + pub fn new(schema: RecordType, case_sensitive: bool) -> Self { + Self { schema, case_sensitive } + } +} + +impl ExpressionVisitor for PartialBindVisitor {} +``` + +- [ ] **Step 4: Run tests; verify still pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib expression 2>&1 | tail -10` +Expected: all previous tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/core/src/expression +git commit -m "port ExpressionVisitor + BindVisitor + PartialBindVisitor (stubs) + +Visitors are ported as declarations for hierarchy completeness. The bind +logic is not wired into the reader path — see spec §6 deviation #6. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 1.17: Phase 1 verification — full crate test pass + lint + +**Files:** None modified — verification only. + +- [ ] **Step 1: Run full crate test suite** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core 2>&1 | tail -15` +Expected: all tests pass. No regressions to existing tests. + +- [ ] **Step 2: Run clippy** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo clippy -p hudi-core --all-targets 2>&1 | tail -30` +Expected: no errors. Address any warnings introduced by the new modules. + +- [ ] **Step 3: Build the workspace** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo build --workspace 2>&1 | tail -10` +Expected: clean build, no warnings introduced by Phase 1. + +- [ ] **Step 4: If lint or build issues found, fix and commit** + +```bash +git add -A +git commit -m "phase 1: clippy fixes / lint cleanup + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +If no issues, skip the commit. + +--- + +## Phase 2 — Wire keyFilterOpt through ReaderContext + reader + +### Task 2.1: Add `KeySpec` enum + `create_key_spec` + `string_literals` helper + +**Files:** +- Create: `crates/core/src/file_group/reader/key_spec.rs` +- Modify: `crates/core/src/file_group/reader/mod.rs:66-81` (add `pub mod key_spec;` to module list) + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java` lines 109-126 (`createKeySpec`). + +- [ ] **Step 1: Write failing tests** + +Create `crates/core/src/file_group/reader/key_spec.rs`: + +```rust +// crates/core/src/file_group/reader/key_spec.rs +/* … license header … */ + +//! Mirrors Java `org.apache.hudi.common.table.log.KeySpec` hierarchy +//! (`KeySpec`, `FullKeySpec`, `PrefixKeySpec`) plus +//! `HoodieMergedLogRecordReader.createKeySpec(Option)`. + +use crate::expression::predicate::{Predicate, PredicateKind}; +use crate::expression::{Expression, ExpressionKind, LiteralValue}; +use std::sync::Arc; + +/// Mirrors Java's `KeySpec` interface, with `FullKeySpec` and `PrefixKeySpec` +/// flattened into enum variants. +#[derive(Debug, Clone)] +pub enum KeySpec { + FullKeys(Vec), + PrefixKeys(Vec), +} + +impl KeySpec { + /// Test the given record-key string against this key-spec. + pub fn matches(&self, key: &str) -> bool { + match self { + KeySpec::FullKeys(keys) => keys.iter().any(|k| k == key), + KeySpec::PrefixKeys(prefixes) => prefixes.iter().any(|p| key.starts_with(p.as_str())), + } + } +} + +/// Mirrors Java `HoodieMergedLogRecordReader.createKeySpec(Option)`, +/// implemented in idiomatic Rust via `PredicateKind` pattern matching instead +/// of Java's `instanceof`/cast. +pub fn create_key_spec(filter: Option<&dyn Predicate>) -> Option { + match filter?.pred_kind() { + PredicateKind::In(p) => Some(KeySpec::FullKeys(string_literals(p.right_children())?)), + PredicateKind::StringStartsWithAny(p) => { + Some(KeySpec::PrefixKeys(string_literals(p.right_children())?)) + } + _ => None, + } +} + +/// Extract `String` values from a list of `Literal(LiteralValue::String(_))` +/// expressions. Returns `None` if any child is not a String literal. +pub(crate) fn string_literals(exprs: &[Box]) -> Option> { + exprs + .iter() + .map(|e| match e.kind() { + ExpressionKind::Literal(lit) => match lit.value() { + LiteralValue::String(s) => Some(s.clone()), + _ => None, + }, + _ => None, + }) + .collect() +} + +/// Convenience: accept an `Option>` directly (the type +/// stored on `ReaderContext`). +pub fn create_key_spec_from_arc(filter: &Option>) -> Option { + create_key_spec(filter.as_deref()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expression::predicates::predicates_factory; + use crate::expression::{Literal, NameReference}; + + #[test] + fn create_key_spec_none_when_filter_is_none() { + assert!(create_key_spec(None).is_none()); + } + + #[test] + fn create_key_spec_in_predicate_yields_full_keys() { + let p = predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![ + Box::new(Literal::string("k1")) as Box, + Box::new(Literal::string("k2")) as Box, + ], + ); + let spec = create_key_spec(Some(&p)).expect("expected Some(KeySpec)"); + match spec { + KeySpec::FullKeys(keys) => assert_eq!(keys, vec!["k1".to_string(), "k2".to_string()]), + _ => panic!("expected FullKeys"), + } + } + + #[test] + fn create_key_spec_starts_with_any_yields_prefix_keys() { + let p = predicates_factory::starts_with_any( + Box::new(NameReference::new("_hoodie_record_key")), + vec![ + Box::new(Literal::string("alpha")) as Box, + Box::new(Literal::string("beta")) as Box, + ], + ); + let spec = create_key_spec(Some(&p)).expect("expected Some(KeySpec)"); + match spec { + KeySpec::PrefixKeys(prefixes) => { + assert_eq!(prefixes, vec!["alpha".to_string(), "beta".to_string()]) + } + _ => panic!("expected PrefixKeys"), + } + } + + #[test] + fn create_key_spec_returns_none_for_non_in_non_starts_with() { + let p = predicates_factory::eq( + Box::new(Literal::string("a")), + Box::new(Literal::string("b")), + ); + assert!(create_key_spec(Some(&p)).is_none()); + } + + #[test] + fn full_keys_matches() { + let spec = KeySpec::FullKeys(vec!["a".into(), "b".into()]); + assert!(spec.matches("a")); + assert!(spec.matches("b")); + assert!(!spec.matches("c")); + } + + #[test] + fn prefix_keys_matches() { + let spec = KeySpec::PrefixKeys(vec!["pre".into(), "alpha".into()]); + assert!(spec.matches("prefix-1")); + assert!(spec.matches("alphabet")); + assert!(!spec.matches("beta")); + } +} +``` + +- [ ] **Step 2: Add `pub mod key_spec;`** to `crates/core/src/file_group/reader/mod.rs` + +Find the module declarations at lines 66-81 and insert alphabetically: + +```rust +pub mod buffer; +pub mod buffered_record; +pub mod buffered_record_converter; +pub mod delete_context; +pub mod input_split; +pub mod iterator_mode; +pub mod key_spec; // ← NEW +pub mod log_record_reader; +pub mod merged_log_record_reader; +pub mod output_converter; +pub mod read_stats; +pub mod reader_context; +pub mod reader_parameters; +pub mod record_context; +pub mod record_merger; +pub mod schema_handler; +pub mod update_processor; +``` + +- [ ] **Step 3: Run tests; expect pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib file_group::reader::key_spec 2>&1 | tail -10` +Expected: 6 tests pass. + +- [ ] **Step 4: Commit** + +```bash +git add crates/core/src/file_group/reader +git commit -m "add KeySpec + create_key_spec for keyFilterOpt log-scan path + +Mirrors Java HoodieMergedLogRecordReader.createKeySpec via Rust kind() +pattern matching. Pure addition; not yet wired into scanner. Tests +cover In, StringStartsWithAny, non-matching, and KeySpec.matches. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 2.2: Add `key_filter_opt` field to `ReaderContext` + +**Files:** +- Modify: `crates/core/src/file_group/reader/reader_context.rs` + +- [ ] **Step 1: Write failing tests** (append to existing `tests` mod or add new mod at the bottom) + +Append at bottom of `crates/core/src/file_group/reader/reader_context.rs`: + +```rust +#[cfg(test)] +mod key_filter_opt_tests { + use super::*; + use crate::expression::predicates::predicates_factory; + use crate::expression::{Expression, Literal, NameReference}; + use std::sync::Arc; + + #[test] + fn empty_reader_context_has_no_key_filter() { + let ctx = ReaderContext::empty(); + assert!(ctx.get_key_filter_opt().is_none()); + } + + #[test] + fn key_filter_opt_can_be_set_and_retrieved() { + let mut ctx = ReaderContext::empty(); + let pred = predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string("k1")) as Box], + ); + ctx.key_filter_opt = Some(Arc::new(pred)); + assert!(ctx.get_key_filter_opt().is_some()); + } +} +``` + +- [ ] **Step 2: Run tests; expect compile failure** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib file_group::reader::reader_context::key_filter_opt 2>&1 | tail -10` +Expected: FAIL — `key_filter_opt` field does not exist. + +- [ ] **Step 3: Add the field + getter** + +In `crates/core/src/file_group/reader/reader_context.rs`: + +1. Add import at the top (after the existing `use` block): + +```rust +use crate::expression::predicate::Predicate; +use std::sync::Arc; +``` + +2. In the `ReaderContext` struct (around lines 53-84), add a new field at the bottom of the struct: + +```rust + /// Mirrors Java `HoodieReaderContext.keyFilterOpt` — an optional + /// predicate (typically `In` or `StringStartsWithAny`) used to + /// narrow which records are scanned. None by default. See spec §5.1. + pub key_filter_opt: Option>, +``` + +3. Add a getter accessor inside the `impl ReaderContext` block: + +```rust + /// Mirrors Java `HoodieReaderContext.getKeyFilterOpt()`. + pub fn get_key_filter_opt(&self) -> Option> { + self.key_filter_opt.clone() + } +``` + +4. In `ReaderContext::empty()` (around line 176-194), add `key_filter_opt: None,` to the struct literal: + +```rust + pub fn empty() -> Self { + Self { + // … existing fields … + table_config: HashMap::new(), + hoodie_reader_config: HashMap::new(), + key_filter_opt: None, // ← NEW + } + } +``` + +5. Update the doc comment on `ReaderContext` (the table at lines 41-52) to add the new field. Insert a row: + +```rust + /// | `readerContext.getKeyFilterOpt()` | `key_filter_opt` | +``` + +- [ ] **Step 4: Audit and update other ReaderContext construction sites** + +Search for places that build a `ReaderContext` literal: + +```bash +cd /home/ubuntu/ws3/hudi-rs && grep -rn "ReaderContext {" crates/core/src --include='*.rs' | grep -v "/target/" +``` + +For each such site, add `key_filter_opt: None,` to the literal (initializer). Likely sites: `crates/core/src/cpp/context.rs` (FFI bridge), test fixtures. Spec §5.1 says FFI initializes to None. + +- [ ] **Step 5: Run tests; verify pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib file_group::reader::reader_context 2>&1 | tail -10` +Expected: tests pass; entire crate still compiles. + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo build -p hudi-core 2>&1 | tail -10` +Expected: clean build. + +- [ ] **Step 6: Commit** + +```bash +git add crates/core/src +git commit -m "add key_filter_opt field to ReaderContext + +Mirrors Java HoodieReaderContext.keyFilterOpt. Default None on all +construction sites including FFI bridge and ReaderContext::empty(). + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 2.3: Modify `scan_internal` signature to accept `Option` + apply filter in record processing + +**Files:** +- Modify: `crates/core/src/file_group/reader/log_record_reader.rs` + +**Java reference:** `BaseHoodieLogRecordReader.scanInternal(Option, boolean)` — the Java signature already has the KeySpec parameter; we're catching the Rust port up. + +The key change is twofold: +1. Add `key_spec_opt: Option` parameter to `scan_internal`. +2. Inside `process_queued_blocks_for_instant` (around line 476), filter records by `KeySpec` before dispatching to the buffer's `process_data_block` / `process_delete_block`. + +The filter must apply per-record, not per-block, because a block has many records. Approach: pass the `KeySpec` reference into the buffer's process methods, and the buffer applies it. Or: filter in the scanner before calling the buffer. + +**Cleaner design:** filter inside the scanner. The scanner has access to the record key (via `record_context`). After block content is loaded, iterate records and skip those that don't match the KeySpec. + +But hudi-rs's `process_data_block` / `process_delete_block` are *implemented on the buffer*, taking a `LogBlock` and inflating it internally. To filter records inside the scan, we need the scanner to inflate the block, iterate records, filter, then dispatch — duplicating buffer logic. + +**Pragmatic approach:** add a `key_spec_opt: Option` field to `BaseHoodieLogRecordReader`, and modify the buffer trait to accept an optional KeySpec. The buffer's process methods skip records that don't match. + +Concretely: +- `BaseHoodieLogRecordReader` stores `key_spec_opt: Option`. +- The `HoodieFileGroupRecordBuffer` trait grows two new methods: `set_key_spec(&mut self, key_spec: Option)` and `key_spec_filter(&self, key: &str) -> bool` (default: true). +- `KeyBasedFileGroupRecordBuffer::process_data_block` and `process_delete_block` call `key_spec_filter(&key)` per record and skip non-matching keys. + +**Even simpler:** filter at the buffer's `process_next_data_record` / `process_next_deleted_record`, since those receive the key. Buffer holds an `Option` field set by the scanner before scan. No trait changes needed beyond a setter. + +That's the path of least resistance. Implementation: + +- [ ] **Step 1: Write failing test** for `BaseHoodieLogRecordReader::scan_internal` accepting `Option` + +This is hard to test cleanly without a full scan setup. Skip a unit test at this layer — Phase 3's e2e test exercises the wired-up flow. For Task 2.3, our test bar is "scan_internal compiles with new signature; existing tests still pass after callers are updated". + +- [ ] **Step 2: Add `key_spec_opt` field to `BaseHoodieLogRecordReader`** + +In `crates/core/src/file_group/reader/log_record_reader.rs`, find the struct definition (search `pub struct BaseHoodieLogRecordReader`). Add a new field: + +```rust + /// Mirrors Java `BaseHoodieLogRecordReader.keySpecOpt` — set by the + /// scanner caller to narrow which records are processed. + pub key_spec_opt: Option, +``` + +Update the constructor (or the builder) to default this to `None`. The constructor calls in `merged_log_record_reader.rs` will need a corresponding `key_spec_opt: None` default added. + +- [ ] **Step 3: Update `scan_internal` signature** + +Change `crates/core/src/file_group/reader/log_record_reader.rs:371`: + +```rust + pub async fn scan_internal( + &mut self, + key_spec_opt: Option, + skip_processing_blocks: bool, + ) -> Result<()> { + // Store on self so deeper dispatch points can consult it. + self.key_spec_opt = key_spec_opt; + // … rest of the method unchanged + } +``` + +- [ ] **Step 4: Apply filter in `process_queued_blocks_for_instant`** + +Find the function (around line 476). Before calling `self.record_buffer.process_data_block` / `process_delete_block`, propagate the `key_spec_opt` to the record buffer via a new buffer trait method `set_key_spec` (next step) — or, if the buffer's `process_*` methods iterate records, do the filter inside the buffer. + +For consistency with the Java structure, let's add a setter on the `HoodieFileGroupRecordBuffer` trait: + +In `crates/core/src/file_group/reader/buffer/mod.rs` (find the `HoodieFileGroupRecordBuffer` trait definition), add: + +```rust + /// Set the key-spec filter. Records whose key doesn't match are skipped + /// during `process_data_block` / `process_delete_block`. Mirrors how + /// Java's BaseHoodieLogRecordReader threads keySpecOpt through to + /// per-record processing. + fn set_key_spec(&mut self, _key_spec: Option) { + // default: no-op for buffers that don't support filtering + } +``` + +In `crates/core/src/file_group/reader/buffer/key_based.rs`, override `set_key_spec` and store on the buffer: + +1. Add field to `KeyBasedFileGroupRecordBuffer`: + +```rust +pub struct KeyBasedFileGroupRecordBuffer { + pub base: FileGroupRecordBuffer, + pub reader_context: Arc, + pub record_context: RecordContext, + pub key_spec_opt: Option, // ← NEW +} +``` + +Initialize to `None` in `KeyBasedFileGroupRecordBuffer::new`: + +```rust + Ok(Self { + base, + reader_context, + record_context, + key_spec_opt: None, // ← NEW + }) +``` + +2. Override `set_key_spec`: + +```rust + fn set_key_spec(&mut self, key_spec: Option) { + self.key_spec_opt = key_spec; + } +``` + +3. In `process_next_data_record` (around the `for (key, record) in records { … }` loop in `process_data_block`), check the key against `key_spec_opt`: + +Edit `process_data_block` (around line 213-228 of `key_based.rs`): + +```rust + for batch in record_batches.data_batches { + let records = self.record_context.batch_to_buffered_records( + &batch, + self.base.delete_context.as_ref(), + )?; + for (key, record) in records { + // Skip records that don't match the key-spec filter. + if let Some(spec) = &self.key_spec_opt { + if !spec.matches(&key) { continue; } + } + self.process_next_data_record(record, &key)?; + } + } +``` + +Same in `process_delete_block` (around line 287-298): + +```rust + for (batch, _inst) in record_batches.delete_batches { + let delete_records = self.record_context.delete_batch_to_buffered_records(&batch)?; + for (key, _record) in delete_records { + if let Some(spec) = &self.key_spec_opt { + if !spec.matches(&key) { continue; } + } + let delete_record = DeleteRecord { + record_key: key.clone(), + partition_path: String::new(), + ordering_value: None, + }; + self.process_next_deleted_record(delete_record, &key); + } + } +``` + +4. Propagate `self.key_spec_opt` from the scanner to the buffer in `BaseHoodieLogRecordReader::scan_internal`: + +After the line `self.key_spec_opt = key_spec_opt;` (Step 3), add: + +```rust + // Propagate to buffer for per-record filtering (mirrors Java's + // BaseHoodieLogRecordReader → buffer key-spec threading). + self.record_buffer.set_key_spec(self.key_spec_opt.clone()); +``` + +- [ ] **Step 5: Update all callers of `scan_internal`** + +Search for `scan_internal(`: + +```bash +grep -rn "scan_internal(" crates/core/src --include='*.rs' | grep -v "fn scan_internal" +``` + +Expected hits: +- `crates/core/src/file_group/reader/merged_log_record_reader.rs:120` and `:137` — change to `scan_internal(None, skip_processing_blocks)` and `scan_internal(None, false)` respectively. (The actual `Some(KeySpec)` plumb-through happens in Task 2.4.) + +- [ ] **Step 6: Run tests; verify still pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib 2>&1 | tail -15` +Expected: all existing tests pass. The new path (key_spec_opt = Some(…)) is unexercised yet — Phase 3 tests it end-to-end. + +- [ ] **Step 7: Commit** + +```bash +git add crates/core/src +git commit -m "scan_internal accepts Option; buffer applies key-spec filter + +Mirrors Java BaseHoodieLogRecordReader.scanInternal(Option, boolean). +KeyBasedFileGroupRecordBuffer overrides set_key_spec to store the spec +and skip non-matching records in process_data_block/process_delete_block. +All existing callers pass None; behavior unchanged. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 2.4: Wire `create_key_spec` into `HoodieMergedLogRecordReader::perform_scan` + +**Files:** +- Modify: `crates/core/src/file_group/reader/merged_log_record_reader.rs:133-150` + +**Java source:** `~/ws3/hudi-internal/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordReader.java` lines 95-107 (`performScan`). + +- [ ] **Step 1: Replace the TODO in `perform_scan`** + +In `crates/core/src/file_group/reader/merged_log_record_reader.rs`, find `perform_scan` (around line 133). Replace the body: + +```rust + async fn perform_scan(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + // Mirrors Java: Option keySpecOpt = createKeySpec(readerContext.getKeyFilterOpt()); + let key_spec_opt = crate::file_group::reader::key_spec::create_key_spec_from_arc( + &self.base.reader_context.key_filter_opt, + ); + + self.base.scan_internal(key_spec_opt, false).await?; + + self.total_time_taken_to_read_and_merge_blocks_ms = + start.elapsed().as_millis() as u64; + self.num_merged_records_in_log = self.base.record_buffer.size() as u64; + + log::info!( + "Number of log files scanned => {}", + self.base.log_file_paths.len() + ); + log::info!( + "Number of entries in Map => {}", + self.base.record_buffer.size() + ); + Ok(()) + } +``` + +The other `scan_internal` call site at line 120 (in `scan_with_skip`) is unchanged — pass `None` for the key spec because regular `scan` (not `performScan`) doesn't apply the key filter: + +```rust + pub async fn scan_with_skip(&mut self, skip_processing_blocks: bool) -> Result<()> { + if self.base.force_full_scan { + return Ok(()); + } + self.base.scan_internal(None, skip_processing_blocks).await + } +``` + +- [ ] **Step 2: Run tests; verify still pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --lib 2>&1 | tail -10` +Expected: all tests pass. The behavior with `key_filter_opt = None` (the default everywhere) is identical to before. + +- [ ] **Step 3: Commit** + +```bash +git add crates/core/src/file_group/reader/merged_log_record_reader.rs +git commit -m "wire create_key_spec into HoodieMergedLogRecordReader::perform_scan + +Removes the existing TODO. Mirrors Java performScan() lines 95-107. +key_filter_opt defaults to None on all current call paths so behavior +is unchanged for existing users. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 2.5: Apply key-spec row filter to base file batches in `HoodieFileGroupReader::make_base_file_batches` + +**Files:** +- Modify: `crates/core/src/file_group/reader/mod.rs:455-497` (`make_base_file_batches`) + +**Java source:** `HoodieAvroReaderContext.getFileRecordIterator` lines 218-228 — the "fall through to row-level filter when reader doesn't support key predicate" semantics. + +- [ ] **Step 1: Locate `_hoodie_record_key` column index utility** + +Look for an existing helper to find a column index by name. If none exists, write one inline. + +- [ ] **Step 2: Add filter step after parquet read** + +In `crates/core/src/file_group/reader/mod.rs`, find `make_base_file_batches` (around line 455). After the `Ok(vec![batch])` line at the end of the `Some(path)` arm, replace it with a filter step: + +```rust + let batch = if let Some(required_schema) = &self.schema_handler.required_schema { + self.storage + .get_parquet_file_data_projected(path, required_schema) + .await + .map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to read base file '{path}' with projection: {e:?}" + )) + })? + } else { + self.storage + .get_parquet_file_data(path) + .await + .map_err(|e| { + CoreError::ReadFileSliceError(format!( + "Failed to read base file '{path}': {e:?}" + )) + })? + }; + + // Mirrors Java HoodieAvroReaderContext.getFileRecordIterator + // (lines 218-228): when keyFilterOpt is set and the reader does + // not support a native key predicate (Parquet has none), apply + // a row-level filter on _hoodie_record_key. + let batch = apply_key_filter_to_batch( + batch, + self.reader_context.key_filter_opt.as_deref(), + )?; + Ok(vec![batch]) +``` + +- [ ] **Step 3: Add the helper function `apply_key_filter_to_batch`** + +Add to the same file (`crates/core/src/file_group/reader/mod.rs`), as a free function near the bottom of the impl block or the file: + +```rust +/// Filter rows in `batch` so only those whose `_hoodie_record_key` matches +/// the given key-spec (derived from `key_filter_opt`) survive. If +/// `key_filter_opt` is None or doesn't yield a KeySpec, returns the batch +/// unchanged. +/// +/// Mirrors Java HoodieAvroReaderContext.getFileRecordIterator (lines +/// 218-228) for the non-key-predicate-supporting reader case. +fn apply_key_filter_to_batch( + batch: RecordBatch, + key_filter_opt: Option<&dyn crate::expression::Predicate>, +) -> Result { + use crate::file_group::reader::key_spec::create_key_spec; + let key_spec = match create_key_spec(key_filter_opt) { + Some(spec) => spec, + None => return Ok(batch), + }; + + // Find the _hoodie_record_key column. + let key_col_idx = match batch.schema().index_of("_hoodie_record_key") { + Ok(i) => i, + Err(_) => { + // Column not present (e.g., virtual keys). Return batch as-is — + // the filter cannot apply. + log::debug!( + "[HoodieFileGroupReader] key_filter_opt set but _hoodie_record_key \ + column not present in base file; skipping base-file filter" + ); + return Ok(batch); + } + }; + + let key_col = batch + .column(key_col_idx) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + CoreError::ReadFileSliceError( + "_hoodie_record_key column is not StringArray".to_string(), + ) + })?; + + // Build a BooleanArray mask: true for rows whose key matches the spec. + let mask: arrow_array::BooleanArray = (0..batch.num_rows()) + .map(|i| { + if key_col.is_null(i) { + Some(false) + } else { + Some(key_spec.matches(key_col.value(i))) + } + }) + .collect(); + + arrow::compute::filter_record_batch(&batch, &mask) + .map_err(|e| CoreError::ReadFileSliceError(format!("filter_record_batch: {e}"))) +} +``` + +(Update existing imports at the top of the file: ensure `arrow_array::StringArray` is in scope — there's already `use arrow_array::{Array, RecordBatch, StringArray};` per the earlier read, so it is.) + +- [ ] **Step 4: Run tests; verify still pass** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core 2>&1 | tail -15` +Expected: all tests pass. With `key_filter_opt = None`, the helper is a no-op early-return. + +- [ ] **Step 5: Commit** + +```bash +git add crates/core/src/file_group/reader/mod.rs +git commit -m "apply keyFilterOpt as row filter on base parquet batches + +Mirrors Java HoodieAvroReaderContext.getFileRecordIterator (lines +218-228) "fall through to row-level filter" path. Parquet has no native +key predicate, so we filter via Arrow filter_record_batch on the +_hoodie_record_key column. No-op when key_filter_opt is None. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 2.6: Manual cross-check vs `readerContext_callstack.md` + +**Files:** None modified — verification only. + +**Reference:** `/home/ubuntu/operations/tasks/quantonMORScanSupport/scanSpecPushDown/Tasks/phase3_rust_fg_integration/migrateFilterOptCode/readerContext_callstack.md` + +- [ ] **Step 1: Re-read the call-stack doc** + +Open `~/operations/tasks/quantonMORScanSupport/scanSpecPushDown/Tasks/phase3_rust_fg_integration/migrateFilterOptCode/readerContext_callstack.md`. Focus on §3.1 (eager-during-construction stack) and §3.2 (lazy `getClosableIterator` stack). + +- [ ] **Step 2: For each `readerContext.*` line in the doc, identify the hudi-rs equivalent** + +Produce a checklist as the body of the Phase 2 commit message — for every line in the validation table (§5 of the doc), name the specific Rust `file:line` that does the same thing. Example: + +``` +Java line 105 readerContext.setHasLogFiles(...) ← rs file_group/reader/mod.rs:??? hudi_file_group_reader_constructor +Java line 113 readerContext.initRecordMerger(props) ← rs file_group/reader/mod.rs:??? init_record_merger +Java line 119-121 readerContext.setSchemaHandler(...) ← rs file_group/reader/mod.rs:??? line 246-253 +… (all 47 lines) +``` + +For lines without a Rust equivalent, document the gap explicitly. The keyFilterOpt-specific call sites for Phase 2 should be: +- Java `HoodieReaderContext.keyFilterOpt` → `crates/core/src/file_group/reader/reader_context.rs:???` +- Java `HoodieMergedLogRecordReader.createKeySpec(...)` → `crates/core/src/file_group/reader/key_spec.rs:???` +- Java `BaseHoodieLogRecordReader.scanInternal(Option, boolean)` → `crates/core/src/file_group/reader/log_record_reader.rs:371` +- Java `HoodieAvroReaderContext.getFileRecordIterator` (lines 218-228) → `crates/core/src/file_group/reader/mod.rs::apply_key_filter_to_batch` + +- [ ] **Step 3: If the cross-check uncovers a gap, file a follow-up note in the design spec or fix it inline** + +For Phase 2, the only required gaps to address are anything related to keyFilterOpt's reader-side flow. Other readerContext interactions (setSchemaHandler, getMergeMode, etc.) are outside Phase 2's scope but should be checked off as already-equivalent. + +- [ ] **Step 4: Save the checklist** + +Append the checklist to `docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md` under a new appendix `## Appendix A — Phase 2 cross-check vs readerContext_callstack.md`. Commit: + +```bash +git add docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md +git commit -m "phase 2 cross-check vs readerContext_callstack.md (appendix) + +Maps each Java readerContext interaction in the call-stack doc to its +hudi-rs file:line equivalent. Validates Phase 2 implementation parity +for the keyFilterOpt path. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Phase 3 — E2E smoke test + +### Task 3.1: Locate the v9 MOR fixture and write helper functions + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` (add helpers at the bottom) + +- [ ] **Step 1: Verify fixture availability** + +Run: +```bash +cd /home/ubuntu/ws3/hudi-rs && grep -rn "V9Mor8I4UCommitTime\|v9_mor_8i4u_commit_time" crates/test/src --include='*.rs' | head -10 +``` +Expected: confirms the fixture variant exists. Note the file path for the fixture (likely under `crates/test/data/...`). + +- [ ] **Step 2: Open the test fixture and find one file group** + +Run: +```bash +cd /home/ubuntu/ws3/hudi-rs && find crates/test -path '*v9_mor_8i4u_commit_time*' -type f 2>/dev/null | head -20 +``` + +Identify the partition path for `city=sf`, the base parquet file, and the log file(s). + +- [ ] **Step 3: Add helper functions to the test file** + +Open `crates/core/tests/file_group_reader_tests.rs` and append at the bottom (after the existing helpers and the city=sf / city=nyc tests). The new helper `read_file_group_with_key_filter` is a deliberate inline variant of the existing `read_file_group` (lines 69-121); we don't refactor `read_file_group` itself in this plan to keep the change surface small. + +```rust +// ============================================================================= +// Phase 3 helpers — keyFilterOpt e2e smoke test +// ============================================================================= + +use hudi_core::expression::predicates::predicates_factory; +use hudi_core::expression::{Expression, Literal, NameReference, Predicate}; +use std::sync::Arc as StdArc; + +/// Variant of `read_file_group` that accepts a `key_filter_opt`. +/// Mirrors the existing helper at lines 69-121, with the addition of +/// `reader_context.key_filter_opt = key_filter_opt;` on the ctx. +async fn read_file_group_with_key_filter( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + key_filter_opt: Option>, +) -> Result { + let (_hudi_configs, storage) = create_configs_and_storage(table_path).await?; + + let base_path = if base_file.is_empty() { + None + } else if partition.is_empty() { + Some(base_file.to_string()) + } else { + Some(format!("{}/{}", partition, base_file)) + }; + let log_paths: Vec = log_files + .iter() + .map(|lf| { + if partition.is_empty() { + lf.to_string() + } else { + format!("{}/{}", partition, lf) + } + }) + .collect(); + + let input_split = InputSplit::new( + base_path, + None, + log_paths, + partition.to_string(), + ); + + let mut reader_context = ReaderContext::empty(); + reader_context.latest_commit_time = "99991231235959999".to_string(); + reader_context.merge_mode = "COMMIT_TIME_ORDERING".to_string(); + reader_context.table_config.insert( + HudiTableConfig::PrecombineField.as_ref().to_string(), + "ts".to_string(), + ); + reader_context.rebuild_record_context(partition.to_string()); + reader_context.key_filter_opt = key_filter_opt; // ← the only addition + + let mut reader = HoodieFileGroupReader::new( + Arc::new(reader_context), + storage, + input_split, + ReaderParameters::default(), + None, + None, + ); + + reader.read().await +} + +/// Look up the `_hoodie_record_key` string for a given `id` in the baseline +/// batch. Used to derive the literal value for the test filter without +/// hard-coding the keygen encoding. +fn lookup_record_key(batch: &arrow_array::RecordBatch, id: i32) -> String { + let id_col_idx = batch.schema().index_of("id").expect("id column missing"); + let key_col_idx = batch + .schema() + .index_of("_hoodie_record_key") + .expect("_hoodie_record_key column missing"); + + let id_col = batch + .column(id_col_idx) + .as_any() + .downcast_ref::() + .expect("id column should be Int32"); + let key_col = batch + .column(key_col_idx) + .as_any() + .downcast_ref::() + .expect("_hoodie_record_key column should be StringArray"); + + for i in 0..batch.num_rows() { + if id_col.value(i) == id { + return key_col.value(i).to_string(); + } + } + panic!("id {id} not found in baseline batch"); +} + +/// Pull the row tuple `(id, name, age)` for the given id, or None. Reuses +/// the exact column projection of the existing `extract_id_name_age` helper. +fn extract_row_with_id_opt( + batch: &arrow_array::RecordBatch, + id: i32, +) -> Option<(i32, String, i32)> { + let id_col = batch.column_by_name("id")? + .as_any().downcast_ref::()?; + let name_col = batch.column_by_name("name")? + .as_any().downcast_ref::()?; + let age_col = batch.column_by_name("age")? + .as_any().downcast_ref::()?; + + for i in 0..batch.num_rows() { + if id_col.value(i) == id { + return Some(( + id_col.value(i), + name_col.value(i).to_string(), + age_col.value(i), + )); + } + } + None +} +``` + +Notes: +- The fixture's `_hoodie_record_key` column is exposed in the merged output because hudi-rs's reader does not strip meta columns. If a test run reveals it isn't present, the helper's `expect("_hoodie_record_key column missing")` will panic — that's a clear signal to investigate the schema rather than a silent bug. +- `extract_row_with_id_opt` returns `(id, name, age)` to match the shape used by the existing `extract_id_name_age` helper. + +- [ ] **Step 4: Compile-check the helpers** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests --no-run 2>&1 | tail -10` +Expected: clean build. + +- [ ] **Step 5: Commit** + +```bash +git add crates/core/tests/file_group_reader_tests.rs +git commit -m "add e2e test helpers for keyFilterOpt phase 3 smoke test + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 3.2: Write the e2e smoke test + +**Files:** +- Modify: `crates/core/tests/file_group_reader_tests.rs` + +- [ ] **Step 1: Add the test** + +Append to `crates/core/tests/file_group_reader_tests.rs`: + +```rust +/// Phase 3 e2e smoke test — keyFilterOpt actually filters rows. +/// +/// Reads partition `city=sf` of v9_mor_8i4u_commit_time twice: +/// 1. baseline (no filter) → expect 2 rows (id=1 V2 update, id=2 base) +/// 2. filtered (In on record-key) → expect 1 row (id=1) +/// +/// See keyFilterOpt design spec §7. +#[tokio::test] +async fn fg_reader_with_key_filter_filters_rows() -> Result<()> { + // Fixture filenames copied from the existing test_e2e_v9_mor_commit_time_sf_merge + // test (lines 218-223) so both tests stay in sync if the fixture is regenerated. + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.path_to_mor_avro(); + let partition = "city=sf"; + let base_file = "fee86b18-67b1-4479-b517-075683aeb2d1-0_0-13-33_20260408053032350.parquet"; + let log_files: Vec<&str> = vec![ + ".fee86b18-67b1-4479-b517-075683aeb2d1-0_20260408053037787.log.1_0-27-73", + ]; + + // ── Read 1: no filter (baseline) ──────────────────────────────── + let baseline_batch = read_file_group_with_key_filter( + &table_path, + partition, + base_file, + log_files.clone(), + None, + ).await?; + + assert_eq!(baseline_batch.num_rows(), 2, "city=sf should have 2 rows in baseline"); + + // Derive the actual _hoodie_record_key string for id=1 from the baseline batch. + // Avoids hard-coding the keygen encoding (id:1 vs 1 vs id=1). + let key_for_id1 = lookup_record_key(&baseline_batch, 1); + + // ── Read 2: filter with In(_hoodie_record_key, []) ──── + // Construction mirrors Java: Predicates.in(NameReference("_hoodie_record_key"), + // [Literal.from(key_for_id1)]) + let key_filter: StdArc = StdArc::new( + predicates_factory::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id1.clone())) as Box], + ), + ); + + let filtered_batch = read_file_group_with_key_filter( + &table_path, + partition, + base_file, + log_files.clone(), + Some(key_filter), + ).await?; + + assert_eq!(filtered_batch.num_rows(), 1, + "filtered read should return only id=1"); + + // ── Cross-validate row content ──────────────────────────────── + let expected = extract_row_with_id_opt(&baseline_batch, 1) + .expect("id=1 should exist in baseline"); + let actual = extract_row_with_id_opt(&filtered_batch, 1) + .expect("id=1 should exist in filtered"); + assert_eq!(expected, actual, "filtered id=1 row must equal baseline id=1 row"); + + // Negative assertion: id=2 (base-only) is NOT in filtered output. + assert!(extract_row_with_id_opt(&filtered_batch, 2).is_none(), + "id=2 must be excluded by the key filter"); + + Ok(()) +} +``` + +- [ ] **Step 2: Run the test** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test -p hudi-core --test file_group_reader_tests fg_reader_with_key_filter_filters_rows -- --nocapture 2>&1 | tail -30` +Expected: PASS. Inspect output for `[HoodieFileGroupReader]` debug logs that confirm the filter ran on both base and log paths. + +- [ ] **Step 3: Commit** + +```bash +git add crates/core/tests/file_group_reader_tests.rs +git commit -m "add Phase 3 e2e smoke test for keyFilterOpt + +Reads city=sf partition of v9_mor_8i4u_commit_time with and without +an In predicate on _hoodie_record_key. Asserts: + * baseline returns 2 rows + * filtered returns 1 row matching baseline content for id=1 + * id=2 (base-only, not in filter) is excluded + +End-to-end validation of the keyFilterOpt port: ReaderContext field, +log-scan KeySpec, base-file row filter all confirmed working. + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +### Task 3.3: Final verification and PR-ready commit + +**Files:** None modified — verification only. + +- [ ] **Step 1: Run the full test suite** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo test --workspace 2>&1 | tail -30` +Expected: all tests pass (existing + new). + +- [ ] **Step 2: Run clippy on the workspace** + +Run: `cd /home/ubuntu/ws3/hudi-rs && cargo clippy --workspace --all-targets 2>&1 | tail -30` +Expected: no errors. + +- [ ] **Step 3: Verify branch state** + +Run: `cd /home/ubuntu/ws3/hudi-rs && git log --oneline davis/phase3migrateFilterOptCode ^master 2>&1 | tail -30` +Expected: commit list shows all Phase 1, 2, 3 commits in order. No accidental amendments. + +- [ ] **Step 4: Final commit if any cleanup needed** + +If clippy or test fixes uncovered changes: + +```bash +git add -A +git commit -m "phase 3: final test cleanup + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +If no cleanup needed, no commit. + +--- + +## Phase 1 → 2 → 3 dependency notes + +- Phase 2 strictly requires Phase 1 (uses `Predicate`, `Literal`, `predicates_factory`). +- Phase 3 strictly requires Phase 2 (uses `key_filter_opt` field, `read_file_group_with_key_filter` helper). +- Within Phase 1, Tasks 1.6 → 1.10 → 1.11 → 1.12 → 1.13 → 1.14 are sequential because each task adds variants to `ExpressionKind` / `PredicateKind`. +- Within Phase 2, Tasks 2.1 → 2.2 → 2.3 → 2.4 → 2.5 → 2.6 are sequential. + +## Out of plan scope (already documented in spec §2) + +- FFI / `FfiReaderContext` integration. +- Engines other than Spark. +- Schema evolution. +- HFile reader for `HoodieAvroReaderContext` HFile branch. +- Per-record `Predicate.eval(StructLike)` runtime evaluation against scan-time rows. diff --git a/docs/superpowers/specs/2026-05-07-keyfilteropt-e2e-coverage-design.md b/docs/superpowers/specs/2026-05-07-keyfilteropt-e2e-coverage-design.md new file mode 100644 index 00000000..30c3eb9a --- /dev/null +++ b/docs/superpowers/specs/2026-05-07-keyfilteropt-e2e-coverage-design.md @@ -0,0 +1,170 @@ +# keyFilterOpt e2e test coverage expansion — design spec + +**Date:** 2026-05-07 +**Owner:** Davis Zhang +**Builds on:** `docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md` (the keyFilterOpt port that landed in 28 commits ending at `35d9603`) + +## 1. Goal + +Expand e2e test coverage of `key_filter_opt` so that each KeySpec variant is exercised, both call sites (log-scan filter via `KeyBasedFileGroupRecordBuffer::process_*`; base-file filter via `apply_key_filter_to_batch`) are independently validated, and every test proves the filter is **real**, not a no-op, via an explicit baseline-vs-filtered AB comparison. + +The current state (after the port): a single e2e test `fg_reader_with_key_filter_filters_rows` covers `In`/FullKeys on `V9Mor8I4UCommitTime` `city=sf`. It validates that filtering reduces 2 rows to 1, and that the surviving row matches the baseline content for `id=1`. Coverage gaps: + +1. `StringStartsWithAny` → `PrefixKeys` KeySpec path has zero e2e coverage. +2. The base-file filter (`apply_key_filter_to_batch`) and log-scan filter (`KeyBasedFileGroupRecordBuffer::process_*`) are exercised together but not isolated. A regression in either is currently undetectable independently. +3. "Filter is a no-op for unsupported predicates" (e.g. `eq`) is not regression-guarded. +4. Empty-filter-result corner case (filter matches nothing) is untested. +5. Delete-block path through `process_delete_block` is not exercised by any filter test. +6. Log-only file groups (no base file) are not exercised by any filter test. + +## 2. Scope + +In-scope: +- 8 new e2e tests across 3 fixtures. +- Shared AB-test helper (`FilterAbResult` + `ab_read_with_filter`) so every test consistently compares baseline vs filtered. +- Fixture locators for two new fixtures (`V9MorNonpart3Commits`, `MorLayoutLogOnly`). +- Schema-specific row extractors for fixtures whose columns differ from `V9Mor8I4UCommitTime`. + +Out-of-scope: +- COW tables (no log files; only base-file filter path, which is covered). +- Merge modes other than `COMMIT_TIME_ORDERING`. +- The `MorLayoutLogCompaction` fixture (compacted log block). The per-record filter applies the same way; redundant with tests 6-7. +- Multi-file-group reads (the reader operates on one file group at a time). +- Combined predicate trees (`And`/`Or`/`Not` wrapping `In`). `create_key_spec` only matches top-level `In` or `StringStartsWithAny`; combined trees route through the no-op path that test 5 covers. + +## 3. Test catalog + +8 tests, each AB (read with `key_filter_opt = None`, then with `Some(filter)`). + +### Group A — `V9Mor8I4UCommitTime`, partition `city=sf` + +| # | Test name | Filter | Expected | Validates | +|---|-----------|--------|----------|-----------| +| 1 | `fg_filter_in_log_updated_key` (renamed from existing `fg_reader_with_key_filter_filters_rows`) | `In([key_for_id1])` — id=1 is updated in log | filtered=1 row, value=Alice-V2 | Filter applies during BOTH log scan and base read; log update path. | +| 2 | `fg_filter_in_base_only_key` | `In([key_for_id2])` — id=2 has no log update | filtered=1 row, value=Bob (base) | **Isolates base-file filter** — log scan finds no records for this key. | +| 3 | `fg_filter_in_no_match` | `In(["nonexistent_key"])` | filtered=0 rows, baseline=2 | Filter actually drops everything when nothing matches; catches a "filter silently no-op" bug. | +| 4 | `fg_filter_starts_with_any_prefix` | `StringStartsWithAny([])` | filtered=1 row | Exercises `PrefixKeys` KeySpec path. | +| 5 | `fg_filter_unsupported_predicate_is_noop` | `eq(name_ref, lit("anything"))` (Eq → `create_key_spec` returns None) | filtered.num_rows == baseline.num_rows; row content equal | Regression guard: unsupported predicate must NOT drop rows. | + +### Group B — `V9MorNonpart3Commits` + +| # | Test name | Filter | Expected | Validates | +|---|-----------|--------|----------|-----------| +| 6 | `fg_filter_in_with_delete_block` | `In([key_for_id0])` — id=0 is deleted in log block | filtered=0 rows; baseline excludes id=0 too (id 0 deleted) | Filter passes through `process_delete_block` correctly. | +| 7 | `fg_filter_in_log_updated_key_nonpart` | `In([key_for_id5])` — id=5 updated in log | filtered=1 row with updated value | Same as test 1 on a non-partitioned + multi-log-file fixture. | + +### Group C — `MorLayoutLogOnly` + +| # | Test name | Filter | Expected | Validates | +|---|-----------|--------|----------|-----------| +| 8 | `fg_filter_in_log_only_filegroup` | `In([key_for_one_id])` | filtered ⊊ baseline | **Isolates log-scan filter** — no base file present, so all output flows through `KeyBasedFileGroupRecordBuffer::process_*` where Task 2.3's `key_spec_opt` filter lives. | + +## 4. Validation pattern + +Every test uses `FilterAbResult` for assertions. Helpers added at the bottom of `crates/core/tests/file_group_reader_tests.rs`: + +```rust +/// AB-test result. Used by every keyFilterOpt e2e test. +struct FilterAbResult { + baseline: arrow_array::RecordBatch, + filtered: arrow_array::RecordBatch, +} + +impl FilterAbResult { + /// Assert filter is real, not a no-op: filtered must have STRICTLY fewer rows than baseline. + fn assert_filter_narrowed(&self) { … } + + /// Assert filter is a no-op (used by test 5 for unsupported predicates). + fn assert_filter_was_noop(&self) { … } + + /// Assert that filtered's id-set is a subset of baseline's id-set. + fn assert_filtered_ids_subset(&self) { … } + + /// Assert the exact set of ids present in filtered (sorted). + fn assert_filtered_ids_eq(&self, expected_ids: &[i32]) { … } +} + +/// Shared driver: read once with None, once with Some(filter), return both. +async fn ab_read_with_filter( + table_path: &str, + partition: &str, + base_file: &str, + log_files: Vec<&str>, + filter: Arc, +) -> Result { … } +``` + +`assert_filter_was_noop` sorts both batches by id before comparing, since two reads of the same file group may produce rows in different orders. + +For test 3 (filter matches nothing), `assert_filter_narrowed` is augmented: assert `filtered.num_rows() == 0` AND `baseline.num_rows() > 0`. + +For tests on fixtures other than `V9Mor8I4UCommitTime`, schema-specific row extractors are added: +- `extract_row_with_id_opt_v9nonpart(batch, id) -> Option<(i32, String, f64, i64)>` — returns `(id, name, price, ts)` for `V9MorNonpart3Commits`. +- `extract_row_with_id_opt_log_only(batch, id) -> Option<(i32, ...)>` — schema TBD at impl time after inspecting the fixture. + +## 5. File layout + +Single file gets all the new tests, helpers, and locators (append only — no refactor of existing code): + +``` +crates/core/tests/file_group_reader_tests.rs (existing — append only) +├── (existing) read_file_group, helpers, ~17 existing tests +├── (existing Phase 3) read_file_group_with_key_filter, lookup_record_key, +│ extract_row_with_id_opt +├── fg_reader_with_key_filter_filters_rows ← RENAMED to fg_filter_in_log_updated_key +└── (NEW) + ├── FilterAbResult struct + impl (4 assertion methods) + ├── ab_read_with_filter(...) async fn + ├── 3 fixture locators: sf_file_group, nonpart_3commits_file_group, log_only_file_group + ├── 2 schema-specific extractors: extract_row_with_id_opt_v9nonpart, + │ extract_row_with_id_opt_log_only + ├── tests 1-5 on V9Mor8I4UCommitTime city=sf + ├── tests 6-7 on V9MorNonpart3Commits + └── test 8 on MorLayoutLogOnly +``` + +The existing `fg_reader_with_key_filter_filters_rows` is renamed to `fg_filter_in_log_updated_key` (test 1) so naming is consistent across the new set. Its body is updated to use `FilterAbResult` for symmetry, but its assertions are unchanged. + +## 6. Risks + +1. **Fixture filename discovery.** Two new fixtures (`v9_mor_nonpart_3commits`, `table_log_only`) need filenames resolved at impl time via: + ```bash + find crates/test/data/tables/v9_mor_nonpart_3commits -type f | grep -v '\.hoodie' + find crates/test/data/tables/table_log_only -type f | grep -v '\.hoodie' + ``` + Followed by a one-shot read of the baseline batch with column/schema inspection before writing assertions. + +2. **`_hoodie_record_key` column may not be present.** The existing test relies on it. If a new fixture uses virtual keys / different keygen, the lookup field changes. The pre-write check (read baseline + inspect schema) catches this. + +3. **`id` and other column names may differ across fixtures.** + - `V9Mor8I4UCommitTime`: `id INT, name STRING, age INT, ts STRING, city STRING`. + - `V9MorNonpart3Commits`: `id INT, name STRING, price DOUBLE, ts LONG`. + - `MorLayoutLogOnly`: schema TBD at impl time. + + Per-fixture extractors handle this. If `id` is missing on `MorLayoutLogOnly`, fall back to whatever unique-id-like column exists (or use `_hoodie_record_key` itself for row identity). + +4. **Test 6 (delete block) baseline assumption.** `V9MorNonpart3Commits` deletes ids 0/1/2 in log. Baseline (no filter) should NOT contain id=0. Filtered with `In([key_for_id0])` should yield 0 rows. **Open question:** how do we look up `key_for_id0` if id=0 isn't in the merged output? Answer: read the **base file directly** (separately, without log merge) to find id=0's `_hoodie_record_key`, OR use the deterministic key encoding (e.g. just `"0"` if SimpleKeyGenerator). At impl time, if the lookup is awkward, simplify by using `lit("0")` directly and asserting that this produces 0 rows. The point of test 6 is to validate the filter passes through the delete-block code path, not to recover deleted-key lookups. + +5. **Order non-determinism in `assert_filter_was_noop`.** Two reads of the same file group may emit rows in different orders. Sort by id before equality comparison. + +6. **Adding new fixtures might require unzipping or generating data.** If `crates/test/data/tables/v9_mor_nonpart_3commits` or `crates/test/data/tables/table_log_only` doesn't exist as expected, the corresponding tests get `#[ignore]` with a TODO comment + follow-up filed. Plan task 8 (final verification) flags this as a known incomplete. + +## 7. Implementation order + +Three phases, each independently testable: + +| Phase | Tasks | +|-------|-------| +| A | Add `FilterAbResult` + `ab_read_with_filter` helper. Refactor existing test to use them. Verify still passes. | +| B | Add tests 2-5 on `V9Mor8I4UCommitTime` (existing fixture, no new locator needed). | +| C | Add `nonpart_3commits_file_group` locator + `extract_row_with_id_opt_v9nonpart` + tests 6-7. | +| D | Add `log_only_file_group` locator + extractor + test 8. | + +Phases are sequential; each ends with a commit. If a phase hits a fixture issue (e.g., column missing), mark its test `#[ignore]` and proceed. + +## 8. Out-of-plan future work + +- Combined-predicate coverage (`And(In, In)`, `Or(In, In)`) once `create_key_spec` is extended to handle them. +- Bootstrap merge filter coverage (currently rejected at construction time per `HoodieFileGroupReader::new` panic). +- COW-table filter coverage when COW write path lands. +- Filter pushdown via the FFI bridge (currently out of scope per the original port spec). diff --git a/docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md b/docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md new file mode 100644 index 00000000..6e783680 --- /dev/null +++ b/docs/superpowers/specs/2026-05-07-keyfilteropt-port-design.md @@ -0,0 +1,386 @@ +# keyFilterOpt port — design spec + +**Date:** 2026-05-07 +**Owner:** Davis Zhang +**Source repo:** `/home/ubuntu/ws3/hudi-rs` +**Reference (Java):** `/home/ubuntu/ws3/hudi-internal` +**Call-stack reference:** `/home/ubuntu/operations/tasks/quantonMORScanSupport/scanSpecPushDown/Tasks/phase3_rust_fg_integration/migrateFilterOptCode/readerContext_callstack.md` + +## 1. Goals + +Bring hudi-rs's `ReaderContext` to parity with Java's `HoodieReaderContext.keyFilterOpt`. The work has three goals: + +1. Port the entire `org.apache.hudi.expression` package (Predicate hierarchy) into hudi-rs. +2. Port the `org.apache.hudi.internal.schema.{Type, Types}` Type system the Predicate hierarchy depends on. Schema-evolution code (`InternalSchema`, `InternalSchemaBuilder`, `AvroInternalSchemaConverter`, schema-evolution actions/visitors, `SchemaChangeUtils`, etc.) is **out of scope**. +3. Add a `key_filter_opt: Option>` field on hudi-rs's `ReaderContext` and wire it through the file-group reader so a key-based In-predicate actually filters output rows end-to-end on a v9 MOR + COMMIT_TIME_ORDERING table. + +The principle is faithful structural mirroring: same Rust class, same fn name, same arg order, same observable side-effect as Java, except where Rust's type system forces a deviation (called out explicitly in §6). + +## 2. Boundaries + +In-scope: +- Spark-only assumption. +- Table version 9, MOR, COMMIT_TIME_ORDERING. +- Predicate hierarchy (Predicate, Expression, Predicates with all 12 inner classes — TrueExpression, FalseExpression, And, Or, Not, BinaryComparison, In, IsNull, IsNotNull, StringStartsWith, StringStartsWithAny, StringContains; Literal, NameReference, BinaryExpression, LeafExpression, ExpressionVisitor, BoundReference, BindVisitor, PartialBindVisitor, Comparators, ArrayData, StructLike). +- `Type` / `Types` full port from `internal.schema`. + +Out-of-scope: +- FFI / `FfiReaderContext` integration. +- Schema evolution (no `InternalSchema`, `AvroInternalSchemaConverter`, no schema-change actions/visitors, no `SchemaChangeUtils`). +- Engines other than Spark (no Flink/Trino/Hive ReaderContext changes). +- Merge modes other than COMMIT_TIME_ORDERING. +- `HoodieAvroReaderContext.getFileRecordIterator` HFile-supporting branch (no HFile reader in hudi-rs). +- `ReusableKeyBasedRecordBuffer` (metadata-table buffer). +- Per-record `Predicate.eval(StructLike)` evaluation against scan-time rows. The hierarchy ports `eval` so unit tests pass and behavior matches Java, but the e2e reader path uses Arrow `filter_record_batch` for the actual filtering. + +## 3. Phases + +The work lands in three independently-reviewable commits. + +| Phase | Deliverable | Verification | +|------|------|------| +| 1 | Port `org.apache.hudi.expression` (Predicate hierarchy) + full port of `org.apache.hudi.internal.schema.{Type, Types}`. **No reader integration yet.** | Unit tests on each Predicate / Type, plus structural pattern-match tests (`predicates::in_(...)` produces `predicates::In` with the right children). | +| 2 | Add `key_filter_opt: Option>` on `ReaderContext`. Add `KeySpec` enum + `create_key_spec()` (Java input/output preserved; body uses Rust `match` on `PredicateKind` instead of Java's `instanceof`/cast — see §6 deviation #2). Apply the filter at two call sites: (a) `merged_log_record_reader::perform_scan` → KeySpec → narrow log scan; (b) `HoodieFileGroupReader::make_base_file_batches` → row filter on base parquet (mirroring `HoodieAvroReaderContext.getFileRecordIterator`'s key-extraction path). | Unit tests on `create_key_spec`, plus per-site tests, plus a manual cross-check pass against `readerContext_callstack.md` confirming each call site listed there has a 1:1 hudi-rs equivalent. The cross-check produces a checklist in the Phase 2 PR description that names specific Rust `file:line` for each Java line. | +| 3 | One e2e smoke test on `v9_mor_8i4u_commit_time` reading a single file group with and without `Predicates.In(_hoodie_record_key, [k])`. Compare row count + row content. | Single integration test in `crates/core/tests/file_group_reader_tests.rs`. | + +Phase 1 is purely additive — it compiles standalone and adds two new top-level modules. Phase 2 wires the field in but defaults to `None` so all existing tests keep passing. Phase 3 demonstrates the end-to-end contract. + +## 4. Phase 1 — file layout & module structure + +Two new module trees under `crates/core/src/`: + +``` +crates/core/src/ +├── internal_schema/ ← NEW (mirrors org.apache.hudi.internal.schema) +│ ├── mod.rs +│ ├── type_.rs ← `Type` trait + Type::PrimitiveType / Type::NestedType (Type.java in full) +│ └── types.rs ← all concrete types from Types.java (full port) +│ +└── expression/ ← NEW (mirrors org.apache.hudi.expression) + ├── mod.rs + ├── expression.rs ← Expression trait + Operator enum + ├── predicate.rs ← Predicate trait + PredicateKind enum + ├── leaf_expression.rs ← LeafExpression trait + ├── binary_expression.rs ← BinaryExpression + ├── literal.rs ← Literal + LiteralValue enum + ├── name_reference.rs ← NameReference + ├── bound_reference.rs ← BoundReference + ├── struct_like.rs ← StructLike trait + ├── array_data.rs ← ArrayData + ├── comparators.rs ← Comparators::for_type + ├── expression_visitor.rs ← ExpressionVisitor trait + ├── bind_visitor.rs ← BindVisitor + ├── partial_bind_visitor.rs ← PartialBindVisitor + └── predicates.rs ← Predicates factory + 12 inner classes (TrueExpression, FalseExpression, And, Or, Not, BinaryComparison, In, IsNull, IsNotNull, StringStartsWith, StringStartsWithAny, StringContains). One file mirrors Java's single-file shape. +``` + +`expression.rs` also defines an `ExpressionKind<'a>` enum (a borrowed view) alongside the `Expression` trait. `predicate.rs` defines `PredicateKind<'a>` alongside `Predicate`. See §4.1 for the rationale. + +Naming choices: + +- Java package `org.apache.hudi.expression` → Rust `expression` (snake_case). +- Java package `org.apache.hudi.internal.schema` → Rust `internal_schema`. +- Java `Predicates.In` → Rust `predicates::In` (struct in module). Factory functions `predicates::in_(...)`, `predicates::and(...)`, `predicates::eq(...)`, etc. (`in` is a Rust keyword, hence the trailing underscore.) +- Polymorphism: Java `interface Expression` becomes Rust `trait Expression: Send + Sync + Debug` (with a `kind() -> ExpressionKind<'_>` accessor — see §4.1). The 12 Predicate variants become concrete structs implementing `Predicate: Expression` (with `kind() -> PredicateKind<'_>`). Anywhere Java passes `Expression` or `Predicate`, Rust uses `Box` / `Arc`. +- Java `Comparator` returned by `Comparators` becomes a Rust `fn(&LiteralValue, &LiteralValue) -> std::cmp::Ordering` returned by `Comparators::for_type(&Type)`. +- Visitor pattern: Java `ExpressionVisitor` → Rust trait with one method per `visit*` overload. `BindVisitor` and `PartialBindVisitor` become structs implementing `ExpressionVisitor>`. + +### 4.1 Pattern-match accessors (`PredicateKind` / `ExpressionKind`) + +The Predicate hierarchy is a *closed* set of variants in Hudi (12 Predicate types, 4 Expression non-Predicate types: Literal, NameReference, BoundReference, plus Predicate-as-Expression). Rust handles closed variant sets best with pattern matching, not `Any::downcast_ref`. To keep the Java-faithful trait hierarchy *and* give downstream Rust code an idiomatic inspection point, we add two `kind()` accessors that return borrowed enum views: + +```rust +pub trait Predicate: Expression + Send + Sync + Debug { + /// Mirrors Java `Predicate.getOperator()`. + fn operator(&self) -> Operator; + /// Borrowed enum view for pattern matching. + fn kind(&self) -> PredicateKind<'_>; +} + +pub enum PredicateKind<'a> { + True, + False, + And(&'a predicates::And), + Or(&'a predicates::Or), + Not(&'a predicates::Not), + BinaryComparison(&'a predicates::BinaryComparison), + In(&'a predicates::In), + IsNull(&'a predicates::IsNull), + IsNotNull(&'a predicates::IsNotNull), + StringStartsWith(&'a predicates::StringStartsWith), + StringStartsWithAny(&'a predicates::StringStartsWithAny), + StringContains(&'a predicates::StringContains), +} + +pub trait Expression: Send + Sync + Debug { + fn data_type(&self) -> &Type; + fn kind(&self) -> ExpressionKind<'_>; + // … other Java-faithful methods +} + +pub enum ExpressionKind<'a> { + Literal(&'a Literal), + NameReference(&'a NameReference), + BoundReference(&'a BoundReference), + Predicate(&'a dyn Predicate), +} +``` + +Each concrete struct's `kind()` is one line: `fn kind(&self) -> PredicateKind<'_> { PredicateKind::In(self) }`. Adding a new variant means adding one enum variant + one `kind()` method, and the compiler points at every match site that needs to handle it. + +`Literal` uses a `LiteralValue` enum for its value field: + +```rust +pub struct Literal { + value: LiteralValue, + data_type: Type, +} + +pub enum LiteralValue { + Bool(bool), + Int(i32), + Long(i64), + Float(f32), + Double(f64), + Date(i32), + Time(i64), + Timestamp(i64), + TimestampMillis(i64), + LocalTimestampMillis(i64), + LocalTimestampMicros(i64), + Decimal { unscaled: i128, precision: u8, scale: u8 }, + String(String), + Binary(Vec), + Fixed(Vec), + UUID([u8; 16]), + Null, +} +``` + +This is the one place where Rust shape diverges from Java (Java has `Literal(T value, Type type)`). The reason is a chain: `kind()` returns `ExpressionKind::Literal(&Literal)` — to extract the value via pattern matching without a second `downcast_ref`, the value field has to be a closed enum. Behavior is identical; the type-parameter shape is the only deviation. + +Crate exports: both `internal_schema` and `expression` are re-exported from `crates/core/src/lib.rs` so downstream code can import at `hudi_core::expression::Predicate` / `hudi_core::internal_schema::Type`. + +No changes to `crates/core/src/expr/filter.rs` and its callers — that is the file-pruning predicate path and remains separate per the explicit decision to not converge the two abstractions. + +## 5. Phase 2 — keyFilterOpt integration call sites + +### 5.1 Field on `ReaderContext` + +```rust +// crates/core/src/file_group/reader/reader_context.rs +pub struct ReaderContext { + // … existing fields … + /// Mirrors Java `HoodieReaderContext.keyFilterOpt`. + /// Constructor-set, never mutated. + pub key_filter_opt: Option>, +} + +impl ReaderContext { + /// Mirrors Java `HoodieReaderContext.getKeyFilterOpt()`. + pub fn get_key_filter_opt(&self) -> Option> { … } +} +``` + +`ReaderContext::empty()` and the FFI bridge initialize `key_filter_opt = None`, matching Java's `Option.empty()` default in `BaseSparkInternalRowReaderContext`. + +### 5.2 Call site A — log scan + +New file `crates/core/src/file_group/reader/key_spec.rs`: + +```rust +pub enum KeySpec { + /// Mirrors Java FullKeySpec — exact-match key list (from Predicates::In). + FullKeys(Vec), + /// Mirrors Java PrefixKeySpec — prefix list (from Predicates::StringStartsWithAny). + PrefixKeys(Vec), +} + +/// Mirrors `HoodieMergedLogRecordReader.createKeySpec(Option)` +/// (HoodieMergedLogRecordReader.java lines 109-126), implemented in idiomatic +/// Rust via `PredicateKind` pattern matching instead of Java's `instanceof`/cast. +pub fn create_key_spec(filter: Option<&dyn Predicate>) -> Option { + match filter?.kind() { + PredicateKind::In(p) => { + Some(KeySpec::FullKeys(string_literals(p.right_children())?)) + } + PredicateKind::StringStartsWithAny(p) => { + Some(KeySpec::PrefixKeys(string_literals(p.right_children())?)) + } + _ => None, + } +} + +/// Extract `String` values from a list of `Literal(LiteralValue::String(_))` +/// expressions. Returns `None` if any child is not a String literal. +fn string_literals(exprs: &[Box]) -> Option> { + exprs.iter().map(|e| match e.kind() { + ExpressionKind::Literal(lit) => match lit.value() { + LiteralValue::String(s) => Some(s.clone()), + _ => None, + }, + _ => None, + }).collect() +} +``` + +The Rust match-on-`kind()` is the structural equivalent of Java's `if (filter.get().getOperator() == Expression.Operator.IN) { ... } else if (filter.get().getOperator() == Expression.Operator.STARTS_WITH) { ... }` plus the cast on the next line. Same input, same output, idiomatic Rust on the inside. + +`merged_log_record_reader::perform_scan`: + +- Drops the existing TODO ("KeySpec filtering not yet implemented in Rust"). +- Calls `create_key_spec(reader_context.get_key_filter_opt())`. +- Threads the resulting `Option` into `BaseHoodieLogRecordReader::scan_internal(key_spec_opt, skip_processing_blocks)`. + +`BaseHoodieLogRecordReader::scan_internal` signature changes from `scan_internal(skip: bool)` to `scan_internal(key_spec_opt: Option, skip: bool)`. All existing non-`perform_scan` call sites pass `None` for `key_spec_opt`. Inside `scan_internal`, the per-record processing in `process_data_block` / `process_delete_block` consults `KeySpec` and skips records whose key isn't in the FullKeys set / doesn't have a matching prefix in PrefixKeys — mirroring Java `BaseHoodieLogRecordReader.scanInternal` filtering. + +### 5.3 Call site B — base file + +In `HoodieFileGroupReader::make_base_file_batches`, after the parquet read returns its batches, apply a row-level filter when `reader_context.key_filter_opt.is_some()`. The filter materializes the keys/prefixes via the same `create_key_spec` (or directly extracts `Predicates::In::right_children` / `Predicates::StringStartsWithAny::right_children`) and uses Arrow `arrow::compute::filter_record_batch` against the `_hoodie_record_key` column. This mirrors Java's "fall through to row-level filter when reader doesn't `supportKeyPredicate`" — we always go through that path because Parquet has no native key predicate. + +The shared key-extraction helper lives on `key_spec.rs` so both sites use the same code. + +### 5.4 No `HoodieFileGroupReader::Builder` changes + +No new builder method — `key_filter_opt` is set on `ReaderContext` *before* the reader is constructed, exactly mirroring Java where `keyFilterOpt` is a constructor arg of `HoodieReaderContext`, not of `HoodieFileGroupReader`. + +### 5.5 Manual cross-check (Phase 2 verification) + +As part of the Phase 2 PR, walk through `readerContext_callstack.md` line by line and produce a checklist in the PR description: for each `readerContext.*` line in the doc that touches `keyFilterOpt` directly or indirectly, confirm hudi-rs has the same call in the same Rust class/function with the same args. Specifically the doc's §3.1 (eager-during-construction stack) and §3.2 (lazy `getClosableIterator` stack). + +Each verification line must name the specific Rust `file:line`, not just "yes, equivalent". + +## 6. Deviations from Java structure + +These are the only places where the Rust port intentionally differs from Java. Each is called out so future readers don't get confused: + +1. **`Literal` is non-generic; value is a `LiteralValue` enum.** Java has `Literal(T value, Type type)`; Rust has `Literal { value: LiteralValue, data_type: Type }`. The reason is the `kind()`-driven inspection pattern (deviation #2): when downstream code matches `ExpressionKind::Literal(lit)` and then needs to read the underlying value, a closed `LiteralValue` enum lets it pattern-match directly without a second `downcast_ref`. Behavior is identical; only the type-parameter shape differs. +2. **`kind() -> {Predicate,Expression}Kind<'_>` accessors.** Not present in Java. Added in Rust because the canonical Rust idiom for inspecting a closed set of variants is enum pattern matching, not `Any::downcast_ref` on a trait object. Each concrete struct's `kind()` is one line returning `Self`-as-enum-variant. Lets `create_key_spec` (and any future inspection code) be a clean `match` instead of a chain of `instanceof`-style downcasts. The trait hierarchy itself stays Java-faithful — `kind()` is pure inspection sugar that returns borrowed references into the existing concrete types. +3. **`Predicate` ownership.** Java passes `Predicate` references; Rust uses `Arc` so predicates can be cloned into ReaderContext, scanners, and buffers without lifetime juggling. Concrete impls must avoid `Rc` (the trait bound `Send + Sync` enforces this). +4. **`Comparator` return.** Java `Comparators::forType` returns `Comparator`; Rust returns `fn(&LiteralValue, &LiteralValue) -> std::cmp::Ordering`. Same dispatch table, different language idiom. +5. **`expr/filter.rs` is not converged.** Java has only one expression abstraction. Rust keeps `Filter` (the existing flat file-pruning struct) separate from the new `Predicate` hierarchy, since they serve different purposes (file pruning vs key filtering) and converging them is out of scope. +6. **`BindVisitor` / `PartialBindVisitor` ported but not wired.** They produce `BoundReference`s from `NameReference`s given a schema; nothing in keyFilterOpt's reader-side consumption needs binding (we extract literal values directly). They are ported for completeness and hierarchy faithfulness. + +## 7. Phase 3 — E2E smoke test + +**Location:** `crates/core/tests/file_group_reader_tests.rs` (existing file, add one new `#[tokio::test]`). + +**Fixture:** `QuickstartTripsTable::V9Mor8I4UCommitTime`. Pick partition `city=sf` which has one updated id (id=1) and one not-updated id (id=2). With both base + log content, this is the smallest interesting file group. + +**Two reads on the same file group:** + +```rust +#[tokio::test] +async fn fg_reader_with_key_filter_filters_rows() -> Result<()> { + let table_path = QuickstartTripsTable::V9Mor8I4UCommitTime.url_to_mor_avro(); + let (configs, storage) = create_configs_and_storage(&table_path).await?; + let (partition, base_file, log_files) = locate_sf_file_group(&storage).await?; + + // ── Read 1: no filter (baseline) ──────────────────────────────── + let baseline_ctx = build_reader_context(/* key_filter_opt = */ None, &configs); + let baseline_batch = run_reader(baseline_ctx, …).await?; + assert_eq!(baseline_batch.num_rows(), 2); // id=1 (V2 from log) + id=2 (from base) + + // Look up the actual _hoodie_record_key string for id=1 from the baseline + // batch so the filter literal matches whatever the keygen produced. + let key_for_id1 = lookup_record_key(&baseline_batch, 1); + + // ── Read 2: filter with In(_hoodie_record_key, []) ─────── + // Construction mirrors Java: Predicates.in(NameReference("_hoodie_record_key"), + // [Literal.from(key_for_id1)]) + let key_filter: Arc = Arc::new(predicates::in_( + Box::new(NameReference::new("_hoodie_record_key")), + vec![Box::new(Literal::string(key_for_id1.clone()))], + )); + let filtered_ctx = build_reader_context(Some(key_filter), &configs); + let filtered_batch = run_reader(filtered_ctx, …).await?; + assert_eq!(filtered_batch.num_rows(), 1); + + // ── Cross-validate: the filtered row equals the matching baseline row. + let expected = extract_row_with_id(&baseline_batch, 1); + let actual = extract_row_with_id(&filtered_batch, 1); + assert_eq!(expected, actual); + + // Negative assertion: id=2 (base-only) is NOT in filtered output. + assert!(extract_row_with_id_opt(&filtered_batch, 2).is_none()); + + Ok(()) +} +``` + +**Helper functions** added to the same test file (no shared test util churn): + +- `locate_sf_file_group(&storage)` — finds the latest base-file + log-files for partition `city=sf`. Reuses existing `Table::file_slices(...)` if available, otherwise walks `.hoodie/timeline`. +- `build_reader_context(key_filter_opt, configs)` — constructs `ReaderContext` with `merge_mode=COMMIT_TIME_ORDERING`, `latest_commit_time` from configs, partition path = `city=sf`, populated `table_config`, and the requested `key_filter_opt`. +- `run_reader(ctx, …)` — wraps the existing `read_file_group` helper but takes a pre-built `ReaderContext`. +- `lookup_record_key(batch, id) -> String` — pulls the `_hoodie_record_key` string for a given `id` from the baseline batch. Avoids hard-coding the keygen encoding (`id:1` vs `1` vs `id=1`). +- `extract_row_with_id` / `extract_row_with_id_opt` — small Arrow helpers comparing row tuples. + +**One test, not multiple variants.** This is a smoke test for the integration. Comprehensive Predicate / KeySpec coverage is in Phase 1 + Phase 2 unit tests. + +## 8. Risks and assumptions + +1. **Record-key encoding ambiguity.** The exact `_hoodie_record_key` string format for `V9Mor8I4UCommitTime` is determined at test-write time by reading one baseline row. If the fixture is regenerated with a different key generator, the literal in the test breaks — kept tight by deriving the literal from the baseline read via `lookup_record_key`, not by hard-coding it. +2. **`BaseHoodieLogRecordReader::scan_internal` signature change** breaks any internal callers. Audit all call sites at Phase 2 implementation time and update each to pass `None` for the new `key_spec_opt` parameter. +3. **Manual cross-check discipline.** The walkthrough against `readerContext_callstack.md` is valuable but only if each verification line names a specific Rust `file:line`. Vague "yes, equivalent" entries defeat the point. +4. **Open assumption — populate_meta_fields.** The v9 MOR fixture is assumed to populate `_hoodie_record_key`. If virtual keys are in play we'd need to filter on the actual key field name; we'll verify when writing the test by inspecting the baseline batch. +5. **Trait-object hygiene.** `Arc` requires concrete impls to be `Send + Sync` and avoid `Rc`. Standard Rust trait-object hygiene; called out so reviewers don't get surprised by `Rc<…>` showing up later. + +--- + +## Appendix A — Phase 2 cross-check vs readerContext_callstack.md + +This appendix maps each Java `readerContext.*` interaction in +`/home/ubuntu/operations/tasks/quantonMORScanSupport/scanSpecPushDown/Tasks/phase3_rust_fg_integration/migrateFilterOptCode/readerContext_callstack.md` +to its hudi-rs file:line equivalent, validating that Phase 2 of the +keyFilterOpt port preserves structural parity for the keyFilterOpt +flow. + +**Date of cross-check:** 2026-05-07 +**Phase 2 SHA range:** `08c5ff7` .. `021960f` + +Commits in range (oldest → newest): +- `3780b8b` — add KeySpec + create_key_spec for keyFilterOpt log-scan path +- `40a5d87` — add key_filter_opt field to ReaderContext +- `540ff15` — scan_internal accepts Option; buffer applies key-spec filter +- `86ff53e` — wire create_key_spec into HoodieMergedLogRecordReader::perform_scan +- `021960f` — apply keyFilterOpt as row filter on base parquet batches + +### keyFilterOpt-relevant interactions (must match) + +| # | Java site | Rust file:line equivalent | Notes | +|---|-----------|---------------------------|-------| +| 1 | `HoodieReaderContext.keyFilterOpt` field declaration (`HoodieReaderContext.java:81`) | `crates/core/src/file_group/reader/reader_context.rs:90` — `pub key_filter_opt: Option>` | Direct structural equivalent. Java uses `Option` (Avro/OSS abstraction); Rust uses `Option>`. Arc needed for trait-object sharing across async call sites. | +| 2 | `HoodieReaderContext.getKeyFilterOpt()` (`HoodieReaderContext.java:221-222`) | `crates/core/src/file_group/reader/reader_context.rs:175` — `pub fn get_key_filter_opt(&self) -> Option>` | Method signature and semantics match. Returns a clone of the Arc (cheap) rather than a borrow, to avoid lifetime entanglement with callers. | +| 3 | `HoodieReaderContext` ctor sets `keyFilterOpt = Option.empty()` for Spark FileFormat path (`BaseSparkInternalRowReaderContext`) | `crates/core/src/file_group/reader/reader_context.rs:206` — `ReaderContext::empty()` initializes `key_filter_opt: None`. FFI bridge at `cpp/context.rs` excluded per spec §2. | Initialisation to `None` matches Java's `Option.empty()`. The Java ctor receiving a non-empty value (incremental scan path) maps to callers explicitly setting `key_filter_opt` on the struct before passing it into the reader. | +| 4 | `HoodieMergedLogRecordReader.createKeySpec(Option)` (`HoodieMergedLogRecordReader.java:109-126`) | `crates/core/src/file_group/reader/key_spec.rs:44` — `pub fn create_key_spec(filter: Option<&dyn Predicate>) -> Option` and `key_spec.rs:71` — `pub fn create_key_spec_from_arc(filter: &Option>) -> Option` | Java uses `instanceof` + cast (`Predicate instanceof In`, `instanceof StringStartsWith`) to route to `FullKeys` vs `PrefixKeys`. Rust uses `pred_kind()` pattern match (`PredicateKind::In`, `PredicateKind::StringStartsWithAny`) — same routing logic, idiomatic Rust. Spec §6 deviation #2 documents this. Non-`In`/non-`StringStartsWithAny` predicates return `None` in both Java and Rust. | +| 5 | `HoodieMergedLogRecordReader.performScan` calls `createKeySpec(readerContext.getKeyFilterOpt())` then `scanInternal(keySpec, false)` (`HoodieMergedLogRecordReader.java:95-107`) | `crates/core/src/file_group/reader/merged_log_record_reader.rs:133` — `async fn perform_scan` calls `create_key_spec_from_arc(&self.base.reader_context.key_filter_opt)` at line 137-139, then `self.base.scan_internal(key_spec_opt, false).await` at line 141 | Structural parity: same two-step pattern (derive KeySpec from filter, then call scan_internal). Rust accesses `key_filter_opt` directly on the struct (no getter call needed since perform_scan is in the same module); Java goes through `getKeyFilterOpt()`. Equivalent. | +| 6 | `BaseHoodieLogRecordReader.scanInternal(Option, boolean)` filters records by KeySpec inside per-record processing (`BaseHoodieLogRecordReader.java`) | `crates/core/src/file_group/reader/log_record_reader.rs:376` — `pub async fn scan_internal(&mut self, key_spec_opt: Option, skip_processing_blocks: bool)`. Propagates to buffer at line 395 via `self.record_buffer.set_key_spec(self.key_spec_opt.clone())`. Buffer applies filter at `crates/core/src/file_group/reader/buffer/key_based.rs:235-236` (`process_data_block`, `if !spec.matches(&key) { continue }`) and `key_based.rs:307-308` (`process_delete_block`) | Full structural parity: `scan_internal` stores `key_spec_opt` on self (line 382), propagates to buffer (line 395), buffer applies `spec.matches(&key)` gate per record in both data-block (line 235) and delete-block (line 307) paths. Java's equivalent is threaded through `KeyBasedFileGroupRecordBuffer.processDataBlock` / `processDeleteBlock`. | +| 7 | `HoodieAvroReaderContext.getFileRecordIterator` lines 218-228 (key-extract path, fall-through row filter for non-HFile readers) | `crates/core/src/file_group/reader/mod.rs:498` — `apply_key_filter_to_batch` called from `make_base_file_batches` (defined at line 456). `apply_key_filter_to_batch` itself at `mod.rs:745` | Java's HFile branch is out of scope (no HFile reader in hudi-rs); Rust always takes the row-filter fall-through path. `apply_key_filter_to_batch` calls `create_key_spec(key_filter_opt)` then row-filters the Arrow `RecordBatch` on `_hoodie_record_key` — same semantics as Java's per-row `recordKey` extraction + `keyFilterOpt.test(recordKey)`. The `make_base_file_batches` → `apply_key_filter_to_batch` chain at line 342/456 mirrors Java's `makeBaseFileIterator` → `getFileRecordIterator`. | + +**Summary:** No gaps found. All 7 keyFilterOpt-relevant Java interactions have confirmed Rust `file:line` equivalents. The two design deviations noted in spec §6 (Arc wrapping, `pred_kind()` dispatch vs `instanceof`) are intentional and preserve semantics. + +### Other readerContext interactions (informational — outside Phase 2 scope) + +The table below accounts for all non-keyFilterOpt `readerContext.*` call sites listed in §3.1, §3.2, and §5 of the call-stack doc. "Equivalent" means hudi-rs has a structurally corresponding field/method; "no equivalent" means the concept does not exist in hudi-rs (typically engine-specific Spark plumbing). No audit depth is warranted for Phase 2. + +| Java call-stack site | Rust equivalent? | Where | +|---|---|---| +| `readerContext.setHasLogFiles(...)` / `getHasLogFiles()` | Equivalent | `ReaderContext.has_log_files: bool` (`reader_context.rs:61`) | +| `readerContext.getRecordContext().setPartitionPath(...)` | Equivalent | `ReaderContext.record_context.partition_path` (`reader_context.rs:79`; set via `rebuild_record_context`) | +| `readerContext.initRecordMerger(props)` | No equivalent | hudi-rs uses DataFusion merge pipeline, not a Java `HoodieRecordMerger` | +| `readerContext.setTablePath(tablePath)` | Equivalent | `ReaderContext.table_path: String` (`reader_context.rs:58`) | +| `readerContext.setLatestCommitTime(latestCommitTime)` | Equivalent | `ReaderContext.latest_commit_time: String` (`reader_context.rs:59`) | +| `readerContext.setShouldMergeUseRecordPosition(...)` | Equivalent | `ReaderContext.should_merge_use_record_position: bool` (`reader_context.rs:64`) | +| `readerContext.setHasBootstrapBaseFile(...)` | Equivalent | `ReaderContext.has_bootstrap_base_file: bool` (`reader_context.rs:62`) | +| `readerContext.setSchemaHandler(...)` | Equivalent | `ReaderContext.schema_handler: FileGroupReaderSchemaHandler` (`reader_context.rs:84`) | +| `readerContext.getSchemaHandler().getOutputConverter()` | No equivalent | Spark-specific `outputConverter`; hudi-rs uses Arrow `RecordBatch` throughout | +| `readerContext.getMergeMode()` | Equivalent | `ReaderContext.merge_mode: String` (`reader_context.rs:67`) | +| `readerContext.getRecordContext().seal(rec)` | No equivalent | Spark InternalRow lifecycle concept; hudi-rs works with owned `RecordBatch` rows | +| `readerContext.getIteratorMode()` / `setIteratorMode(...)` | Equivalent | `ReaderContext.iterator_mode: String` (`reader_context.rs:66`) | +| `readerContext.getInstantRange()` + `applyInstantRangeFilter` | Equivalent | `ReaderContext.instant_range: Option` (`reader_context.rs:69`); filter applied in file-group reader pipeline | +| `readerContext.getFileRecordIterator(...)` (StoragePathInfo / StoragePath branches) | Equivalent | `make_base_file_batches` at `mod.rs:456`; reads base Parquet via DataFusion | +| `readerContext.getRecordContext().convertPartitionValueToEngineType(...)` | No equivalent | Spark partition value type conversion; hudi-rs handles via Arrow schema directly | +| `readerContext.mergeBootstrapReaders(...)` | No equivalent | Bootstrap merge is not implemented in hudi-rs Phase 2 scope | +| `readerContext.getStorageConfiguration()` | No equivalent | JVM Hadoop `Configuration`; hudi-rs uses `object_store` + filesystem path directly | +| `readerContext.getRecordContext().constructFinalHoodieRecord(...)` | No equivalent | Spark `HoodieRecord` wrapping; hudi-rs returns `RecordBatch` directly | diff --git a/filegroup/FS_logBlockConsumption.md b/filegroup/FS_logBlockConsumption.md new file mode 100644 index 00000000..b10bc0d5 --- /dev/null +++ b/filegroup/FS_logBlockConsumption.md @@ -0,0 +1,545 @@ + +👤 USER: +for hudi commit time ordering, here are some pointers + +org.apache.hudi.common.table.read.buffer.TestKeyBasedFileGroupRecordBuffer + + --- + Part 1: Annotated Code Walkthrough + + The Pipeline + + FileSlice = BaseFile (parquet) + LogFile1 (commit1) + LogFile2 (commit2) + ... + + The processing has 3 phases: (A) scan log files into a map, (B) merge base records with the map, (C) emit remaining log-only records. + + --- + **Phase A: Scan Log Blocks into records Map** + + Log blocks are processed in commit order (oldest → newest). For each block, the buffer calls into KeyBasedFileGroupRecordBuffer: +```java + Data blocks — processDataBlock() (line 73): + // For each record in the log data block: + try (ClosableIterator recordIterator = recordsIteratorSchemaPair.getLeft()) { + while (recordIterator.hasNext()) { + T nextRecord = recordIterator.next(); + boolean isDelete = recordContext.isDeleteRecord(nextRecord, deleteContext); + // Wrap the engine record into a BufferedRecord (extracts key, ordering value, etc.) + BufferedRecord bufferedRecord = BufferedRecords.fromEngineRecord( + nextRecord, schema, readerContext.getRecordContext(), orderingFieldNames, isDelete); + // Merge into the map ↓ + processNextDataRecord(bufferedRecord, bufferedRecord.getRecordKey()); + } + } + + processNextDataRecord() (line 105) — the core delta merge: + public void processNextDataRecord(BufferedRecord record, Serializable recordKey) { + BufferedRecord existingRecord = records.get(recordKey); // Look up existing entry for this key + totalLogRecords++; + // ┌─────────────────────────────────────────────────────────────────┐ + // │ For COMMIT_TIME_ORDERING, deltaMerge always returns newRecord. │ + // │ The new record unconditionally overwrites the old one. │ + // └─────────────────────────────────────────────────────────────────┘ + bufferedRecordMerger.deltaMerge(record, existingRecord) + .ifPresent(bufferedRecord -> + records.put(recordKey, bufferedRecord.toBinary(readerContext.getRecordContext()))); + } + + Delete blocks — processDeleteBlock() (line 113): + public void processDeleteBlock(HoodieDeleteBlock deleteBlock) { + Iterator it = Arrays.stream(deleteBlock.getRecordsToDelete()).iterator(); + while (it.hasNext()) { + DeleteRecord record = it.next(); + processNextDeletedRecord(record, record.getRecordKey()); + } + } + + processNextDeletedRecord() (line 122): + public void processNextDeletedRecord(DeleteRecord deleteRecord, Serializable recordIdentifier) { + BufferedRecord existingRecord = records.get(recordIdentifier); + totalLogRecords++; + // ┌──────────────────────────────────────────────────────────────────┐ + // │ For COMMIT_TIME_ORDERING, deltaMerge always returns the delete. │ + // │ The delete unconditionally overwrites any existing record. │ + // └──────────────────────────────────────────────────────────────────┘ + Option recordOpt = bufferedRecordMerger.deltaMerge(deleteRecord, existingRecord); + recordOpt.ifPresent(deleteRec -> + records.put(recordIdentifier, BufferedRecords.fromDeleteRecord(deleteRec, ...))); + } + + The CommitTimeRecordMerger (line 105 of BufferedRecordMergerFactory.java): + private static class CommitTimeRecordMerger implements BufferedRecordMerger { + // Data vs data: always pick new + public Option> deltaMerge(BufferedRecord newRecord, BufferedRecord existingRecord) { + return Option.of(newRecord); + } + // Delete vs data: always pick delete + public Option deltaMerge(DeleteRecord deleteRecord, BufferedRecord existingRecord) { + return Option.of(deleteRecord); + } + // Base vs log: always pick log (newer commit) + public BufferedRecord finalMerge(BufferedRecord olderRecord, BufferedRecord newerRecord) { + return newerRecord; + } + } +``` + After Phase A, records is a Map holding the last-writer-wins version of each key that appeared in any log block. + + --- +**Phase B: Iterate Base File, Merge with Log Map** +```java + doHasNext() in KeyBasedFileGroupRecordBuffer (line 142): + protected boolean doHasNext() throws IOException { + // Phase B: walk through every base file record + while (baseFileIterator.hasNext()) { + if (hasNextBaseRecord(baseFileIterator.next())) { + return true; + } + } + // Phase C: emit remaining log-only records (inserts from logs) + return hasNextLogRecord(); + } + + hasNextBaseRecord(T baseRecord) (line 135): + protected boolean hasNextBaseRecord(T baseRecord) { + String recordKey = readerContext.getRecordContext().getRecordKey(baseRecord, readerSchema); + // Remove from log map (so it won't be emitted again in Phase C) + BufferedRecord logRecordInfo = records.remove(recordKey); + return hasNextBaseRecord(baseRecord, logRecordInfo); + } + + hasNextBaseRecord(T baseRecord, BufferedRecord logRecordInfo) in FileGroupRecordBuffer (line 238): + protected boolean hasNextBaseRecord(T baseRecord, BufferedRecord logRecordInfo) { + if (logRecordInfo != null) { + // ┌─────────────────────────────────────────────────────────────────────┐ + // │ This key exists in BOTH the base file and log map. │ + // │ finalMerge() picks the winner. │ + // │ For COMMIT_TIME_ORDERING: always returns logRecordInfo (the newer) │ + // └─────────────────────────────────────────────────────────────────────┘ + BufferedRecord baseRecordInfo = BufferedRecords.fromEngineRecord( + baseRecord, readerSchema, readerContext.getRecordContext(), orderingFieldNames, false); + BufferedRecord mergeResult = bufferedRecordMerger.finalMerge(baseRecordInfo, logRecordInfo); + + // ┌──────────────────────────────────────────────────────────────────┐ + // │ processUpdate decides what to emit: │ + // │ - if mergeResult.isDelete() and !emitDeletes → return null │ + // │ - if mergeResult.isDelete() and emitDeletes → emit delete │ + // │ - otherwise → emit the merged record as UPDATE_AFTER │ + // └──────────────────────────────────────────────────────────────────┘ + nextRecord = updateProcessor.processUpdate( + logRecordInfo.getRecordKey(), baseRecordInfo, mergeResult, mergeResult.isDelete()); + return nextRecord != null; + } + + // No log entry for this key → emit the base file record as-is + nextRecord = bufferedRecordConverter.convert(readerContext.getRecordContext().seal(baseRecord)); + return true; + } +``` + --- + **Phase C: Emit Log-Only Records (New Inserts from Logs)** + + hasNextLogRecord() in FileGroupRecordBuffer (line 255): +```java + protected boolean hasNextLogRecord() { + if (logRecordIterator == null) { + initializeLogRecordIterator(); // iterates over remaining entries in records map + } + while (logRecordIterator.hasNext()) { + BufferedRecord nextRecordInfo = logRecordIterator.next(); + // ┌──────────────────────────────────────────────────────────────────┐ + // │ previousRecord = null (no base file match for this key) │ + // │ For non-deletes: emitted as INSERT │ + // │ For deletes: skipped (unless emitDeletes=true) │ + // └──────────────────────────────────────────────────────────────────┘ + nextRecord = updateProcessor.processUpdate( + nextRecordInfo.getRecordKey(), null, nextRecordInfo, nextRecordInfo.isDelete()); + if (nextRecord != null) { + return true; + } + } + return false; + } +``` + +❯ which column is used for deciding commit time? + + Searched for 2 patterns (ctrl+o to expand) + +⏺ The answer is: no column is compared. The ordering is implicit, not column-based. + + With COMMIT_TIME_ORDERING, the CommitTimeRecordMerger unconditionally returns the newer record — it never reads any field value. The "commit time ordering" comes from + the order in which log blocks are processed, not from comparing a column. + + Here's the chain: + + 1. Log blocks are read in commit order — the file group reader iterates log files/blocks sequentially from oldest to newest commit + 2. Each record overwrites the previous — deltaMerge always returns the new record: + public Option> deltaMerge(BufferedRecord newRecord, BufferedRecord existingRecord) { + return Option.of(newRecord); // no field comparison + } + 3. Log wins over base file — finalMerge always returns the log record (which came from a later commit than the base file): + public BufferedRecord finalMerge(BufferedRecord olderRecord, BufferedRecord newerRecord) { + return newerRecord; // no field comparison + } + + So the "commit time" is encoded in the processing order of log blocks, not in any metadata column like _hoodie_commit_time or _hoodie_commit_seqno. Those meta fields + exist in the data but are not consulted by this merger. + + + +my question is +during phase A, does it mean it mains a running map, the map content keeps being updated as it iterate records inside a log file and across log files? + +I would like to see the full chain from reading 1 single records from the log file and it directly update the running map inline + + +🤖 ASSISTANT: +Yes, the map is a running accumulator — every record read from every log block immediately mutates it in-place. Let me trace the exact call chain. + +🤖 ASSISTANT: +Now I have the full picture. Here is the complete call chain from one record in a log file to the map being updated: + +--- + +## Full Chain: 1 Record → Map Update + +**`scanInternal()` in `BaseHoodieLogRecordReader`** (line 192) + +This runs in **two passes** before any record hits the map: + +**Pass 1** (line 249–317): Forward-scan all log blocks across all log files. Builds: +- `orderedInstantsList` — instant times in arrival order (oldest → newest) +- `instantToBlocksMap` — maps each instant time to its list of blocks +- Any rollback targets are removed from both structures + +**Pass 2** (line 336–375): Iterates `orderedInstantsList` in **reverse** (newest → oldest), adding blocks to `currentInstantLogBlocks` deque with `addLast`. The deque ends up with newest instants at the head, oldest at the tail. + +**`processQueuedBlocksForInstant(currentInstantLogBlocks, ...)` at line 385** drains the deque using `pollLast()` — so it processes oldest instants first. For each block: + +``` +BaseHoodieLogRecordReader.processQueuedBlocksForInstant() [line 428] + └─ recordBuffer.processDataBlock(dataBlock, keySpecOpt) [line 437] +``` + +**`KeyBasedFileGroupRecordBuffer.processDataBlock()` at line 73**: opens an iterator over records inside that one block. For each record in the iterator: + +```java +T nextRecord = recordIterator.next(); // read raw engine record +boolean isDelete = recordContext.isDeleteRecord(...); +BufferedRecord bufferedRecord = BufferedRecords.fromEngineRecord( + nextRecord, schema, recordContext, orderingFieldNames, isDelete); // wrap into BufferedRecord +processNextDataRecord(bufferedRecord, bufferedRecord.getRecordKey()); // hit the map +``` + +**`processNextDataRecord()` at line 105** — the map mutation: + +```java +BufferedRecord existingRecord = records.get(recordKey); // look up current map entry +totalLogRecords++; +bufferedRecordMerger.deltaMerge(record, existingRecord) // for COMMIT_TIME: always returns record + .ifPresent(r -> records.put(recordKey, r.toBinary(...))); // ← inline map update +``` + +--- + +## The Critical Insight on Ordering + +The deque is drained with `pollLast()`, and the deque was filled in reverse-instant order with `addLast`. That means: + +- **i1 (oldest)** is at the tail → polled first → written to map first +- **i2** overwrites i1's entries +- **i3 (newest)** overwrites i2's entries → final map state + +This is how commit-time ordering works without comparing any field: the **insertion order into the map is commit order**, and `deltaMerge` unconditionally returns the new record, so the last write (newest commit) survives. + +--- + +## Answer to Your Question + +Yes, `records` is a single running map that is mutated **inline, record by record, block by block, across all log files**. There is no intermediate buffering between reading a record and writing it to the map. The commit ordering is baked into the processing order, not into any per-record timestamp comparison. + + +👤 USER: +where is the start of the scan of log files, and base files. show me code and generate call stack tree + +🤖 ASSISTANT: +Now let me find the `FileGroupRecordBufferLoader` default implementation — that's where `scanInternal` gets triggered: + + +🤖 ASSISTANT: +Now I have the complete picture. Here is the full call stack tree: + +--- + +## Call Stack Tree: Log File Scan + Base File Scan + +### Entry Point — Engine calls `HoodieFileGroupReader` + +``` +HoodieFileGroupReader.getClosableIterator() [HoodieFileGroupReader.java:301] + └─ getBufferedRecordIterator(IteratorMode) [line 291] + └─ initRecordIterators() [line 292] + │ + ├─── makeBaseFileIterator() [line 129] ← BASE FILE SCAN (iterator created here) + │ └─ readerContext.getFileRecordIterator(...) [line 157/163] + │ (returns a lazy Parquet/HFile iterator — not yet consumed) + │ + └─── recordBufferLoader.getRecordBuffer(...) [line 134] ← LOG FILE SCAN (eagerly consumed here) + │ + └─ DefaultFileGroupRecordBufferLoader.getRecordBuffer() [DefaultFileGroupRecordBufferLoader.java:54] + ├─ new KeyBasedFileGroupRecordBuffer(...) [line 78] + └─ scanLogFiles(readerContext, storage, inputSplit, ..., recordBuffer) [line 81] + │ + └─ LogScanningRecordBufferLoader.scanLogFiles() [LogScanningRecordBufferLoader.java:37] + └─ HoodieMergedLogRecordReader.newBuilder()...build() [line 41-52] + └─ HoodieMergedLogRecordReader constructor [HoodieMergedLogRecordReader.java:66] + └─ performScan() [line 75] (forceFullScan=true) + └─ BaseHoodieLogRecordReader.scanInternal() [BaseHoodieLogRecordReader.java:192] + │ + ├─ [Pass 1] HoodieLogFormatReader.next() × N blocks [line 249-317] + │ (reads block headers, builds instantToBlocksMap, strips rollbacks) + │ + ├─ [Pass 2] orderedInstantsList reverse iteration [line 336-375] + │ (resolves compaction, builds currentInstantLogBlocks deque) + │ + └─ processQueuedBlocksForInstant(deque) [line 385] + └─ [for each block, oldest→newest via pollLast] + ├─ recordBuffer.processDataBlock(dataBlock, keySpecOpt) [line 437] + │ └─ KeyBasedFileGroupRecordBuffer.processDataBlock() [line 73] + │ └─ [for each record in block] + │ BufferedRecords.fromEngineRecord(...) + │ └─ processNextDataRecord(bufferedRecord, key) [line 105] + │ └─ records.put(key, deltaMerge(new, existing)) ← MAP UPDATED + │ + └─ recordBuffer.processDeleteBlock(deleteBlock) [line 440] + └─ KeyBasedFileGroupRecordBuffer.processDeleteBlock() [line 113] + └─ processNextDeletedRecord(deleteRecord, key) [line 122] + └─ records.put(key, fromDeleteRecord(...)) ← MAP UPDATED +``` + +After `initRecordIterators()` returns, `records` map is fully populated. The base file iterator is set on the buffer but **not yet consumed**. + +--- + +### Consumer Loop — Engine iterates `HoodieFileGroupReaderIterator` + +``` +HoodieFileGroupReaderIterator.hasNext() [HoodieFileGroupReader.java:332] + └─ HoodieFileGroupReader.hasNext() [line 243] + └─ recordBuffer.hasNext() [line 247] + └─ KeyBasedFileGroupRecordBuffer.doHasNext() [line 142] + │ + ├─ [Phase B] while baseFileIterator.hasNext() [line 146] + │ └─ hasNextBaseRecord(baseFileIterator.next()) [line 147] ← BASE FILE CONSUMED HERE + │ └─ records.remove(recordKey) (lookup + evict from map) + │ └─ finalMerge(baseRecord, logRecord) (log wins for COMMIT_TIME) + │ └─ updateProcessor.processUpdate(...) + │ + └─ [Phase C] hasNextLogRecord() [line 153] + (drains remaining map entries = log-only inserts) +``` + +--- + +### Key Timing Distinction + +| What | When | +|---|---| +| Log file scan (`scanInternal`) | **Eagerly, upfront** — all log blocks fully consumed into `records` map before the caller gets back an iterator | +| Base file scan (`baseFileIterator`) | **Lazily, on demand** — Parquet iterator created eagerly but records only read as consumer calls `hasNext()`/`next()` | + + +👤 USER: +zoom into the phase A where we read and merge all log files. when is org.apache.hudi.common.table.log.block.HoodieLogBlock#content populated? does it hold all log file content in memory? + + +🤖 ASSISTANT: +Now I have the complete picture. Here is the full answer: + +--- + +## When is `content` populated? + +There are **two paths** depending on the block version: + +### Path A — Old blocks (`DEFAULT_VERSION`) +`HoodieLogFileReader.readBlock()` line 167–168: +```java +boolean shouldReadLazily = nextBlockVersion.getVersion() != HoodieLogFormatVersion.DEFAULT_VERSION; +// → for DEFAULT_VERSION: shouldReadLazily = false +Option content = HoodieLogBlock.tryReadContent(inputStream, contentLength, shouldReadLazily); +``` +`tryReadContent` (`HoodieLogBlock.java:332`): +```java +if (readLazily) { inputStream.seek(...); return Option.empty(); } // ← skip +byte[] content = new byte[contentLength]; +inputStream.readFully(content, 0, contentLength); // ← OLD PATH: eagerly read here +return Option.of(content); +``` +Content bytes land in `content` during Pass 1 of `scanInternal`, inline while scanning. + +### Path B — Modern blocks (version > DEFAULT) — the normal case +`shouldReadLazily = true` → `tryReadContent` just **seeks past the bytes** and returns `Option.empty()`. The block object is constructed with: +- `content = Option.empty()` +- `inputStreamSupplier` = lambda that re-opens the file +- `blockContentLocation` = {file path, contentPositionInLogFile, blockSize, blockEndPos} + +Content bytes are loaded later in `HoodieDataBlock.readRecordsFromBlockPayload()` (`line 272`): +```java +if (readBlockLazily && !getContent().isPresent()) { + inflate(); // ← content loaded HERE, on demand when getRecordIterator() is called +} +try { + return deserializeRecords(getContent().get(), type); +} finally { + deflate(); // ← content bytes released immediately after deserialization +} +``` +`inflate()` (`HoodieLogBlock.java:401`) seeks back into the file and reads the block: +```java +content = Option.of(new byte[(int) this.getBlockContentLocation().get().getBlockSize()]); +inputStream.seek(this.getBlockContentLocation().get().getContentPositionInLogFile()); +inputStream.readFully(content.get(), 0, content.get().length); +``` + +--- + +## Does it hold all log file content in memory? + +**No.** The design is explicitly block-by-block. Here is what lives in memory at each phase: + +| Phase | What's in memory | +|---|---| +| Pass 1 (`logFormatReaderWrapper.next()` loop) | Block metadata only (headers/footers + `blockContentLocation`). Content bytes skipped for modern blocks. All block objects from all log files accumulate in `instantToBlocksMap`. | +| Pass 2 (building deque) | Same block objects, just reordered. No new allocations. | +| `processQueuedBlocksForInstant` — for each block | `inflate()` reads **one block's** `byte[]` into heap → `deserializeRecords` parses it → `deflate()` drops the `byte[]` reference → GC can reclaim it → next block | +| Growing throughout Phase A | The `records` map accumulates merged `BufferedRecord` entries (one per unique key seen so far) | + +So at peak within Phase A, memory holds: +1. All lightweight block header/metadata objects (small) +2. **One block's raw `byte[]`** (can be large — a single data block can be 100s of MB) +3. The `records` map built so far + +The `deflate()` call immediately after `deserializeRecords` is the key: it drops the `byte[]` so the GC can reclaim the space before the next block is inflated. The design deliberately avoids holding more than one block's worth of raw bytes at a time. + +--- + +## Call stack for content loading (modern blocks) + +``` +processQueuedBlocksForInstant(deque) [BaseHoodieLogRecordReader.java:428] + └─ recordBuffer.processDataBlock(dataBlock) [line 437] + └─ KeyBasedFileGroupRecordBuffer.processDataBlock() [line 73] + └─ getRecordsIterator(dataBlock, keySpecOpt) [line 74] + └─ HoodieDataBlock.getRecordIterator(type) [HoodieDataBlock.java:157] + └─ readRecordsFromBlockPayload(type) [line 272] + ├─ inflate() ← content byte[] allocated + readFully() from disk + ├─ deserializeRecords(content.get(), type) ← parse records from bytes + └─ deflate() ← content = Option.empty(), byte[] eligible for GC +``` + + +👤 USER: +for table version 9 what's the condition for that org.apache.hudi.common.table.log.block.HoodieLogBlock#content population? + + +🤖 ASSISTANT: +Now I have everything. Here is the precise answer: + +--- + +## `content` population for table version 9 + +### Step 1 — Pass 1 (`readBlock()`): content is **never** populated + +The decision at `HoodieLogFileReader.readBlock()` line 167 is driven by the **log format version**, not the table version: + +```java +boolean shouldReadLazily = nextBlockVersion.getVersion() != HoodieLogFormatVersion.DEFAULT_VERSION; +// log format version 1 != 0 (DEFAULT) +// → true for ALL blocks written by table version 9 +Option content = HoodieLogBlock.tryReadContent(inputStream, contentLength, shouldReadLazily); +// → seeks past the bytes, returns Option.empty() +``` + +`HoodieLogFormat.CURRENT_VERSION = 1` and `DEFAULT_VERSION = 0`. Table version 9 always writes blocks at format version 1 → `shouldReadLazily = true` → content bytes are **skipped** in the stream. Every block object enters `instantToBlocksMap` with `content = Option.empty()`. + +--- + +### Step 2 — Pass 1 filter: which blocks survive to `processQueuedBlocksForInstant` + +This is where table version 9 **diverges** from versions < 8. The filter in `scanInternal` at line 269: + +```java +if (logBlock.getBlockType() != COMMAND_BLOCK) { + if (this.tableVersion.lesserThan(HoodieTableVersion.EIGHT) && !allowInflightInstants) { + // ← for version 9, this entire block is SKIPPED + HoodieTimeline commitsTimeline = ... + if (commitsTimeline.filterInflights().containsInstant(instantTime) + || !commitsTimeline.filterCompletedInstants()...) { + continue; // inflight blocks filtered OUT here (only for versions < 8) + } + } +} +``` + +For version 9 (`!lesserThan(EIGHT)` → `true`), the inflight/uncommitted check is **never executed**. The only filters that apply are: + +| Filter | Applies to version 9? | +|---|---| +| `CORRUPT_BLOCK` → skip | Yes | +| `instantTime > latestInstantTime` → skip | Yes | +| Rollback removal (via `COMMAND_BLOCK`) | Yes | +| `instantRange` filter | Yes (if set) | +| Inflight/uncommitted timeline check | **No — bypassed entirely** | + +--- + +### Step 3 — `inflate()`: content **is** populated here + +For each block that survived the filters and was dequeued in `processQueuedBlocksForInstant`: + +``` +processQueuedBlocksForInstant → recordBuffer.processDataBlock(block) + → KeyBasedFileGroupRecordBuffer.getRecordsIterator(dataBlock) + → dataBlock.getEngineRecordIterator(readerContext) [HoodieDataBlock.java:241] + → readRecordsFromBlockPayload(readerContext) [line 306] + if (readBlockLazily && !getContent().isPresent()) // true for version 9 blocks + inflate() ← content = new byte[blockSize]; readFully(...) + deserializeRecords(readerContext, getContent().get()) + deflate() ← content = Option.empty(), byte[] eligible for GC +``` + +`inflate()` (`HoodieLogBlock.java:401`) opens a **new stream** via `inputStreamSupplier`, seeks to `contentPositionInLogFile`, and reads exactly `blockSize` bytes: + +```java +content = Option.of(new byte[(int) this.getBlockContentLocation().get().getBlockSize()]); +inputStream.seek(this.getBlockContentLocation().get().getContentPositionInLogFile()); +inputStream.readFully(content.get(), 0, content.get().length); +``` + +--- + +### Summary: what's unique to version 9 + +``` +Pass 1 loop: block arrives from log file + │ + ├─ instantTime > latestInstantTime? → skip (drop, content never populated) + ├─ CORRUPT_BLOCK? → skip + ├─ COMMAND_BLOCK rollback target? → removed from map + ├─ outside instantRange? → skip + │ + │ ← inflight timeline check is NOT applied (version 9 skips it) + │ + └─ passes all filters → added to instantToBlocksMap with content = Option.empty() + +processQueuedBlocksForInstant: + └─ inflate() → content byte[] allocated and filled ← only moment content is populated + └─ deserializeRecords() + └─ deflate() → content dropped +``` + +**Bottom line:** for version 9, `content` is populated exactly once per block — inside `inflate()` during `processQueuedBlocksForInstant` — and only for blocks that are non-corrupt, non-rolled-back, within `latestInstantTime`, and within `instantRange`. Inflight blocks are **not** pre-filtered (unlike versions < 8), so they too will have `inflate()` called and their content loaded into memory. + diff --git a/filegroup/FS_logFileReadInputs_fileOrderRequestTime.md b/filegroup/FS_logFileReadInputs_fileOrderRequestTime.md new file mode 100644 index 00000000..7fbbae75 --- /dev/null +++ b/filegroup/FS_logFileReadInputs_fileOrderRequestTime.md @@ -0,0 +1,340 @@ + +👤 USER: +BaseHoodieLogRecordReader.scanInternal() It seems like the scan internal function will not just blindly read all the log for a given file group. So I have two questions. First is how does it know which file group it should process? + +🤖 ASSISTANT: +Let me trace the inputs from the very top to understand what controls the scope of `scanInternal`. + +🤖 ASSISTANT: +Now I have everything needed. Here's the full answer: + +--- + +## External Inputs + +There are **4 key inputs** that control what `scanInternal` reads. They are set long before `scanInternal` is called: + +### Input 1: `logFiles` — **Which file group, and which log files** + +The scope is pre-determined by the `FileSlice`, which the engine (Spark/Flink) selects via the file system view. The chain is: + +``` +Engine (Spark/Flink) selects a FileSlice for a file group + └─ HoodieFileGroupReader.Builder.withFileSlice(fileSlice) [HoodieFileGroupReader.java:396] + ├─ this.baseFileOption = fileSlice.getBaseFile() + ├─ this.logFiles = fileSlice.getLogFiles() ← Stream + └─ this.partitionPath = fileSlice.getPartitionPath() +``` + +The `FileSlice` object already holds exactly the log files for **one file group** (`FileSlice.java:67`): + +```java +// FileSlice.java:67 — already scoped to one fileGroupId +private final TreeSet logFiles; // sorted by getReverseLogFileComparator() +``` + +A `HoodieLogFile` encodes the file group identity in its **filename**: +``` +.fileId_deltaCommitTime_logVersion_writeToken.log +``` +So all log files in a `FileSlice` share the same `fileId`. + +These log files flow through: +``` +HoodieFileGroupReader.Builder.build() [line 521] + └─ new InputSplit(baseFileOption, Either.left(logFiles), ...) [line 521] + └─ InputSplit constructor [InputSplit.java:56] + // ┌──────────────────────────────────────────────────────────────┐ + // │ Sorts log files AND filters out CDC log files │ + // │ Sort order: deltaCommitTime ASC → logVersion ASC │ + // │ → writeToken ASC → suffix ASC │ + // └──────────────────────────────────────────────────────────────┘ + this.logFiles = recordsToMerge.asLeft() + .sorted(HoodieLogFile.getLogFileComparator()) + .filter(logFile -> !logFile.getFileName().endsWith(CDC_LOGFILE_SUFFIX)) + .collect(Collectors.toList()); +``` + +Then into the log reader builder: +``` +LogScanningRecordBufferLoader.scanLogFiles() [LogScanningRecordBufferLoader.java:41] + └─ HoodieMergedLogRecordReader.newBuilder() + .withLogFiles(inputSplit.getLogFiles()) ← the sorted list + .build() +``` + +And into `HoodieMergedLogRecordReader` constructor → `BaseHoodieLogRecordReader` stores it as `this.logFiles`. + +### Input 2: `latestInstantTime` — **High-watermark for block filtering** + +```java +// BaseHoodieLogRecordReader.java:144 +this.latestInstantTime = readerContext.getLatestCommitTime(); +``` + +Set by: +``` +HoodieFileGroupReader constructor [HoodieFileGroupReader.java:113] + └─ readerContext.setLatestCommitTime(latestCommitTime) +``` + +### Input 3: `instantRange` — **Optional incremental range filter** + +```java +// BaseHoodieLogRecordReader constructor, stored as this.instantRange +// LogScanningRecordBufferLoader.java:47 +.withInstantRange(readerContext.getInstantRange()) +``` + +### Input 4: `allowInflightInstants` — **Whether to include uncommitted blocks** + +```java +// LogScanningRecordBufferLoader.java:50 +.withAllowInflightInstants(readerParameters.allowInflightInstants()) +``` + + +👤 USER: +for commit time ordering, it requires we process the data based on their completion time, under this scenario, show me how we enforce such order of processing records across log files + +## There is no completion-time sorting of log files within a file slice on the merge/read path +``` +⏺ FileSlice.() + │ logFiles = new TreeSet<>(HoodieLogFile.getReverseLogFileComparator()) + │ // stored: [t3_v1, t2_v1, t1_v2, t1_v1] (descending by deltaCommitTime, then version) + │ + FileSlice.addLogFile(logFile) // called per log file discovered + │ logFiles.add(logFile) // TreeSet auto-sorts on insert + │ + └── HoodieFileGroupReader.constructInputSplits() + └── InputSplit.(baseFile, logFileStream, ...) + │ // InputSplit.java:56 + │ this.logFiles = logFileStream + │ .sorted(HoodieLogFile.getLogFileComparator()) // RE-SORT: ascending + │ .filter(not CDC) + │ .collect(toList()) + │ // result: [t1_v1, t1_v2, t2_v1, t3_v1] (ascending by deltaCommitTime, then version) + │ + └── HoodieFileGroupReader.readLogFiles() + └── LogScanningRecordBufferLoader.scanLogFiles(inputSplit, recordBuffer) + │ + └── HoodieMergedLogRecordReader.newBuilder() + │ .withLogFiles(inputSplit.getLogFiles()) // [t1_v1, t1_v2, t2_v1, t3_v1] + │ .withRecordBuffer(recordBuffer) + │ .build() + │ + └── HoodieMergedLogRecordReader.() + └── BaseHoodieLogRecordReader.scanInternal() + │ + │── HoodieLogFormatReader.(logFiles) + │ │ // receives [t1_v1, t1_v2, t2_v1, t3_v1] + │ └── currentReader = new HoodieLogFileReader(logFiles.remove(0)) // t1_v1 + │ + │── [FORWARD PASS] ────────────────────────────────────────── + │ while (logFormatReaderWrapper.hasNext()) + │ │ // iterates: t1_v1 → t1_v2 → t2_v1 → t3_v1 + │ │ logBlock = logFormatReaderWrapper.next() + │ │ instantTime = logBlock.header.get(INSTANT_TIME) + │ │ + │ │ // first time seeing an instant → append to orderedInstantsList + │ │ orderedInstantsList.add(instantTime) + │ │ instantToBlocksMap.get(instantTime).add(logBlock) + │ │ + │ // result: + │ // orderedInstantsList = [t1, t2, t3] + │ // instantToBlocksMap = {t1→[A,B], t2→[C], t3→[D,E]} + │ + │── [REVERSE PASS] ────────────────────────────────────────── + │ for (i = orderedInstantsList.size()-1; i >= 0; i--) // t3, t2, t1 + │ │ logBlocks = instantToBlocksMap.get(instantTime) + │ │ Collections.reverse(logBlocks) + │ │ logBlocks.forEach(b -> currentInstantLogBlocks.addLast(b)) + │ │ + │ // deque after each iteration: + │ // i=2 (t3): front [E, D] back + │ // i=1 (t2): front [E, D, C] back + │ // i=0 (t1): front [E, D, C, B, A] back + │ + └── [PROCESS] ──────────────────────────────────────────────── + processQueuedBlocksForInstant(currentInstantLogBlocks) + │ + while (!logBlocks.isEmpty()) + │ lastBlock = logBlocks.pollLast() + │ // poll order: A(t1) → B(t1) → C(t2) → D(t3) → E(t3) + │ + ├── processDataBlock(dataBlock) + │ └── while (recordIterator.hasNext()) + │ └── processNextDataRecord(record, key) + │ │ existing = records.get(key) + │ │ merged = bufferedRecordMerger.deltaMerge(record, existing) + │ └── records.put(key, merged) + │ + └── processDeleteBlock(deleteBlock) + └── for each DeleteRecord + └── processNextDeletedRecord(deleteRecord, key) + │ existing = records.get(key) + │ merged = bufferedRecordMerger.deltaMerge(deleteRecord, existing) + └── records.put(key, merged) +``` +Key takeaway: The double-reversal (reverse iteration of instants + addLast in the deque + pollLast to consume) produces oldest-first +processing. The last writer to records.put(key, ...) is always the newest deltaCommitTime, so it wins when the merger favors the incoming +record. + +### Step 1: FileSlice — reverse-sorted TreeSet + +```java +// FileSlice.java:77 +this.logFiles = new TreeSet<>(HoodieLogFile.getReverseLogFileComparator()); +``` + +Log files stored **descending by deltaCommitTime** (aka requested/instant time). The `LogFileComparator` (HoodieLogFile.java:215-240) sorts by: deltaCommitTime → logVersion → writeToken → suffix. The reversed version flips this. + +### Step 2: InputSplit — re-sorted to forward order + +```java +// InputSplit.java:56 +this.logFiles = recordsToMerge.asLeft() + .sorted(HoodieLogFile.getLogFileComparator()) // forward: oldest deltaCommitTime first + .filter(...) + .collect(Collectors.toList()); +``` + +Result: `[log_t1_v1, log_t1_v2, log_t2_v1, log_t3_v1, ...]` — ascending by deltaCommitTime, then logVersion. + +### Step 3: HoodieLogFormatReader — sequential iteration, oldest first + +```java +// HoodieLogFormatReader.java:61 +HoodieLogFile nextLogFile = logFiles.remove(0); // takes from front = oldest + +// hasNext() line 87: same pattern +HoodieLogFile nextLogFile = logFiles.remove(0); +``` + +Blocks are yielded in the order they appear within each file, file by file, oldest deltaCommitTime first. + +### Step 4: Forward scan — builds orderedInstantsList + +```java +// BaseHoodieLogRecordReader.java:249-310 +while (logFormatReaderWrapper.hasNext()) { + HoodieLogBlock logBlock = logFormatReaderWrapper.next(); + String instantTime = logBlock.getLogBlockHeader().get(INSTANT_TIME); + // ... + // Line 292: first appearance of an instant → added to list + orderedInstantsList.add(instantTime); + // Line 293: all blocks for that instant accumulated + logBlocksList.add(logBlock); + instantToBlocksMap.put(instantTime, logBlocksList); +} +``` + +`orderedInstantsList` = `[t1, t2, t3]` — order of first block appearance = deltaCommitTime ascending. + +`instantToBlocksMap`: +- `t1 → [block_A, block_B]` (in file-read order) +- `t2 → [block_C]` +- `t3 → [block_D, block_E]` + +### Step 5: Reverse pass — builds the deque + +```java +// BaseHoodieLogRecordReader.java:336-375 +for (int i = orderedInstantsList.size() - 1; i >= 0; i--) { // t3, t2, t1 + List logBlocks = instantToBlocksMap.get(instantTime); + Collections.reverse(logBlocks); // [block_D, block_E] → [block_E, block_D] + logBlocks.forEach(block -> currentInstantLogBlocks.addLast(block)); +} +``` + +Iteration order: **t3 first, then t2, then t1**. Each instant's blocks are reversed then appended to the **back** of the deque. + +Deque state after the loop (front → back): + +``` +front: [block_E, block_D, block_C, block_B, block_A] :back + ←── t3 ──→ ←t2→ ←──── t1 ────→ +``` + +### Step 6: Processing — pollLast (oldest instant first) + +```java +// BaseHoodieLogRecordReader.java:432 +HoodieLogBlock lastBlock = logBlocks.pollLast(); +``` + +Poll order from the back: **block_A → block_B → block_C → block_D → block_E** + +This means: **t1's blocks processed first (in original file order), then t2, then t3.** + +The `Collections.reverse` + `addLast` + `pollLast` pattern is effectively a double-reversal that restores original file-read order within each instant, while the reverse iteration of instants + pollLast produces oldest-instant-first processing. + +### Step 7: LogScanningRecordBufferLoader → recordBuffer + +```java +// LogScanningRecordBufferLoader.java:44,49 +.withLogFiles(inputSplit.getLogFiles()) // passes the forward-sorted list +.withRecordBuffer(recordBuffer) // the buffer receives processDataBlock/processDeleteBlock calls +``` + +The scanner calls `recordBuffer.processDataBlock()` / `recordBuffer.processDeleteBlock()` directly during `processQueuedBlocksForInstant()` — there's no separate buffering step. + +### Step 8: KeyBasedFileGroupRecordBuffer — record-level merge + +```java +// KeyBasedFileGroupRecordBuffer.java:105-110 +public void processNextDataRecord(BufferedRecord record, Serializable recordKey) { + BufferedRecord existingRecord = records.get(recordKey); // lookup by key + totalLogRecords++; + bufferedRecordMerger.deltaMerge(record, existingRecord) + .ifPresent(bufferedRecord -> records.put(recordKey, bufferedRecord.toBinary(...))); +} + +// KeyBasedFileGroupRecordBuffer.java:122-128 +public void processNextDeletedRecord(DeleteRecord deleteRecord, Serializable recordIdentifier) { + BufferedRecord existingRecord = records.get(recordIdentifier); + totalLogRecords++; + Option recordOpt = bufferedRecordMerger.deltaMerge(deleteRecord, existingRecord); + recordOpt.ifPresent(deleteRec -> records.put(recordIdentifier, BufferedRecords.fromDeleteRecord(...))); +} +``` + +For each record in each block, the buffer does a **key-based lookup** in the `records` map, then calls `bufferedRecordMerger.deltaMerge(incoming, existing)`. The merge result replaces the existing entry. + +--- + +### Net effect + +| Processing order | What | +|---|---| +| 1st | t1's blocks (oldest deltaCommitTime), in original file-read order | +| 2nd | t2's blocks | +| 3rd | t3's blocks (newest deltaCommitTime) | + +For a given key that appears in multiple instants, `deltaMerge` is called with: +1. `deltaMerge(t1_record, null)` → stores t1_record +2. `deltaMerge(t2_record, t1_record)` → stores merge result +3. `deltaMerge(t3_record, t2_result)` → stores final result + +With `CommitTimeRecordMerger` (which always returns the incoming record), **the latest deltaCommitTime wins**. With other mergers, the outcome depends on ordering field comparison within `deltaMerge`. + +**Processing order through the deque:** + +``` +Deque after reverse pass: + + front back + ┌──────────┬──────────┬──────────┬───--───┬──────────┐ + │ blkE(t3) │ blkD(t3) │ blkC(t2) │blkB(t1)│ blkA(t1) │ + └──────────┴──────────┴──────────┴─────--─┴──────────┘ + ↑ pollLast() + + Poll order: blkA(t1) → blkB(t1) → blkC(t2) → blkD(t3) → blkE(t3) + oldest instant first ──────────────► newest instant last +``` + +For a key `K` appearing in t1 and t3: +``` +deltaMerge(t1_record, null) → records["K"] = t1_record +deltaMerge(t3_record, t1_record) → records["K"] = merge_result // latest instant is "incoming" +``` \ No newline at end of file diff --git a/filegroup/callStackOfFgReader b/filegroup/callStackOfFgReader new file mode 100644 index 00000000..50c9f602 --- /dev/null +++ b/filegroup/callStackOfFgReader @@ -0,0 +1,41 @@ + + HoodieFileGroupReader.getClosableIterator() [HoodieFileGroupReader.java:301] + └─ getBufferedRecordIterator(IteratorMode) [line 291] + └─ initRecordIterators() [line 292] + │ + ├─── makeBaseFileIterator() [line 129] ← BASE FILE SCAN (iterator created here) + │ └─ readerContext.getFileRecordIterator(...) [line 157/163] + │ (returns a lazy Parquet/HFile iterator — not yet consumed) + │ + └─── recordBufferLoader.getRecordBuffer(...) [line 134] ← LOG FILE SCAN (eagerly consumed here) + │ + └─ DefaultFileGroupRecordBufferLoader.getRecordBuffer() [DefaultFileGroupRecordBufferLoader.java:54] + ├─ new KeyBasedFileGroupRecordBuffer(...) [line 78] + └─ scanLogFiles(readerContext, storage, inputSplit, ..., recordBuffer) [line 81] + │ + └─ LogScanningRecordBufferLoader.scanLogFiles() [LogScanningRecordBufferLoader.java:37] + └─ HoodieMergedLogRecordReader.newBuilder()...build() [line 41-52] + └─ HoodieMergedLogRecordReader constructor [HoodieMergedLogRecordReader.java:66] + └─ performScan() [line 75] (forceFullScan=true) + └─ BaseHoodieLogRecordReader.scanInternal() [BaseHoodieLogRecordReader.java:192] + │ + ├─ [Pass 1] HoodieLogFormatReader.next() × N blocks [line 249-317] + │ (reads block headers, builds instantToBlocksMap, strips rollbacks) + │ + ├─ [Pass 2] orderedInstantsList reverse iteration [line 336-375] + │ (resolves compaction, builds currentInstantLogBlocks deque) + │ + └─ processQueuedBlocksForInstant(deque) [line 385] + └─ [for each block, oldest→newest via pollLast] + ├─ recordBuffer.processDataBlock(dataBlock, keySpecOpt) [line 437] + │ └─ KeyBasedFileGroupRecordBuffer.processDataBlock() [line 73] + │ └─ [for each record in block] + │ BufferedRecords.fromEngineRecord(...) + │ └─ processNextDataRecord(bufferedRecord, key) [line 105] + │ └─ records.put(key, deltaMerge(new, existing)) ← MAP UPDATED + │ + └─ recordBuffer.processDeleteBlock(deleteBlock) [line 440] + └─ KeyBasedFileGroupRecordBuffer.processDeleteBlock() [line 113] + └─ processNextDeletedRecord(deleteRecord, key) [line 122] + └─ records.put(key, fromDeleteRecord(...)) ← MAP UPDATED + diff --git a/filegroup/callStackOfFgReaderFsInput.md b/filegroup/callStackOfFgReaderFsInput.md new file mode 100644 index 00000000..7d966468 --- /dev/null +++ b/filegroup/callStackOfFgReaderFsInput.md @@ -0,0 +1,474 @@ +# Invariants +Invariant +``` + ════════════════════════════════════════════════════════════════════════════════════ + SUMMARY: Properties of the FileSlice / InputSplit reaching the FG Reader + ════════════════════════════════════════════════════════════════════════════════════ + + P1 Committed-only filtering + - original: filterUncommittedFiles() — base file AND log files + both checked via completionTimeQueryView.isCompleted() + - MDT path: filterUncommittedLogs() — log files only + base file accepted if it passes P5 (isFileSliceCommitted) + even when its compaction is still inflight + + P2 Log files sorted by LogFileComparator [unchanged] + - Applied in buildFileGroups line 259, and again in InputSplit:56 + + P3 Log-to-base binding via completion time [unchanged] + - addLogFile(completionTimeQueryView, logFile) [ATFSV:260 / HFG:128-148] + + P4 Replaced file groups excluded + - original: isFileGroupReplaced() — no time bound + - MDT path: isFileGroupReplacedBeforeOrOn(maxInstantTime) + only replaces ≤ queryInstant are considered + + P5 Base instant bounded by visible timeline [unchanged, applied twice] + - isFileSliceCommitted() inside HoodieFileGroup [HFG:158-163] + - additional bound: baseInstantTime ≤ maxInstantTime [HFG:215] + + P6 CDC log files excluded [unchanged] + - InputSplit constructor line 57 + + P7 Bootstrap base file mapping [unchanged] + - addBootstrapBaseFileIfPresent() [ATFSV:1096] + + P8 Pending compaction file slice injected [unchanged] + - buildFileGroups addPendingCompactionFileSlice=true [ATFSV:245-254] + + NEW Compaction slice merging (no equivalent in original) + - fetchMergedFileSlice() merges pre-compaction base + with log files spanning the compaction boundary [ATFSV:1595-1620] + + Legend: + ATFSV = AbstractTableFileSystemView + HFG = HoodieFileGroup +``` + +# Call stack tree +``` + Engine (Spark SQL: SELECT * FROM hoodie_table) + │ + ▼ + Spark3HoodiePruneFileSourcePartitions.apply() + │ + ▼ + HoodieFileIndex.filterFileSlices() [HoodieFileIndex.scala:227] + │ + ▼ + HoodieFileIndex.prunePartitionsAndGetFileSlices() [HoodieFileIndex.scala:351] + │ + ▼ + BaseHoodieTableFileIndex.getInputFileSlices(partitions...) [BaseHoodieTableFileIndex.java:252] + │ + └── ensurePreloadedPartitions(partitionPaths) [line 259] + │ + └── loadFileSlicesForPartitions(missingPartitions) [line 267] + │ + │ // Resolve query instant: + │ activeTimeline = getActiveTimeline() [line 279] + │ // ─► metaClient.getCommitsAndCompactionTimeline() + │ // DATA TABLE timeline, not MDT's own timeline + │ queryInstant = specifiedQueryInstant + │ .or(() -> latestInstant.requestedTime) [line 281] + │ + ├─── STEP 1: FILE LISTING via MDT [line 285] + │ listPartitionPathFiles(partitions, activeTimeline) + │ │ + │ └── tableMetadata.getAllFilesInPartitions( [line 489] + │ missingPartitionPathsMap.keySet(), + │ getPartitionPathFilter(activeTimeline)) + │ │ + │ ▼ [BaseTableMetadata] + │ fetchAllFilesInPartitionPaths() [line 315] + │ └── readIndexRecordsWithKeys() [line 331] + │ │ [HoodieBackedTableMetadata] + │ └── getLatestMergedFileSlicesBeforeOrOn + │ (MDT partition, queryInstant) [line 467-468] + │ // reads MDT's own file slices + │ // to locate the avro log records + │ // holding file listings + │ │ + │ └── HoodieMetadataPayload + │ .getFileList() + │ // returns List + │ // — raw path strings only, + │ // no FileSlice structure yet + │ + │ + └─── STEP 2: COW fast-path guard [line 292-297] + if (useLatestBaseFilesPathFilterForListing + && isCOW || isReadOptimized) + → generatePartitionFileSlicesPostROTablePathFilter() + // skips HoodieTableFileSystemView entirely: + // StoragePathInfo → HoodieBaseFile + // → FileSlice directly [line 320-323] + // NO P1/P3/P4/P5/P7/P8 applied + // only for COW snapshot / READ_OPTIMIZED + // + // *** NOT taken in the MOR stack trace above *** + │ + │ + └─── STEP 3: MOR / general path [line 298] + filterFiles(partitions, activeTimeline, + allFiles, queryInstant) + │ + │ // Construct a fresh HoodieTableFileSystemView + │ // from the raw StoragePathInfo list returned + │ // by MDT. The DATA TABLE's timeline is passed. + new HoodieTableFileSystemView( [line 342] + metaClient, activeTimeline, allFiles) + │ + └── addFilesToView(partitionPath, statuses) [ATFSV:191] + │ + └── buildFileGroups(partitionPath, [ATFSV:193] + statuses, + visibleCommitsAndCompactionTimeline, + addPendingCompaction=true) + │ + │ // Group raw files into base/log maps + │ baseFiles grouped by fileId [line 232] + │ logFiles grouped by fileId [line 234] + │ + │ // Per file group: + │ HoodieFileGroup(partition, [line 241] + │ fileId, timeline) + │ // timeline = DATA TABLE's + │ // visibleCommitsAndCompactionTimeline + │ // lastInstant captured here [HFG:92] + │ + ├── A-1: addBaseFile(baseFile) [line 243] + │ + ├── A-2: PROPERTY 8 [line 245-255] + │ if pendingCompaction exists + │ group.addNewFileSliceAtInstant( + │ compactionInstantTime) + │ // injects empty slice at + │ // compaction request time so + │ // later log files land there + │ + └── A-3: PROPERTY 3 [line 259-260] + logFiles.sorted( + HoodieLogFile + .getLogFileComparator()) + .forEach(logFile -> + group.addLogFile( + completionTimeQueryView, + logFile)) + │ + └── getBaseInstantTime( [HFG:129] + completionTimeQueryView, + logFile) + │ + │ // completionTimeQueryView + │ // .getCompletionTime( + │ // latestSliceKey, + │ // logFile.deltaCommitTime) + │ // [HFG:134] + │ // Walk slices newest→oldest + │ // find largest base instant + │ // whose completion time ≤ + │ // log's completion time [HFG:138] + │ // pending → firstKey() [HFG:147] + └──► fileSlices.get(baseInstant) + .addLogFile(logFile) + [HFG:125] + ▼ + (HoodieTableFileSystemView now holds fully-built HoodieFileGroup objects) + │ + ▼ + [queryInstant.isPresent() == true in the stack trace above] + │ + ▼ + getLatestMergedFileSlicesBeforeOrOn( [ATFSV:1068] + partitionPath, queryInstant) + │ + └── getLatestMergedFileSliceBeforeOrOnInternal( [ATFSV:1077] + partitionStr, + maxInstantTime = queryInstant, + currentInstantTime = queryInstant, + includeInflight = false) + │ + │ fetchAllStoredFileGroups(partition) [ATFSV:1085] + │ + ├── PROPERTY 4 (time-scoped variant) [ATFSV:1086] + │ .filter(fg -> + │ !isFileGroupReplacedBeforeOrOn( + │ fg.getFileGroupId(), maxInstantTime)) + │ // Only REPLACE instants ≤ maxInstantTime exclude the group. + │ // A future replace is invisible to this query instant. + │ // [differs from getLatestFileSlices which uses + │ // isFileGroupReplaced() with no time bound] + │ + ├── PROPERTY 5 (re-applied per file-group) [ATFSV:1088] + │ fileGroup.getLatestFileSliceBeforeOrOn(maxInstantTime) + │ │ + │ └── getAllFileSlices() [HFG:183-188] + │ .filter(this::isFileSliceCommitted) + │ │ + │ │ isFileSliceCommitted(slice): [HFG:158-163] + │ │ (a) baseInstantTime + │ │ ≤ lastInstant.requestedTime + │ │ (b) timeline.containsOrBefore + │ │ TimelineStarts(baseInstant) + │ │ + │ .filter(slice -> baseInstantTime [HFG:215] + │ ≤ maxInstantTime) + │ .findFirst() + │ // Two-level gate: committed (P5) AND ≤ maxInstantTime + │ + ├── PROPERTY 1 (logs only — base file NOT re-checked) [ATFSV:1091-1092] + │ fileSlice = filterUncommittedLogs(fileSlice) + │ │ + │ └── fileSlice.getLogFiles() [ATFSV:627] + │ .filter(logFile -> + │ completionTimeQueryView + │ .isCompleted(logFile.getDeltaCommitTime())) + │ // uncommitted log files dropped + │ // base file passes through unconditionally + │ // [contrast: getLatestFileSlices calls + │ // filterUncommittedFiles which checks base + │ // file too via completionTimeQueryView] + │ + ├── NEW: pending-compaction slice merging [ATFSV:1091] + │ fetchMergedFileSlice(fileGroup, fileSlice, + │ currentInstantTime, includeInflight=false) + │ │ + │ └── getPendingCompactionOperationWithInstant( [ATFSV:1600] + │ fileGroup.getFileGroupId()) + │ if present && fileSlice.baseInstant + │ == compactionInstantTime: [ATFSV:1612] + │ prevFileSlice = + │ fileGroup.getLatestFileSliceBefore( [ATFSV:1613] + │ compactionInstantTime) + │ return mergeCompactionPendingFileSlices( [ATFSV:1615] + │ fileSlice, prevFileSlice) + │ // produces virtual slice: + │ // baseFile = prevFileSlice.baseFile + │ // logFiles = prevFileSlice.logFiles + │ // + fileSlice.logFiles (current) + │ else: return fileSlice unchanged + │ + └── PROPERTY 7 [ATFSV:1096] + .map(this::addBootstrapBaseFileIfPresent) + // if baseInstantTime == METADATA_BOOTSTRAP_INSTANT_TS + // attach external bootstrap base file path + ▼ + FileSlice { baseFile, logFiles } returned to filterFiles() + │ + ▼ + (collected into List per partition, cached in + cachedAllInputFileSlices, handed back to HoodieFileIndex) + │ + │ NOTE: PROPERTY 2 and PROPERTY 6 are applied downstream + │ inside HoodieFileGroupReader / InputSplit constructor, + │ identical to the original path. + ▼ + HoodieFileGroupReader.Builder.withFileSlice(fileSlice) [line 396] + │ this.baseFileOption = fileSlice.getBaseFile(); [line 397] + │ this.logFiles = fileSlice.getLogFiles(); [line 398] + │ + └── .build() [line 496] + │ + └── new InputSplit(baseFileOption, Either.left(logFiles), ...) [line 521] + │ + │ ┌──────────────────────────────────────────────────────────────────────┐ + │ │ InputSplit constructor [line 51] │ + │ │ │ + │ │ this.logFiles = recordsToMerge.asLeft() │ + │ │ │ + │ │ // PROPERTY 2: Final log file sort order │ + │ │ .sorted(HoodieLogFile.getLogFileComparator()) [line 56] │ + │ │ // LogFileComparator.compare(): [HoodieLogFile:215] │ + │ │ // 1st: deltaCommitTime (request time) ASC [line 239] │ + │ │ // 2nd: logVersion ASC [line 235] │ + │ │ // 3rd: writeToken ASC (nulls first) [line 223] │ + │ │ // 4th: suffix ASC [line 227] │ + │ │ │ + │ │ // PROPERTY 6: CDC log files excluded │ + │ │ .filter(logFile -> !logFile.getFileName() │ + │ │ .endsWith(HoodieCDCUtils.CDC_LOGFILE_SUFFIX)) [line 57] │ + │ │ │ + │ │ .collect(Collectors.toList()); [line 58] │ + │ └──────────────────────────────────────────────────────────────────────┘ + │ + ▼ + HoodieFileGroupReader.initRecordIterators() [line 128] +``` + +## Compaction slice merging +--- +Compaction slice merging: what it does and why + +Background: what the file group looks like when compaction is pending + +When a compaction is requested but not yet completed, the file group holds two file slices simultaneously: + +``` + fileSlices (TreeMap, reverse order): + + key = compactionInstantTime → FileSlice-C + baseFile = absent (compaction hasn't produced the parquet yet) + logFiles = [delta commits written AFTER the compaction was requested] + — these landed here via P3/P8 binding + + key = prevInstantTime → FileSlice-P (the "penultimate" slice) + baseFile = last completed parquet file + logFiles = [delta commits written BEFORE the compaction was requested] +``` + +FileSlice-C was injected by P8 (addNewFileSliceAtInstant(compactionInstantTime)) so that new log files have somewhere to land. It has no base file because thecompaction hasn't run yet. + +If a reader were handed FileSlice-C directly it would see: no base file, only the post-compaction-request log files. It would be missing all the data in FileSlice-P — the last real parquet and all prior delta commits. + + --- +What fetchMergedFileSlice does +```java + // ATFSV:1600-1617 + Option> compactionOpWithInstant = + getPendingCompactionOperationWithInstant(fileGroup.getFileGroupId()); + + if (compactionOpWithInstant.isPresent()) { + String compactionInstantTime = compactionOpWithInstant.get().getKey(); + + // Is the slice we resolved actually the compaction placeholder? + if (fileSlice.getBaseInstantTime().equals(compactionInstantTime)) { // [line 1612] + Option prevFileSlice = + fileGroup.getLatestFileSliceBefore(compactionInstantTime); // [line 1613] + if (prevFileSlice.isPresent()) { + return mergeCompactionPendingFileSlices(fileSlice, prevFileSlice.get()); // [line 1615] + } + } + } + return fileSlice; // no pending compaction, or slice is not the compaction placeholder +``` +The trigger condition is: the file slice that getLatestFileSliceBeforeOrOn(maxInstantTime) resolved has baseInstantTime == compactionInstantTime. That means the query +instant landed on or after the compaction request, and the file group is still mid-compaction. The slice the reader would get is incomplete. So the code fetches the +penultimate slice and merges them. + + --- +What mergeCompactionPendingFileSlices produces +```java + // ATFSV:1573-1583 + private static FileSlice mergeCompactionPendingFileSlices( + FileSlice lastSlice, // FileSlice-C: compaction placeholder + FileSlice penultimateSlice) // FileSlice-P: last real base file + { + // Anchor identity on the PENULTIMATE slice's base instant time + FileSlice merged = new FileSlice( + penultimateSlice.getPartitionPath(), + penultimateSlice.getBaseInstantTime(), // ← prevInstantTime, not compactionInstantTime + penultimateSlice.getFileId()); + + // Take the base file from the PENULTIMATE slice (the real parquet) + if (penultimateSlice.getBaseFile().isPresent()) { + merged.setBaseFile(penultimateSlice.getBaseFile().get()); + } + + // Append ALL log files: penultimate first, then last (post-compaction-request) + penultimateSlice.getLogFiles().forEach(merged::addLogFile); // pre-compaction deltas + lastSlice.getLogFiles().forEach(merged::addLogFile); // post-compaction deltas + + return merged; + } +``` +The result is a single FileSlice that spans the compaction boundary: +``` + merged FileSlice: + baseInstantTime = prevInstantTime ← identity of the pre-compaction slice + baseFile = last real parquet ← from FileSlice-P + logFiles = [pre-compaction deltas ... post-compaction deltas] + ← penultimate's logs + lastSlice's logs, in that order +``` + --- +Net effect: before and after + +BEFORE merge (what the file group contains physically): +``` + FileSlice-C [baseInstant = compactionInstantTime] + baseFile = (none) + logFiles = [log.3, log.4] ← written after compaction was requested + + FileSlice-P [baseInstant = prevInstantTime] + baseFile = data-20231001.parquet + logFiles = [log.1, log.2] ← written before compaction was requested +``` + +AFTER mergeCompactionPendingFileSlices: +``` + merged FileSlice [baseInstant = prevInstantTime] + baseFile = data-20231001.parquet + logFiles = [log.1, log.2, log.3, log.4] +``` +The reader gets one coherent slice — the parquet from before the compaction request, plus every delta commit written since, as if the compaction never happened. No data from either side of the compaction boundary is lost or duplicated. + +# Stack traces +MDT File listing +``` +"ScalaTest-run-running-TestMergeModeCommitTimeOrdering@1" prio=5 tid=0x1 nid=NA runnable + java.lang.Thread.State: RUNNABLE + at org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestMergedFileSliceBeforeOrOnInternal(AbstractTableFileSystemView.java:1082) + at org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestMergedFileSlicesBeforeOrOn(AbstractTableFileSystemView.java:1069) + at org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionFileSlices(HoodieTableMetadataUtil.java:1517) + at org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(HoodieTableMetadataUtil.java:1457) + at org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$readIndexRecords$8(HoodieBackedTableMetadata.java:468) + at org.apache.hudi.metadata.HoodieBackedTableMetadata$$Lambda$4332.418417286.apply(Unknown Source:-1) + at java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) + - locked <0x6ca2> (a java.util.concurrent.ConcurrentHashMap$ReservationNode) + at org.apache.hudi.metadata.HoodieBackedTableMetadata.readIndexRecords(HoodieBackedTableMetadata.java:467) + at org.apache.hudi.metadata.HoodieBackedTableMetadata.readIndexRecordsWithKeys(HoodieBackedTableMetadata.java:434) + at org.apache.hudi.metadata.HoodieBackedTableMetadata.readIndexRecordsWithKeys(HoodieBackedTableMetadata.java:428) + at org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartitionPaths(BaseTableMetadata.java:331) + at org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartitions(BaseTableMetadata.java:161) + at org.apache.hudi.BaseHoodieTableFileIndex.listPartitionPathFiles(BaseHoodieTableFileIndex.java:489) + at org.apache.hudi.BaseHoodieTableFileIndex.loadFileSlicesForPartitions(BaseHoodieTableFileIndex.java:285) + at org.apache.hudi.BaseHoodieTableFileIndex.ensurePreloadedPartitions(BaseHoodieTableFileIndex.java:267) + at org.apache.hudi.BaseHoodieTableFileIndex.getInputFileSlices(BaseHoodieTableFileIndex.java:253) + at org.apache.hudi.HoodieFileIndex.prunePartitionsAndGetFileSlices(HoodieFileIndex.scala:351) + at org.apache.hudi.HoodieFileIndex.filterFileSlices(HoodieFileIndex.scala:227) + at org.apache.spark.sql.hudi.analysis.Spark3HoodiePruneFileSourcePartitions$$anonfun$apply$1.applyOrElse(Spark3HoodiePruneFileSourcePartitions.scala:55) + at org.apache.spark.sql.hudi.analysis.Spark3HoodiePruneFileSourcePartitions$$anonfun$apply$1.applyOrElse(Spark3HoodiePruneFileSourcePartitions.scala:43) +``` + +at org.apache.hudi.BaseHoodieTableFileIndex.listPartitionPathFiles example return value +``` +org.apache.hudi.BaseHoodieTableFileIndex#listPartitionPathFiles returns list of files for the target table applying all kinds of fileters +0 = {StoragePathInfo@27959} "StoragePathInfo{path=file:/private/var/folders/sl/gfxz9xjx57ddcttsjcthbnm40000gn/T/spark-c905e48f-3cc4-400b-84d2-90b271acd3fd/.ab31b6ac-c468-4696-903e-9d2bf015a7be-0_20260406193910367.log.1_0-66-119, length=953, isDirectory=false, blockReplication=0, blockSize=33554432, modificationTime=0, locations=null}" +1 = {StoragePathInfo@27960} "StoragePathInfo{path=file:/private/var/folders/sl/gfxz9xjx57ddcttsjcthbnm40000gn/T/spark-c905e48f-3cc4-400b-84d2-90b271acd3fd/ab31b6ac-c468-4696-903e-9d2bf015a7be-0_0-26-41_20260406193834536.parquet, length=435137, isDirectory=false, blockReplication=0, blockSize=33554432, modificationTime=0, locations=null}" +2 = {StoragePathInfo@27961} "StoragePathInfo{path=file:/private/var/folders/sl/gfxz9xjx57ddcttsjcthbnm40000gn/T/spark-c905e48f-3cc4-400b-84d2-90b271acd3fd/.ab31b6ac-c468-4696-903e-9d2bf015a7be-0_20260406193838793.log.1_0-48-87, length=1144, isDirectory=false, blockReplication=0, blockSize=33554432, modificationTime=0, locations=null}" +``` + +Then we assemble the files into file groups and get the file slices +``` +"ScalaTest-run-running-TestMergeModeCommitTimeOrdering@1" prio=5 tid=0x1 nid=NA runnable + java.lang.Thread.State: RUNNABLE + at org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestMergedFileSliceBeforeOrOnInternal(AbstractTableFileSystemView.java:1085) + at org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestMergedFileSlicesBeforeOrOn(AbstractTableFileSystemView.java:1069) + at org.apache.hudi.BaseHoodieTableFileIndex.lambda$filterFiles$7(BaseHoodieTableFileIndex.java:355) + at org.apache.hudi.BaseHoodieTableFileIndex$$Lambda$4659.1351560056.apply(Unknown Source:-1) + at org.apache.hudi.common.util.Option.map(Option.java:112) + at org.apache.hudi.BaseHoodieTableFileIndex.lambda$filterFiles$9(BaseHoodieTableFileIndex.java:354) + at org.apache.hudi.BaseHoodieTableFileIndex$$Lambda$4658.27469226.apply(Unknown Source:-1) + at java.util.stream.Collectors.lambda$uniqKeysMapAccumulator$1(Collectors.java:178) + at java.util.stream.Collectors$$Lambda$173.798278875.accept(Unknown Source:-1) + at java.util.stream.ReduceOps$3ReducingSink.accept(ReduceOps.java:169) + at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1655) + at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) + at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) + at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) + at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) + at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) + at org.apache.hudi.BaseHoodieTableFileIndex.filterFiles(BaseHoodieTableFileIndex.java:350) + at org.apache.hudi.BaseHoodieTableFileIndex.loadFileSlicesForPartitions(BaseHoodieTableFileIndex.java:298) + at org.apache.hudi.BaseHoodieTableFileIndex.ensurePreloadedPartitions(BaseHoodieTableFileIndex.java:267) + at org.apache.hudi.BaseHoodieTableFileIndex.getInputFileSlices(BaseHoodieTableFileIndex.java:253) + at org.apache.hudi.HoodieFileIndex.prunePartitionsAndGetFileSlices(HoodieFileIndex.scala:351) + at org.apache.hudi.HoodieFileIndex.filterFileSlices(HoodieFileIndex.scala:227) + at org.apache.spark.sql.hudi.analysis.Spark3HoodiePruneFileSourcePartitions$$anonfun$apply$1.applyOrElse(Spark3HoodiePruneFileSourcePartitions.scala:55) + at org.apache.spark.sql.hudi.analysis.Spark3HoodiePruneFileSourcePartitions$$anonfun$apply$1.applyOrElse(Spark3HoodiePruneFileSourcePartitions.scala:43) +``` +Example return result +``` +FileSlice(fileGroupId=HoodieFileGroupId(partitionPath=, fileId=ab31b6ac-c468-4696-903e-9d2bf015a7be-0), baseInstantTime=20260406193834536, baseFile=HoodieBaseFile(fileId=ab31b6ac-c468-4696-903e-9d2bf015a7be-0, commitTime=20260406193834536, bootstrapBaseFile=Optional.empty), logFiles=[HoodieLogFile(pathStr=file:/private/var/folders/sl/gfxz9xjx57ddcttsjcthbnm40000gn/T/spark-c905e48f-3cc4-400b-84d2-90b271acd3fd/.ab31b6ac-c468-4696-903e-9d2bf015a7be-0_20260406193910367.log.1_0-66-119, fileSize=953), HoodieLogFile(pathStr=file:/private/var/folders/sl/gfxz9xjx57ddcttsjcthbnm40000gn/T/spark-c905e48f-3cc4-400b-84d2-90b271acd3fd/.ab31b6ac-c468-4696-903e-9d2bf015a7be-0_20260406193838793.log.1_0-48-87, fileSize=1144)]) +``` +could you trace further down on the path of how the file slices are built, compared to my original thought, any changes to the invariants \ No newline at end of file diff --git a/filegroup/class_domain_design_957b2dbd.md b/filegroup/class_domain_design_957b2dbd.md new file mode 100644 index 00000000..62fa1e9f --- /dev/null +++ b/filegroup/class_domain_design_957b2dbd.md @@ -0,0 +1,1249 @@ +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +Given the call stack tree which gives you a procedural view for reading log files of a file slice + + HoodieFileGroupReader.getClosableIterator() [HoodieFileGroupReader.java:301] + └─ getBufferedRecordIterator(IteratorMode) [line 291] + └─ initRecordIterators() [line 292] + │ + ├─── makeBaseFileIterator() [line 129] ← BASE FILE SCAN (iterator created here) + │ └─ readerContext.getFileRecordIterator(...) [line 157/163] + │ (returns a lazy Parquet/HFile iterator — not yet consumed) + │ + └─── recordBufferLoader.getRecordBuffer(...) [line 134] ← LOG FILE SCAN (eagerly consumed here) + │ + └─ DefaultFileGroupRecordBufferLoader.getRecordBuffer() [DefaultFileGroupRecordBufferLoader.java:54] + ├─ new KeyBasedFileGroupRecordBuffer(...) [line 78] + └─ scanLogFiles(readerContext, storage, inputSplit, ..., recordBuffer) [line 81] + │ + └─ LogScanningRecordBufferLoader.scanLogFiles() [LogScanningRecordBufferLoader.java:37] + └─ HoodieMergedLogRecordReader.newBuilder()...build() [line 41-52] + └─ HoodieMergedLogRecordReader constructor [HoodieMergedLogRecordReader.java:66] + └─ performScan() [line 75] (forceFullScan=true) + └─ BaseHoodieLogRecordReader.scanInternal() [BaseHoodieLogRecordReader.java:192] + +Limit ourselves to the procedure window from HoodieFileGroupReader to DefaultFileGroupRecordBufferLoader.getRecordBuffer, show me class/domain interaction with visualization. There should be general purpose util class, info they contain/functionality they provide(how they transform input/constructor input to the output made available via API consumed by other classes' caller), then application level class that assembles util for target functionality. + +We can give data members, class names, functionality descriptions.Then we should make sure functionality x provided by class A how that is consumed by the other class B. + +Also the relevant hiearchy of interfaces should be annotated within the actual child class and how we end up picking that child class. + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Let me read the relevant source files to understand the class interactions. + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🤖 Agent: Explore HoodieFileGroupReader + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🤖 Agent: Explore InputSplit and ReaderParameters + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Now I have a complete picture. Here's the class/domain interaction visualization: + +--- + +## Class/Domain Interaction: `HoodieFileGroupReader` → `DefaultFileGroupRecordBufferLoader.getRecordBuffer` + +### Layer 1: General-Purpose Utility / Data Classes + +These are "bags of data" or simple transformers — no application logic, just holding/converting info. + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ UTILITY / DATA CLASSES │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────────────┐ ┌──────────────────────────────┐ │ +│ │ InputSplit │ │ ReaderParameters │ │ +│ │─────────────────────────│ │──────────────────────────────│ │ +│ │ baseFileOption: Option │ │ useRecordPosition: boolean │ │ +│ │ logFiles: List │ │ emitDelete: boolean │ │ +│ │ partitionPath: String │ │ sortOutput: boolean │ │ +│ │ start: long │ │ allowInflightInstants: bool │ │ +│ │ length: long │ │──────────────────────────────│ │ +│ │─────────────────────────│ │ Constructed via Builder │ │ +│ │ hasNoRecordsToMerge() │ │ Pure query object — no logic │ │ +│ │ hasLogFiles() │ └──────────────────────────────┘ │ +│ │ getBaseFileOption() │ │ +│ └─────────────────────────┘ ┌──────────────────────────────┐ │ +│ │ HoodieReadStats │ │ +│ ┌─────────────────────────┐ │──────────────────────────────│ │ +│ │ BufferedRecordConverter │ │ numInserts/Updates/Deletes │ │ +│ │─────────────────────────│ │ totalLogReadTimeMs │ │ +│ │ Converts engine record T│ │ totalLogRecords/Blocks/Files │ │ +│ │ → BufferedRecord │ │ totalCorrupt/RollbackBlocks │ │ +│ │─────────────────────────│ │──────────────────────────────│ │ +│ │ createConverter( │ │ Mutable stats accumulator │ │ +│ │ iteratorMode, │ │ Written by: LogScanning, │ │ +│ │ schema, recordCtx, │ │ UpdateProcessor, buffer │ │ +│ │ orderingFields) │ │ Read by: caller after scan │ │ +│ │ → adapt based on mode: │ └──────────────────────────────┘ │ +│ │ ENGINE_RECORD → wrap │ │ +│ │ RECORD_KEY → extract │ │ +│ └─────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### Layer 2: Context / Service-Provider Class + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ CONTEXT / SERVICE PROVIDER (abstract) │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ HoodieReaderContext │ │ +│ │ (abstract — engine-specific subclass e.g. SparkReaderContext) │ │ +│ │─────────────────────────────────────────────────────────────────│ │ +│ │ storageConfiguration: StorageConfiguration │ │ +│ │ tableConfig: HoodieTableConfig │ │ +│ │ mergeMode: RecordMergeMode │ │ +│ │ recordMerger: Option │ │ +│ │ schemaHandler: FileGroupReaderSchemaHandler │ │ +│ │ recordContext: RecordContext │ │ +│ │ iteratorMode: IteratorMode (mutable — set before iteration) │ │ +│ │ instantRangeOpt: Option │ │ +│ │ keyFilterOpt: Option │ │ +│ │─────────────────────────────────────────────────────────────────│ │ +│ │ PROVIDES TO CONSUMERS: │ │ +│ │ • getFileRecordIterator(path,start,len,schema,..) → Iterator│ │ +│ │ [abstract — Spark impl reads Parquet via ColumnarBatchReader] │ │ +│ │ • getMergeMode() → RecordMergeMode │ │ +│ │ • getSchemaHandler() → schemas for read/write │ │ +│ │ • getRecordContext() → RecordContext for seal/key extraction │ │ +│ │ • getRecordMerger() → merger for combining base+log records │ │ +│ │ • getInstantRange() → time-based filtering of log blocks │ │ +│ │ • applyInstantRangeFilter(iter) → filtered iterator │ │ +│ │ • getRecordSizeEstimator() → for spillable map sizing │ │ +│ │ • getRecordSerializer() → for spillable map disk spill │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### Layer 3: Application-Level Classes — Assembly & Orchestration + +``` +┌──────────────────────────────────────────────────────────────────────────────┐ +│ APPLICATION-LEVEL ORCHESTRATION │ +│ │ +│ ┌────────────────────────────────────────────────────────────────────────┐ │ +│ │ HoodieFileGroupReader │ │ +│ │ THE TOP-LEVEL ORCHESTRATOR — owns the full read pipeline │ │ +│ │────────────────────────────────────────────────────────────────────────│ │ +│ │ readerContext ──────────────→ HoodieReaderContext │ │ +│ │ inputSplit ─────────────────→ InputSplit │ │ +│ │ readerParameters ───────────→ ReaderParameters │ │ +│ │ recordBufferLoader ─────────→ FileGroupRecordBufferLoader (strategy)│ │ +│ │ readStats ──────────────────→ HoodieReadStats │ │ +│ │ bufferedRecordConverter ────→ BufferedRecordConverter │ │ +│ │ baseFileIterator ───────────→ ClosableIterator (lazy parquet scan) │ │ +│ │ recordBuffer ───────────────→ HoodieFileGroupRecordBuffer (from ↓) │ │ +│ │────────────────────────────────────────────────────────────────────────│ │ +│ │ getClosableIterator() │ │ +│ │ └→ getBufferedRecordIterator(ENGINE_RECORD) │ │ +│ │ ├→ readerContext.setIteratorMode(ENGINE_RECORD) │ │ +│ │ └→ initRecordIterators() │ │ +│ │ ├→ makeBaseFileIterator() │ │ +│ │ │ Uses: readerContext.getFileRecordIterator(...) │ │ +│ │ │ Uses: BufferedRecordConverter.createConverter(...) │ │ +│ │ │ Output: this.baseFileIterator = ClosableIterator │ │ +│ │ │ │ │ +│ │ └→ IF has records to merge: │ │ +│ │ recordBufferLoader.getRecordBuffer( │ │ +│ │ readerContext, storage, inputSplit, │ │ +│ │ orderingFieldNames, metaClient, props, │ │ +│ │ readerParameters, readStats, callback) │ │ +│ │ → receives Pair │ │ +│ │ → recordBuffer.setBaseFileIterator(baseFileIterator) │ │ +│ └────────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ │ delegates to (via strategy interface) │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────────┐ │ +│ │ DefaultFileGroupRecordBufferLoader │ │ +│ │ ASSEMBLER — picks the right buffer impl + triggers log scan │ │ +│ │────────────────────────────────────────────────────────────────────────│ │ +│ │ implements: FileGroupRecordBufferLoader │ │ +│ │ extends: LogScanningRecordBufferLoader │ │ +│ │ pattern: Singleton (stateless) │ │ +│ │────────────────────────────────────────────────────────────────────────│ │ +│ │ getRecordBuffer(readerContext, storage, inputSplit, ...) │ │ +│ │ │ │ │ +│ │ ├─ STEP 1: Read config to determine merge strategy │ │ +│ │ │ isSkipMerge ← props[MERGE_TYPE] == "skip_merge" │ │ +│ │ │ partialUpdateModeOpt ← tableConfig │ │ +│ │ │ │ │ +│ │ ├─ STEP 2: Create UpdateProcessor via factory │ │ +│ │ │ UpdateProcessor.create(readStats, readerContext, │ │ +│ │ │ emitDeletes, callback, props) │ │ +│ │ │ → StandardUpdateProcessor | PayloadUpdateProcessor │ │ +│ │ │ optionally wrapped in CallbackProcessor │ │ +│ │ │ │ │ +│ │ ├─ STEP 3: Instantiate buffer (STRATEGY SELECTION) ──────────┐ │ │ +│ │ │ see decision tree below │ │ │ +│ │ │ │ │ │ +│ │ └─ STEP 4: scanLogFiles(..., recordBuffer) ← inherited │ │ │ +│ │ builds HoodieMergedLogRecordReader │ │ │ +│ │ which calls recordBuffer.processDataBlock() │ │ │ +│ │ and recordBuffer.processDeleteBlock() │ │ │ +│ │ populating the buffer's ExternalSpillableMap │ │ │ +│ │ → returns validBlockInstants │ │ │ +│ └────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +### Buffer Strategy Selection (Decision Tree in `DefaultFileGroupRecordBufferLoader`) + +``` + ┌─────────────────────────┐ + │ isSkipMerge? │ + │ (MERGE_TYPE="skip_merge")│ + └──────┬──────────┬────────┘ + YES │ │ NO + ▼ ▼ + ┌──────────────────┐ ┌──────────────────┐ + │ Unmerged │ │ sortOutputs()? │ + │ FileGroupRecord │ └───┬──────────┬───┘ + │ Buffer │ YES │ │ NO + └──────────────────┘ ▼ ▼ + ┌──────────────────┐ ┌──────────────────────┐ + │ SortedKeyBased │ │ useRecordPosition() │ + │ FileGroupRecord │ │ && baseFile present? │ + │ Buffer │ └───┬──────────┬──────┘ + └──────────────────┘ YES │ │ NO + ▼ ▼ + ┌──────────────────┐ ┌──────────────────┐ + │ PositionBased │ │ ★ KeyBased │ + │ FileGroupRecord │ │ FileGroupRecord │ + │ Buffer │ │ Buffer (DEFAULT) │ + └──────────────────┘ └──────────────────┘ +``` + +### Interface Hierarchy for Record Buffers + +``` + «interface» «interface» + HoodieFileGroupRecordBuffer FileGroupRecordBufferLoader + ┌──────────────────────────────┐ ┌────────────────────────────────┐ + │ getBufferType() │ │ getRecordBuffer(...) │ + │ processDataBlock(...) │ │ → Pair │ + │ processNextDataRecord(...) │ ├────────────────────────────────┤ + │ processDeleteBlock(...) │ │ static createDefault() │ + │ setBaseFileIterator(...) │ │ → DefaultFileGroupRecordBuf… │ + │ hasNext() / next() │ │ static createStreamingRecords… │ + │ close() │ │ static createReusable(ctx) │ + └──────────────┬───────────────┘ └──────────────┬─────────────────┘ + │ implements │ implements + ▼ ▼ + «abstract» ┌────────────────────────────────┐ + FileGroupRecordBuffer │ DefaultFileGroupRecordBuffer │ + ┌──────────────────────────────┐ │ Loader │ + │ records: ExternalSpillableMap│ │────────────────────────────────│ + │ readerContext │ │ extends LogScanningRecord │ + │ bufferedRecordMerger │ │ BufferLoader │ + │ updateProcessor │ │ Singleton (INSTANCE) │ + │ baseFileIterator │ └────────────────────────────────┘ + │──────────────────────────────│ │ + │ abstract doHasNext() │ │ extends + │ hasNext() → doHasNext() │ ▼ + │ hasNextBaseRecord(rec) │ «abstract» + │ hasNextLogRecord() │ LogScanningRecordBufferLoader + └──────────────┬───────────────┘ ┌────────────────────────────────┐ + │ extends │ scanLogFiles(ctx, storage, │ + ┌────────┼────────┬────────┐ │ inputSplit, metaClient, │ + ▼ ▼ ▼ ▼ │ props, params, stats, buf) │ + ┌──────────┐┌────────┐┌────────┐┌──────┐ │ → builds HoodieMergedLog │ + │KeyBased ││Sorted ││Position││Unmrgd│ │ RecordReader, which calls │ + │FGRecord ││KeyBased││Based ││FGRec │ │ buffer.processDataBlock() │ + │Buffer ││FGRBuf ││FGRBuf ││Buf │ │ → returns validBlockInstants │ + │──────────││────────││────────││──────│ └────────────────────────────────┘ + │KEY_BASED ││KEY_BASE││POS_BAS ││UNMRGD│ + │_MERGE ││D_MERGE ││ED_MERGE││ │ + └──────────┘└────────┘└────────┘└──────┘ +``` + +### Data Flow: How Class A's Output Feeds Class B + +``` + ① HoodieFileGroupReader.initRecordIterators() + ───────────────────────────────────────────── + CALLS: readerContext.getFileRecordIterator(basePath, start, len, dataSchema, reqSchema, storage) + GETS: ClosableIterator ← lazy Parquet row iterator (not yet consumed) + STORES: this.baseFileIterator + + ② HoodieFileGroupReader.initRecordIterators() + ───────────────────────────────────────────── + CALLS: recordBufferLoader.getRecordBuffer( + readerContext, ← provides: mergeMode, schemaHandler, recordMerger, + sizeEstimator, serializer, instantRange + storage, ← passed through to log reader + inputSplit, ← provides: logFiles list, baseFileOption, partitionPath + orderingFieldNames, ← for merge ordering + metaClient, ← provides: tableConfig (partialUpdateMode) + props, ← provides: MERGE_TYPE config + readerParameters, ← provides: sortOutputs(), useRecordPosition(), + emitDeletes(), allowInflightInstants() + readStats, ← mutable accumulator, written during scan + callback) ← optional update callback + + ③ DefaultFileGroupRecordBufferLoader.getRecordBuffer() + ───────────────────────────────────────────────────── + READS FROM readerContext: + • getMergeMode() → decides UpdateProcessor variant + • (via schemaHandler later) → schema for record conversion in buffer + READS FROM metaClient: + • tableConfig.getPartialUpdateMode() → passed to buffer constructor + READS FROM props: + • HoodieReaderConfig.MERGE_TYPE → isSkipMerge decision + READS FROM readerParameters: + • sortOutputs() → SortedKeyBased branch + • useRecordPosition() → PositionBased branch + • emitDeletes() → passed to UpdateProcessor + • allowInflightInstants() → passed to log scanner + + CREATES: UpdateProcessor via UpdateProcessor.create( + readStats, readerContext, emitDeletes, callback, props) + → StandardUpdateProcessor (default) + → PayloadUpdateProcessor (if CUSTOM merge + non-meta payload) + → optionally wrapped in CallbackProcessor + + CREATES: FileGroupRecordBuffer (one of 4 variants) + constructor receives: readerContext, metaClient, mergeMode, + partialUpdateModeOpt, props, orderingFieldNames, updateProcessor + → internally creates ExternalSpillableMap for log records + + ④ LogScanningRecordBufferLoader.scanLogFiles() [called by ③] + ─────────────────────────────────────────────── + READS FROM inputSplit: + • getLogFiles() → list of log files to scan + • getPartitionPath() → partition context + READS FROM readerContext: + • getInstantRange() → time-based block filtering + READS FROM props: + • MAX_DFS_STREAM_BUFFER_SIZE → I/O buffer sizing + + CREATES: HoodieMergedLogRecordReader (via builder) + → receives the recordBuffer instance + → calls recordBuffer.processDataBlock() / processDeleteBlock() + for each log block encountered + → this EAGERLY populates recordBuffer.records (the spillable map) + + WRITES TO readStats: + totalLogReadTimeMs, totalUpdatedRecordsCompacted, + totalLogFilesCompacted, totalLogRecords, totalLogBlocks, + totalCorruptLogBlock, totalRollbackBlocks + + RETURNS: validBlockInstants (List) + + ⑤ Back in HoodieFileGroupReader.initRecordIterators() + ───────────────────────────────────────────────────── + STORES: recordBuffer (now populated with all log records) + STORES: validBlockInstants + CALLS: recordBuffer.setBaseFileIterator(baseFileIterator) + → the buffer now owns both data sources for lazy merge iteration +``` + +### Summary Table + +| Class | Role | Key Inputs | Key Outputs | Consumed By | +|-------|------|-----------|-------------|-------------| +| `InputSplit` | Data holder | baseFile, logFiles, partition, byte range | Accessors only | FileGroupReader, BufferLoader, LogScanning | +| `ReaderParameters` | Data holder | 4 booleans (position, delete, sort, inflight) | Accessors only | FileGroupReader, BufferLoader | +| `HoodieReadStats` | Mutable accumulator | — | insert/update/delete counts, log metrics | Written by LogScanning + UpdateProcessor; read by caller | +| `BufferedRecordConverter` | Factory + converter | iteratorMode, schema, recordCtx | `convert(T) → BufferedRecord` | FileGroupRecordBuffer during merge iteration | +| `HoodieReaderContext` | Abstract service provider | storageConfig, tableConfig, schemas | fileRecordIterator, mergeMode, merger, sizeEstimator, serializer | FileGroupReader (base file iter), BufferLoader (merge config), Buffer (merge ops) | +| `UpdateProcessor` | Strategy (factory-created) | readStats, readerCtx, emitDeletes, callback | `processUpdate(key, prev, merged, isDel) → record` | FileGroupRecordBuffer during base+log merge | +| `HoodieFileGroupReader` | **Top orchestrator** | all of the above via Builder | `ClosableIterator` of merged records | Engine (Spark/Flink query) | +| `DefaultFileGroupRecordBufferLoader` | **Assembler** (singleton) | readerCtx, inputSplit, params, props | Pair | FileGroupReader.initRecordIterators() | +| `LogScanningRecordBufferLoader` | **Log scan executor** | inputSplit.logFiles, buffer, readerCtx | populates buffer + returns validInstants | DefaultFileGroupRecordBufferLoader (via inheritance) | +| `KeyBasedFileGroupRecordBuffer` | **Merge buffer** (default) | log records (eagerly loaded), base iter (lazy) | merged `BufferedRecord` stream via hasNext/next | FileGroupReader iteration loop | + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +cool, now given the even lower layer procedural view + └─ DefaultFileGroupRecordBufferLoader.getRecordBuffer() [DefaultFileGroupRecordBufferLoader.java:54] + ├─ new KeyBasedFileGroupRecordBuffer(...) [line 78] + └─ scanLogFiles(readerContext, storage, inputSplit, ..., recordBuffer) [line 81] + │ + └─ LogScanningRecordBufferLoader.scanLogFiles() [LogScanningRecordBufferLoader.java:37] + └─ HoodieMergedLogRecordReader.newBuilder()...build() [line 41-52] + └─ HoodieMergedLogRecordReader constructor [HoodieMergedLogRecordReader.java:66] + └─ performScan() [line 75] (forceFullScan=true) + └─ BaseHoodieLogRecordReader.scanInternal() [BaseHoodieLogRecordReader.java:192] + │ + ├─ [Pass 1] HoodieLogFormatReader.next() × N blocks [line 249-317] + │ (reads block headers, builds instantToBlocksMap, strips rollbacks) + │ + ├─ [Pass 2] orderedInstantsList reverse iteration [line 336-375] + │ (resolves compaction, builds currentInstantLogBlocks deque) + │ + └─ processQueuedBlocksForInstant(deque) [line 385] + └─ [for each block, oldest→newest via pollLast] + ├─ recordBuffer.processDataBlock(dataBlock, keySpecOpt) [line 437] + │ └─ KeyBasedFileGroupRecordBuffer.processDataBlock() [line 73] + │ └─ [for each record in block] + │ BufferedRecords.fromEngineRecord(...) + │ └─ processNextDataRecord(bufferedRecord, key) [line 105] + │ └─ records.put(key, deltaMerge(new, existing)) ← MAP UPDATED + │ + └─ recordBuffer.processDeleteBlock(deleteBlock) [line 440] + └─ KeyBasedFileGroupRecordBuffer.processDeleteBlock() [line 113] + └─ processNextDeletedRecord(deleteRecord, key) [line 122] + └─ records.put(key, fromDeleteRecord(...)) ← MAP UPDATED + + + +let's do the same analysis + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🤖 Agent: Explore log scanning classes + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here's the full class/domain interaction visualization for the log-scanning layer: + +--- + +## Class/Domain Interaction: `DefaultFileGroupRecordBufferLoader.getRecordBuffer` → Log Scan Internals + +### Layer 1: Low-Level I/O & Data Containers + +These classes are pure data holders or I/O primitives — no merge logic, no application decisions. + +``` +┌──────────────────────────────────────────────────────────────────────────────┐ +│ I/O & DATA CONTAINERS │ +│ │ +│ ┌──────────────────────────────┐ ┌──────────────────────────────────┐ │ +│ │ HoodieLogFormatReader │ │ HoodieLogBlock (abstract) │ │ +│ │──────────────────────────────│ │──────────────────────────────────│ │ +│ │ logFiles: List │ │ logBlockHeader: Map │ │ +│ │ storage: HoodieStorage │ │ blockContentLocation: Option │ │ +│ │ readerSchema: HoodieSchema │ │ content: Option │ │ +│ │ bufferSize: int │ │ readBlockLazily: boolean │ │ +│ │──────────────────────────────│ │──────────────────────────────────│ │ +│ │ Iterates across multiple log │ │ HeaderMetadataType: │ │ +│ │ files sequentially, yielding │ │ INSTANT_TIME │ │ +│ │ HoodieLogBlock objects one │ │ TARGET_INSTANT_TIME │ │ +│ │ at a time via hasNext/next │ │ SCHEMA │ │ +│ │ │ │ COMMAND_BLOCK_TYPE │ │ +│ │ Consumed by: │ │ COMPACTED_BLOCK_TIMES │ │ +│ │ BaseHoodieLogRecordReader │ │ RECORD_POSITIONS │ │ +│ │ .scanInternal() Pass 1 │ │──────────────────────────────────│ │ +│ └──────────────────────────────┘ │ getBlockType() → LogBlockType │ │ +│ │ isDataOrDeleteBlock() │ │ +│ │ isCompactedLogBlock() │ │ +│ │ getRecordPositions() → Bitmap │ │ +│ └───────────┬──────────────────────┘ │ +│ extends │ │ +│ ┌─────────────────────┼──────────────┐ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────────────────────┐ ┌─────────────────────────────────────┐ │ +│ │ HoodieDataBlock (abstract) │ │ HoodieDeleteBlock │ │ +│ │──────────────────────────────│ │─────────────────────────────────────│ │ +│ │ records: Option> │ │ recordsToDelete: DeleteRecord[] │ │ +│ │ keyFieldName: String │ │─────────────────────────────────────│ │ +│ │ readerSchema: HoodieSchema │ │ getRecordsToDelete() → DeleteRec[] │ │ +│ │──────────────────────────────│ │ getBlockType() → DELETE_BLOCK │ │ +│ │ getEngineRecordIterator(ctx) │ │ │ │ +│ │ → ClosableIterator │ │ Consumed by: │ │ +│ │ getSchema() → writer schema │ │ recordBuffer.processDeleteBlock() │ │ +│ │ containsPartialUpdates() │ └─────────────────────────────────────┘ │ +│ │ getBlockType() → │ │ +│ │ AVRO/PARQUET/HFILE_DATA_BLK│ ┌─────────────────────────────────────┐ │ +│ │ │ │ DeleteRecord │ │ +│ │ Consumed by: │ │─────────────────────────────────────│ │ +│ │ recordBuffer │ │ hoodieKey: HoodieKey │ │ +│ │ .processDataBlock() │ │ (recordKey + partitionPath) │ │ +│ └──────────────────────────────┘ │ orderingValue: Comparable │ │ +│ │─────────────────────────────────────│ │ +│ │ Pure value object. │ │ +│ │ Created from: HoodieDeleteBlock │ │ +│ │ Consumed by: processNextDeletedRec() │ │ +│ └─────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +### Layer 2: Record Representation & Merge Primitives + +``` +┌──────────────────────────────────────────────────────────────────────────────┐ +│ RECORD REPRESENTATION & MERGE PRIMITIVES │ +│ │ +│ ┌──────────────────────────────────┐ ┌──────────────────────────────────┐ │ +│ │ BufferedRecord │ │ BufferedRecords (factory) │ │ +│ │──────────────────────────────────│ │──────────────────────────────────│ │ +│ │ recordKey: String │ │ Static factory methods: │ │ +│ │ record: T (engine-native row) │ │ │ │ +│ │ orderingValue: Comparable │ │ fromEngineRecord(T, schema, │ │ +│ │ schemaId: Integer │ │ recordCtx, orderingFields, │ │ +│ │ hoodieOperation: HoodieOperation │ │ isDelete) │ │ +│ │──────────────────────────────────│ │ → extracts recordKey via ctx │ │ +│ │ isDelete() │ │ → extracts orderingVal via ctx │ │ +│ │ toBinary(recordCtx) │ │ → wraps in BufferedRecord │ │ +│ │ → serializes T for disk spill │ │ │ │ +│ │ seal(recordCtx) → finalize record │ │ fromDeleteRecord(deleteRecord, │ │ +│ │ replaceRecord(T) → swap payload │ │ recordCtx) │ │ +│ │──────────────────────────────────│ │ → creates BufferedRecord with │ │ +│ │ Universal currency flowing through│ │ null record, isDelete=true │ │ +│ │ the entire merge pipeline. │ └──────────────────────────────────┘ │ +│ │ Created by: BufferedRecords │ │ +│ │ Stored in: ExternalSpillableMap │ ┌──────────────────────────────────┐ │ +│ │ Merged by: BufferedRecordMerger │ │ DeleteContext │ │ +│ │ Emitted by: FileGroupRecordBuffer │ │──────────────────────────────────│ │ +│ └──────────────────────────────────┘ │ customDeleteMarkerKeyValue: │ │ +│ │ Option> │ │ +│ ┌──────────────────────────────────┐ │ hasBuiltInDeleteField: boolean │ │ +│ │ «interface» │ │ hoodieOperationPos: int │ │ +│ │ BufferedRecordMerger │ │──────────────────────────────────│ │ +│ │──────────────────────────────────│ │ Knows HOW to detect deletes │ │ +│ │ deltaMerge(newRec, existingRec) │ │ from schema metadata. │ │ +│ │ → Option> │ │ Consumed by: FileGroupRecord- │ │ +│ │ Log-vs-log merge: keeps latest │ │ Buffer constructor │ │ +│ │ by orderingValue │ │ │ │ +│ │ │ └──────────────────────────────────┘ │ +│ │ deltaMerge(deleteRec, existingRec)│ │ +│ │ → Option │ ┌──────────────────────────────────┐ │ +│ │ Delete-vs-existing merge │ │ ExternalSpillableMap │ │ +│ │ │ │ implements Map │ │ +│ │ finalMerge(olderRec, newerRec) │ │──────────────────────────────────│ │ +│ │ → BufferedRecord │ │ maxInMemorySizeInBytes: long │ │ +│ │ Base-file-vs-log merge (later) │ │ inMemoryMap: HashMap │ │ +│ │──────────────────────────────────│ │ diskBasedMap: DiskMap │ │ +│ │ Consumed by: │ │ (ROCKS_DB or BITCASK) │ │ +│ │ KeyBasedFileGroupRecordBuffer │ │ currentInMemoryMapSize: long │ │ +│ │ in processNextDataRecord() and │ │──────────────────────────────────│ │ +│ │ processNextDeletedRecord() │ │ put(key, val): │ │ +│ └──────────────────────────────────┘ │ if mem full → spill to disk │ │ +│ │ if key exists in mem → update │ │ +│ │ get(key): │ │ +│ │ check mem first, then disk │ │ +│ │──────────────────────────────────│ │ +│ │ Backing store for all log records│ │ +│ │ in FileGroupRecordBuffer.records │ │ +│ │ K = Serializable (record key) │ │ +│ │ V = BufferedRecord │ │ +│ └──────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +### Layer 3: Application-Level — Scan Orchestration & Buffer Population + +``` +┌──────────────────────────────────────────────────────────────────────────────────┐ +│ SCAN ORCHESTRATION & BUFFER POPULATION │ +│ │ +│ ┌────────────────────────────────────────────────────────────────────────────┐ │ +│ │ BaseHoodieLogRecordReader (abstract) │ │ +│ │ THE SCAN ENGINE — two-pass algorithm over log blocks │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ logFiles: List instantRange: Option │ │ +│ │ storage: HoodieStorage forceFullScan: boolean │ │ +│ │ recordBuffer: HoodieFileGroupRecordBuffer ← target for processed data │ │ +│ │ currentInstantLogBlocks: Deque ← staging area │ │ +│ │ validBlockInstants: List ← output: which instants survived │ │ +│ │ latestInstantTime: String payloadClassFQN: String │ │ +│ │ readerSchema: HoodieSchema reverseReader: boolean │ │ +│ │ totalLogFiles/Records/Blocks/Rollbacks/CorruptBlocks: AtomicLong (stats) │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ │ │ +│ │ scanInternal(keySpecOpt, skipProcessingBlocks): │ │ +│ │ │ │ +│ │ ┌─ PASS 1: Sequential block collection ──────────────────────────────┐ │ │ +│ │ │ reader = new HoodieLogFormatReader(storage, logFiles, ...) │ │ │ +│ │ │ while reader.hasNext(): │ │ │ +│ │ │ block = reader.next() ← yields HoodieLogBlock │ │ │ +│ │ │ │ │ │ │ +│ │ │ ├─ CORRUPT_BLOCK → totalCorruptBlocks++, skip │ │ │ +│ │ │ ├─ ROLLBACK (COMMAND_BLOCK) → │ │ │ +│ │ │ │ targetInstant = header[TARGET_INSTANT_TIME] │ │ │ +│ │ │ │ targetRollbackInstants.add(targetInstant) │ │ │ +│ │ │ │ totalRollbacks++ │ │ │ +│ │ │ └─ DATA or DELETE block → │ │ │ +│ │ │ blockInstant = header[INSTANT_TIME] │ │ │ +│ │ │ if NOT rolled back AND within instantRange: │ │ │ +│ │ │ instantToBlocksMap[blockInstant].add(block) │ │ │ +│ │ │ totalLogBlocks++ │ │ │ +│ │ └────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌─ PASS 2: Reverse-chronological instant processing ─────────────────┐ │ │ +│ │ │ orderedInstantsList = instantToBlocksMap.keySet() sorted │ │ │ +│ │ │ for instant in REVERSE(orderedInstantsList): │ │ │ +│ │ │ blocks = instantToBlocksMap[instant] │ │ │ +│ │ │ │ │ │ │ +│ │ │ ├─ if block.isCompactedLogBlock(): │ │ │ +│ │ │ │ compactedInstants = header[COMPACTED_BLOCK_TIMES] │ │ │ +│ │ │ │ → skip all blocks from those compacted instants │ │ │ +│ │ │ │ (compaction already includes their data) │ │ │ +│ │ │ │ │ │ │ +│ │ │ └─ add blocks to currentInstantLogBlocks deque │ │ │ +│ │ │ validBlockInstants.add(instant) │ │ │ +│ │ │ │ │ │ +│ │ │ processQueuedBlocksForInstant(currentInstantLogBlocks, keySpec) │ │ │ +│ │ └────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌─ BLOCK DISPATCH: processQueuedBlocksForInstant(deque, keySpec) ────┐ │ │ +│ │ │ while deque not empty: │ │ │ +│ │ │ block = deque.pollLast() ← oldest block first │ │ │ +│ │ │ │ │ │ │ +│ │ │ ├─ if DATA block: │ │ │ +│ │ │ │ recordBuffer.processDataBlock(block, keySpecOpt) ──────────┼───┼──→ [Buffer] +│ │ │ │ │ │ │ +│ │ │ └─ if DELETE block: │ │ │ +│ │ │ recordBuffer.processDeleteBlock(block) ────────────────────┼───┼──→ [Buffer] +│ │ └────────────────────────────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────────────────────────┘ │ +│ │ extends │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────────────┐ │ +│ │ HoodieMergedLogRecordReader │ │ +│ │ Thin subclass — adds timing + builder entry point │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ implements: Iterable>, Closeable │ │ +│ │ extends: BaseHoodieLogRecordReader │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ timer: HoodieTimer │ │ +│ │ numMergedRecordsInLog: long │ │ +│ │ totalTimeTakenToReadAndMergeBlocks: long │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ performScan(): │ │ +│ │ timer.startTimer() │ │ +│ │ scanInternal(keySpec, false) ← delegates to parent │ │ +│ │ totalTimeTaken = timer.endTimer() │ │ +│ │ numMergedRecords = recordBuffer.size() │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ Created by: LogScanningRecordBufferLoader.scanLogFiles() via Builder │ │ +│ │ Builder wires: readerContext, storage, logFiles, instantRange, │ │ +│ │ partition, recordBuffer, metaClient, allowInflightInstants │ │ +│ └────────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──── receives blocks from scan engine via processDataBlock/Delete ────┐ │ +│ ▼ │ │ +│ ┌────────────────────────────────────────────────────────────────────────────┐ │ +│ │ KeyBasedFileGroupRecordBuffer │ │ +│ │ THE MERGE BUFFER — accumulates & deduplicates log records by key │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ «interface» HoodieFileGroupRecordBuffer │ │ +│ │ └─ «abstract» FileGroupRecordBuffer │ │ +│ │ └─ KeyBasedFileGroupRecordBuffer (this class) │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ records: ExternalSpillableMap> │ │ +│ │ bufferedRecordMerger: BufferedRecordMerger │ │ +│ │ readerContext: HoodieReaderContext │ │ +│ │ deleteContext: DeleteContext │ │ +│ │ totalLogRecords: long │ │ +│ │────────────────────────────────────────────────────────────────────────────│ │ +│ │ │ │ +│ │ processDataBlock(dataBlock, keySpecOpt): │ │ +│ │ iter = getRecordsIterator(dataBlock, keySpecOpt) │ │ +│ │ → dataBlock.getEngineRecordIterator(readerContext) │ │ +│ │ → optionally applies schema evolution transformer │ │ +│ │ for each engine record T: │ │ +│ │ bufferedRec = BufferedRecords.fromEngineRecord( │ │ +│ │ record, schema, recordCtx, orderingFields, isDelete) │ │ +│ │ processNextDataRecord(bufferedRec, recordKey) │ │ +│ │ │ │ +│ │ processNextDataRecord(newRecord, key): │ │ +│ │ existingRecord = records.get(key) ← spillable map lookup │ │ +│ │ totalLogRecords++ │ │ +│ │ bufferedRecordMerger │ │ +│ │ .deltaMerge(newRecord, existingRecord) │ │ +│ │ .ifPresent(merged → │ │ +│ │ records.put(key, merged.toBinary(recordCtx))) │ │ +│ │ ↑ ↑ │ │ +│ │ │ └─ serialize T for possible disk spill │ │ +│ │ └─ spillable map: in-memory up to threshold, then to disk │ │ +│ │ │ │ +│ │ processDeleteBlock(deleteBlock): │ │ +│ │ for deleteRecord in deleteBlock.getRecordsToDelete(): │ │ +│ │ processNextDeletedRecord(deleteRecord, recordKey) │ │ +│ │ │ │ +│ │ processNextDeletedRecord(deleteRecord, key): │ │ +│ │ existingRecord = records.get(key) │ │ +│ │ bufferedRecordMerger │ │ +│ │ .deltaMerge(deleteRecord, existingRecord) │ │ +│ │ .ifPresent(surviving → │ │ +│ │ records.put(key, BufferedRecords.fromDeleteRecord( │ │ +│ │ surviving, recordCtx))) │ │ +│ └────────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────────┘ +``` + +### Data Flow: Step-by-Step Record Journey + +``` + ① HoodieLogFormatReader.next() + ────────────────────────────── + INPUT: raw bytes from log file on storage + OUTPUT: HoodieLogBlock (with lazy content — header already parsed) + ───────────────────────────────────────────────────────────────── + Key header fields extracted: + INSTANT_TIME → which commit wrote this block + TARGET_INSTANT_TIME → (rollback blocks) which instant to undo + COMPACTED_BLOCK_TIMES → (compacted blocks) which instants were folded in + SCHEMA → writer schema for data blocks + + + ② BaseHoodieLogRecordReader.scanInternal() — Pass 1 + ────────────────────────────────────────────────── + INPUT: stream of HoodieLogBlock from ① + OUTPUT: instantToBlocksMap: Map> + targetRollbackInstants: Set + + LOGIC: Classifies each block: + CORRUPT → counted, skipped + COMMAND (rollback) → target instant added to rollback set + DATA/DELETE → if not rolled back & within instantRange, + grouped by instant time + + + ③ BaseHoodieLogRecordReader.scanInternal() — Pass 2 + ────────────────────────────────────────────────── + INPUT: instantToBlocksMap from ② + OUTPUT: currentInstantLogBlocks: Deque (ordered oldest→newest) + validBlockInstants: List + + LOGIC: Iterates instants NEWEST→OLDEST: + If a block is a compacted log block, marks its constituent + instants as "already included" → those instants' blocks get skipped. + Surviving blocks are pushed onto the deque. + Net effect: deduplication at the instant level. + + + ④ BaseHoodieLogRecordReader.processQueuedBlocksForInstant() + ────────────────────────────────────────────────────────── + INPUT: deque of HoodieLogBlock (oldest first via pollLast) + OUTPUT: recordBuffer is mutated (records map populated) + + DISPATCHES: + HoodieDataBlock → recordBuffer.processDataBlock(block, keySpec) + HoodieDeleteBlock → recordBuffer.processDeleteBlock(block) + + + ⑤ KeyBasedFileGroupRecordBuffer.processDataBlock() + ───────────────────────────────────────────────── + INPUT: HoodieDataBlock + optional KeySpec filter + USES: dataBlock.getEngineRecordIterator(readerContext) + → deserializes block bytes into engine-native rows (T) + composeEvolvedSchemaTransformer(dataBlock) + → if writer schema ≠ reader schema, creates projection function + BufferedRecords.fromEngineRecord(record, schema, ctx, orderingFields, isDelete) + → extracts recordKey and orderingValue from the engine row + → wraps as BufferedRecord + CALLS: processNextDataRecord(bufferedRecord, recordKey) + + + ⑥ KeyBasedFileGroupRecordBuffer.processNextDataRecord() + ───────────────────────────────────────────────────── + INPUT: BufferedRecord (new from log), Serializable key + USES: records.get(key) → ExternalSpillableMap lookup + → checks HashMap first (O(1)), then DiskMap if spilled + bufferedRecordMerger.deltaMerge(newRecord, existingRecord) + → compares orderingValue: keeps the one with higher ordering + → if existingRecord is null, new record wins trivially + OUTPUT: records.put(key, merged.toBinary(recordCtx)) + → toBinary() serializes the engine record T for disk-spill safety + → ExternalSpillableMap: if inMemorySize > threshold, spills to disk + (RocksDB or BitCask depending on config) + + + ⑦ KeyBasedFileGroupRecordBuffer.processDeleteBlock() + ────────────────────────────────────────────────── + INPUT: HoodieDeleteBlock + USES: deleteBlock.getRecordsToDelete() → DeleteRecord[] + each DeleteRecord has: recordKey, partitionPath, orderingValue + CALLS: processNextDeletedRecord(deleteRecord, recordKey) + + + ⑧ KeyBasedFileGroupRecordBuffer.processNextDeletedRecord() + ──────────────────────────────────────────────────────── + INPUT: DeleteRecord + Serializable key + USES: records.get(key) → existing entry lookup + bufferedRecordMerger.deltaMerge(deleteRecord, existingRecord) + → compares ordering: delete wins if ordering ≥ existing + → returns Option (surviving delete) + OUTPUT: records.put(key, BufferedRecords.fromDeleteRecord(surviving, recordCtx)) + → creates BufferedRecord with record=null, isDelete=true +``` + +### How Class A's Functionality Is Consumed by Class B + +| Producer (Class A) | What It Provides | Consumer (Class B) | How B Uses It | +|---|---|---|---| +| `HoodieLogFormatReader` | `next()` → `HoodieLogBlock` | `BaseHoodieLogRecordReader.scanInternal()` Pass 1 | Reads header to classify block type, build `instantToBlocksMap` | +| `HoodieLogBlock` | `header[INSTANT_TIME]` | `BaseHoodieLogRecordReader` | Groups blocks by instant, filters by `instantRange` | +| `HoodieLogBlock` | `header[TARGET_INSTANT_TIME]` | `BaseHoodieLogRecordReader` | Identifies which instant a rollback block undoes | +| `HoodieLogBlock` | `header[COMPACTED_BLOCK_TIMES]` | `BaseHoodieLogRecordReader` Pass 2 | Marks compacted instants to skip (already folded in) | +| `HoodieDataBlock` | `getEngineRecordIterator(ctx)` → `Iterator` | `KeyBasedFGRecordBuffer.processDataBlock()` | Deserializes block bytes into engine-native rows for merge | +| `HoodieDataBlock` | `getSchema()` (writer schema) | `FileGroupRecordBuffer.composeEvolvedSchemaTransformer()` | Detects schema evolution, builds projection function if needed | +| `HoodieDeleteBlock` | `getRecordsToDelete()` → `DeleteRecord[]` | `KeyBasedFGRecordBuffer.processDeleteBlock()` | Iterates delete entries to merge against existing map entries | +| `BufferedRecords` (factory) | `fromEngineRecord(T,...)` → `BufferedRecord` | `KeyBasedFGRecordBuffer.processDataBlock()` | Wraps raw engine row with key + ordering for map insertion | +| `BufferedRecords` (factory) | `fromDeleteRecord(del, ctx)` → `BufferedRecord` | `KeyBasedFGRecordBuffer.processNextDeletedRecord()` | Creates tombstone entry (record=null) for map insertion | +| `BufferedRecordMerger` | `deltaMerge(new, existing)` → `Option` | `KeyBasedFGRecordBuffer.processNextDataRecord()` | Resolves conflicts: latest-by-ordering wins | +| `Buff + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here is the full class/domain interaction visualization for the lower layer: + +--- + +## Class/Domain Interaction: `DefaultFileGroupRecordBufferLoader.getRecordBuffer` → `scanInternal` + +### Layer 1: General-Purpose Utility / Data Classes + +``` +┌──────────────────────────────────────────────────────────────────────────────┐ +│ UTILITY / DATA / VALUE CLASSES │ +│ │ +│ ┌──────────────────────────────┐ ┌──────────────────────────────────────┐ │ +│ │ DeleteRecord │ │ BufferedRecord │ │ +│ │──────────────────────────────│ │──────────────────────────────────────│ │ +│ │ hoodieKey: HoodieKey │ │ recordKey: String │ │ +│ │ orderingValue: Comparable │ │ record: T (engine-native row) │ │ +│ │──────────────────────────────│ │ orderingValue: Comparable │ │ +│ │ Immutable value object. │ │ schemaId: Integer │ │ +│ │ Factory: create(key, part, │ │ hoodieOperation: HoodieOperation │ │ +│ │ orderingVal) │ │──────────────────────────────────────│ │ +│ │ │ │ isDelete() → checks operation │ │ +│ │ CONSUMED BY: │ │ toBinary(ctx) → serialize record T │ │ +│ │ • HoodieDeleteBlock (array) │ │ for spillable map storage │ │ +│ │ • BufferedRecords factory │ │ seal(ctx) → finalize for output │ │ +│ │ • BufferedRecordMerger │ │ │ │ +│ │ (deltaMerge overload) │ │ CONSUMED BY: everything — this is │ │ +│ └──────────────────────────────┘ │ the universal record envelope │ │ +│ └──────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────┐ ┌──────────────────────────────────────┐ │ +│ │ DeleteContext │ │ KeySpec (interface) │ │ +│ │──────────────────────────────│ │──────────────────────────────────────│ │ +│ │ customDeleteMarkerKeyValue │ │ getKeys(): List │ │ +│ │ hasBuiltInDeleteField │ │ isFullKey(): boolean │ │ +│ │ hoodieOperationPos │ │──────────────────────────────────────│ │ +│ │ readerSchema │ │ Constructed from readerContext's │ │ +│ │──────────────────────────────│ │ keyFilterOpt (predicate → key list) │ │ +│ │ Built from: props + schema │ │ │ │ +│ │ Tells: "how do I detect a │ │ CONSUMED BY: │ │ +│ │ delete in a record?" │ │ • BaseHoodieLogRecordReader │ │ +│ │ │ │ → passed to processDataBlock() │ │ +│ │ CONSUMED BY: │ │ • HoodieDataBlock │ │ +│ │ • FileGroupRecordBuffer │ │ → filters records during deser │ │ +│ │ (delete detection in merge)│ └──────────────────────────────────────┘ │ +│ └──────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ ExternalSpillableMap> │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ maxInMemorySizeInBytes: long inMemoryMap: HashMap │ │ +│ │ diskBasedMap: DiskMap (lazy) currentInMemoryMapSize: long │ │ +│ │ keySizeEstimator: SizeEstimator valueSizeEstimator: SizeEstimator │ │ +│ │ diskMapType: ROCKS_DB | BITCASK valueSerializer: CustomSerializer │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ put(key, value): │ │ +│ │ • If key exists in-memory → update in-place (no spill check) │ │ +│ │ • If new key + memory full → spill to disk via DiskMap │ │ +│ │ • Otherwise → insert to inMemoryMap │ │ +│ │ get(key): check inMemoryMap first → then diskBasedMap │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ Sizing fed by: readerContext.getRecordSizeEstimator() │ │ +│ │ readerContext.getRecordSerializer() │ │ +│ │ │ │ +│ │ CONSUMED BY: FileGroupRecordBuffer as `records` field │ │ +│ │ → all processDataBlock / processDeleteBlock writes go here │ │ +│ │ → all merge reads (hasNextBaseRecord) pull from here │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ BufferedRecords (static factory) │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ fromEngineRecord(T record, schema, recordCtx, orderingFields, isDel) │ │ +│ │ → extracts recordKey, orderingValue from engine record T │ │ +│ │ → wraps into BufferedRecord │ │ +│ │ │ │ +│ │ fromDeleteRecord(DeleteRecord, recordCtx) │ │ +│ │ → converts DeleteRecord → BufferedRecord (record=null, isDel=true) │ │ +│ │ │ │ +│ │ CONSUMED BY: KeyBasedFileGroupRecordBuffer.processDataBlock() │ │ +│ │ KeyBasedFileGroupRecordBuffer.processNextDeletedRecord() │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ BufferedRecordMerger (interface) │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ deltaMerge(newRecord, existingRecord) → Option> │ │ +│ │ "given a NEW log record and an EXISTING buffered record, │ │ +│ │ produce the merged result (or empty if record should be dropped)" │ │ +│ │ │ │ +│ │ deltaMerge(deleteRecord, existingRecord) → Option │ │ +│ │ "given a DELETE and an existing record, decide outcome" │ │ +│ │ │ │ +│ │ finalMerge(olderBaseRec, newerLogRec) → BufferedRecord │ │ +│ │ "merge a base-file record with its corresponding log record" │ │ +│ │ │ │ +│ │ CONSUMED BY: KeyBasedFileGroupRecordBuffer │ │ +│ │ processNextDataRecord → deltaMerge (log-vs-log within buffer) │ │ +│ │ processNextDeletedRecord → deltaMerge (delete-vs-existing) │ │ +│ │ hasNextBaseRecord → finalMerge (base-vs-log at read time) │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +### Layer 2: Log File I/O — Block Reading Infrastructure + +``` +┌──────────────────────────────────────────────────────────────────────────────┐ +│ LOG FILE I/O & BLOCK INFRASTRUCTURE │ +│ │ +│ «abstract» HoodieLogBlock │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ logBlockHeader: Map │ │ +│ │ logBlockFooter: Map │ │ +│ │ content: Option (lazy-inflatable) │ │ +│ │ blockContentLocation: Option │ │ +│ │ readBlockLazily: boolean │ │ +│ │─────────────────────────────────────────────────│ │ +│ │ getBlockType() → HoodieLogBlockType │ │ +│ │ inflate() / deflate() (lazy content mgmt) │ │ +│ │ getRecordPositions() → Roaring64NavigableMap │ │ +│ │ isCompactedLogBlock() → checks COMPACTED_BLOCK_ │ │ +│ │ TIMES header │ │ +│ │ HeaderMetadataType.INSTANT_TIME → commit instant │ │ +│ │ HeaderMetadataType.TARGET_INSTANT_TIME → rollback│ │ +│ └───────────────┬──────────────────┬──────────────┘ │ +│ │ │ │ +│ ┌──────────┘ └──────────┐ │ +│ ▼ ▼ │ +│ «abstract» HoodieDataBlock HoodieDeleteBlock │ +│ ┌──────────────────────────────┐ ┌──────────────────────────────┐ │ +│ │ records: Option>│ │ recordsToDelete: DeleteRecord│ │ +│ │ keyFieldName: String │ │ [] │ │ +│ │ readerSchema: HoodieSchema │ │──────────────────────────────│ │ +│ │ enablePointLookups: boolean │ │ getRecordsToDelete() │ │ +│ │──────────────────────────────│ │ → lazy deserialize from │ │ +│ │ getEngineRecordIterator(ctx) │ │ content bytes │ │ +│ │ → deserializes block bytes │ │ getBlockType() │ │ +│ │ into Iterator of │ │ → DELETE_BLOCK │ │ +│ │ engine-native records │ │ │ │ +│ │ getRecordIterator(keys, │ │ CONSUMED BY: │ │ +│ │ fullKey, type) │ │ processQueuedBlocksFor- │ │ +│ │ → filtered by KeySpec │ │ Instant → recordBuffer. │ │ +│ │──────────────────────────────│ │ processDeleteBlock(this) │ │ +│ │ Subtypes (by format): │ └──────────────────────────────┘ │ +│ │ AVRO_DATA_BLOCK │ │ +│ │ HFILE_DATA_BLOCK │ HoodieLogBlockType enum: │ +│ │ PARQUET_DATA_BLOCK │ ┌──────────────────────┐ │ +│ │ CDC_DATA_BLOCK │ │ COMMAND_BLOCK │ │ +│ │ │ │ DELETE_BLOCK │ │ +│ │ CONSUMED BY: │ │ CORRUPT_BLOCK │ │ +│ │ processQueuedBlocksFor- │ │ AVRO_DATA_BLOCK │ │ +│ │ Instant → recordBuffer. │ │ HFILE_DATA_BLOCK │ │ +│ │ processDataBlock(this, │ │ PARQUET_DATA_BLOCK │ │ +│ │ keySpecOpt) │ │ CDC_DATA_BLOCK │ │ +│ └──────────────────────────────┘ └──────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ HoodieLogFormatReader │ │ +│ │ implements: HoodieLogFormat.Reader │ │ +│ │──────────────────────────────────────────────────────────────────────│ │ +│ │ logFiles: List currentReader: HoodieLogFileReader │ │ +│ │ storage: HoodieStorage readerSchema: HoodieSchema │ │ +│ │ bufferSize: int recordKeyField: String │ │ +│ │──────────────────────────────────────────────────────────────────────│ │ +│ │ hasNext() / next() → HoodieLogBlock │ │ +│ │ Iterates across ALL log files sequentially. │ │ +│ │ Each next() returns one deserialized HoodieLogBlock │ │ +│ │ (could be DataBlock, DeleteBlock, CommandBlock, CorruptBlock) │ │ +│ │ │ │ +│ │ CONSUMED BY: BaseHoodieLogRecordReader.scanInternal() Pass 1 │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +### Layer 3: Application-Level — Log Scan Orchestration & Buffer Population + +``` +┌──────────────────────────────────────────────────────────────────────────────────┐ +│ APPLICATION-LEVEL ORCHESTRATION │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────────────────┐ │ +│ │ BaseHoodieLogRecordReader (abstract — the scan engine) │ │ +│ │──────────────────────────────────────────────────────────────────────────────│ │ +│ │ logFiles: List storage: HoodieStorage │ │ +│ │ readerContext: HoodieReaderContext hoodieTableMetaClient │ │ +│ │ recordBuffer: HoodieFileGroupRecordBuffer ← INJECTED, written to │ │ +│ │ instantRange: Option latestInstantTime: String │ │ +│ │ forceFullScan: boolean reverseReader: boolean │ │ +│ │ bufferSize: int validBlockInstants: List │ │ +│ │ currentInstantLogBlocks: Deque (staging area for Pass 2) │ │ +│ │ │ │ +│ │ ── Scan counters (AtomicLong) ── │ │ +│ │ totalLogFiles, totalLogBlocks, totalLogRecords │ │ +│ │ totalRollbacks, totalCorruptBlocks │ │ +│ │──────────────────────────────────────────────────────────────────────────────│ │ +│ │ │ │ +│ │ scanInternal(keySpecOpt, skipProcessingBlocks): │ │ +│ │ ┌────────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ PASS 1: Sequential block reading via HoodieLogFormatReader │ │ │ +│ │ │ ───────────────────────────────────────────────────────────────────── │ │ │ +│ │ │ while (logFormatReader.hasNext()): │ │ │ +│ │ │ block = logFormatReader.next() → HoodieLogBlock │ │ │ +│ │ │ │ │ │ +│ │ │ switch (block.getBlockType()): │ │ │ +│ │ │ CORRUPT_BLOCK → totalCorruptBlocks++, skip │ │ │ +│ │ │ COMMAND_BLOCK → if ROLLBACK: │ │ │ +│ │ │ targetInstant = header[TARGET_INSTANT_TIME] │ │ │ +│ │ │ targetRollbackInstants.add(targetInstant) │ │ │ +│ │ │ totalRollbacks++ │ │ │ +│ │ │ DATA_BLOCK / DELETE_BLOCK → │ │ │ +│ │ │ blockInstant = header[INSTANT_TIME] │ │ │ +│ │ │ instantToBlocksMap.computeIfAbsent(blockInstant).add(block) │ │ │ +│ │ │ │ │ │ +│ │ │ OUTPUT: instantToBlocksMap: Map> │ │ │ +│ │ │ targetRollbackInstants: Set │ │ │ +│ │ │ orderedInstantsList: ordered keys of instantToBlocksMap │ │ │ +│ │ └────────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ PASS 2: Reverse-chronological instant resolution │ │ │ +│ │ │ ───────────────────────────────────────────────────────────────────── │ │ │ +│ │ │ for each instant in orderedInstantsList (REVERSE order, newest→oldest):│ │ │ +│ │ │ │ │ │ +│ │ │ SKIP if instant ∈ targetRollbackInstants (rolled back) │ │ │ +│ │ │ SKIP if instant filtered out by instantRange │ │ │ +│ │ │ SKIP if instant > latestInstantTime && !allowInflightInstants │ │ │ +│ │ │ │ │ │ +│ │ │ for each block in instant's block list: │ │ │ +│ │ │ if block.isCompactedLogBlock(): │ │ │ +│ │ │ compactedInstants = header[COMPACTED_BLOCK_TIMES].split(",") │ │ │ +│ │ │ → remove those instants from orderedInstantsList │ │ │ +│ │ │ (they are superseded by this compaction) │ │ │ +│ │ │ │ │ │ +│ │ │ currentInstantLogBlocks.addLast(block) ← accumulate │ │ │ +│ │ │ │ │ │ +│ │ │ validBlockInstants.add(instant) │ │ │ +│ │ │ │ │ │ +│ │ │ OUTPUT: currentInstantLogBlocks: Deque │ │ │ +│ │ │ (ordered newest-instant-first because of reverse iteration, │ │ │ +│ │ │ within each instant blocks are in log-file order) │ │ │ +│ │ │ validBlockInstants: List │ │ │ +│ │ └────────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ PASS 3: processQueuedBlocksForInstant(currentInstantLogBlocks) │ │ │ +│ │ │ ───────────────────────────────────────────────────────────────────── │ │ │ +│ │ │ while (!logBlocks.isEmpty()): │ │ │ +│ │ │ block = logBlocks.pollLast() ← OLDEST first (FIFO from bottom) │ │ │ +│ │ │ │ │ │ +│ │ │ if (block instanceof HoodieDataBlock): │ │ │ +│ │ │ recordBuffer.processDataBlock(dataBlock, keySpecOpt) ──────┐ │ │ │ +│ │ │ │ │ │ │ +│ │ │ else if (block instanceof HoodieDeleteBlock): │ │ │ │ +│ │ │ recordBuffer.processDeleteBlock(deleteBlock) ──────────────┤ │ │ │ +│ │ │ │ │ │ │ +│ │ │ (other block types silently skipped) DELEGATES │ │ │ +│ │ │ TO BUFFER │ │ │ +│ │ └──────────────────────────────────────────────────────────────┼──────┘ │ │ +│ └───────────────────────────────────────────────────────────────┼──────────┘ │ +│ │ │ +│ ┌────────────────────────────────────────────────────────────────┼──────────┐ │ +│ │ HoodieMergedLogRecordReader │ │ │ +│ │ extends BaseHoodieLogRecordReader │ │ │ +│ │ implements Iterable>, Closeable │ │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ timer: HoodieTimer │ │ +│ │ numMergedRecordsInLog: long │ │ +│ │ totalTimeTakenToReadAndMergeBlocks: long │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ performScan(): │ │ +│ │ timer.startTimer() │ │ +│ │ super.scanInternal(keySpecOpt, skipProcessingBlocks=false) │ │ +│ │ totalTimeTakenToReadAndMergeBlocks = timer.endTimer() │ │ +│ │ numMergedRecordsInLog = recordBuffer.getTotalLogRecords() │ │ +│ │ │ │ +│ │ Thin wrapper: adds timing + stats on top of base scan logic │ │ +│ │ Built via Builder pattern in LogScanningRecordBufferLoader │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌───────────────────────────────────────────────────────────┘ │ +│ │ recordBuffer.processDataBlock / processDeleteBlock │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ KeyBasedFileGroupRecordBuffer │ │ +│ │ extends FileGroupRecordBuffer │ │ +│ │ implements HoodieFileGroupRecordBuffer │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ records: ExternalSpillableMap> │ │ +│ │ bufferedRecordMerger: BufferedRecordMerger │ │ +│ │ readerContext: HoodieReaderContext │ │ +│ │ totalLogRecords: long │ │ +│ │──────────────────────────────────────────────────────────────────────────│ │ +│ │ │ │ +│ │ processDataBlock(dataBlock, keySpecOpt): │ │ +│ │ ┌──────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ 1. iter = getRecordsIterator(dataBlock, keySpecOpt) │ │ │ +│ │ │ → dataBlock.getEngineRecordIterator(readerContext) │ │ │ +│ │ │ → optionally applies schema evolution transformer │ │ │ +│ │ │ → optionally applies KeySpec filter (point lookups) │ │ │ +│ │ │ │ │ │ +│ │ │ 2. for each engine record T in iter: │ │ │ +│ │ │ bufferedRec = BufferedRecords.fromEngineRecord( │ │ │ +│ │ │ record, writerSchema, recordCtx, orderingFields, isDelete) │ │ │ +│ │ │ → extracts: recordKey, orderingValue from record T │ │ │ +│ │ │ → wraps into BufferedRecord │ │ │ +│ │ │ │ │ │ +│ │ │ processNextDataRecord(bufferedRec, recordKey) │ │ │ +│ │ └──────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ processNextDataRecord(newRecord, key): │ │ +│ │ ┌──────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ existing = records.get(key) ← may be null (first time) or previous │ │ │ +│ │ │ totalLogRecords++ │ │ │ +│ │ │ merged = bufferedRecordMerger.deltaMerge(newRecord, existing) │ │ │ +│ │ │ → if existing==null: returns newRecord │ │ │ +│ │ │ → if both present: merges by ordering value / merge strategy │ │ │ +│ │ │ → Option.empty() means "drop this record" │ │ │ +│ │ │ merged.ifPresent(rec → │ │ │ +│ │ │ records.put(key, rec.toBinary(recordContext))) │ │ │ +│ │ │ ↑ toBinary serializes the engine record T so it can spill │ │ │ +│ │ │ to disk in ExternalSpillableMap if memory is full │ │ │ +│ │ └──────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ processDeleteBlock(deleteBlock): │ │ +│ │ ┌──────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ deleteRecords = deleteBlock.getRecordsToDelete() │ │ │ +│ │ │ → lazy deserialization of byte[] → DeleteRecord[] │ │ │ +│ │ │ │ │ │ +│ │ │ for each DeleteRecord: │ │ │ +│ │ │ key = deleteRecord.getRecordKey() │ │ │ +│ │ │ processNextDeletedRecord(deleteRecord, key) │ │ │ +│ │ └──────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ processNextDeletedRecord(deleteRecord, key): │ │ +│ │ ┌──────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ existing = records.get(key) │ │ │ +│ │ │ totalLogRecords++ │ │ │ +│ │ │ mergedDel = bufferedRecordMerger.deltaMerge(deleteRecord, existing) │ │ │ +│ │ │ → decides: does delete win over existing record? │ │ │ +│ │ │ → returns Option (present = delete wins) │ │ │ +│ │ │ │ │ │ +│ │ │ if mergedDel.isPresent(): │ │ │ +│ │ │ records.put(key, BufferedRecords.fromDeleteRecord( │ │ │ +│ │ │ mergedDel.get(), recordContext)) │ │ │ +│ │ │ → creates BufferedRecord with record=null, isDelete=true │ │ │ +│ │ └──────────────────────────────────────────────────────────────────────┘ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────────┘ +``` + +### Interface Hierarchy (annotated at the implementing class) + +``` + «interface» HoodieFileGroupRecordBuffer + │ + │ implements + ▼ + «abstract» FileGroupRecordBuffer + ┌──────────────────────────────────────────────────┐ + │ Owns: ExternalSpillableMap, BufferedRecordMerger, │ + │ DeleteContext, UpdateProcessor, baseFileIterator│ + │ Template methods: doHasNext() [abstract], │ + │ hasNextBaseRecord(), hasNextLogRecord() │ + └──────────────┬────────────┬────────────┬─────────┘ + │ │ │ + ┌─────────┘ ┌───────┘ ┌───────┘ + ▼ ▼ ▼ + KeyBased PositionBased Unmerged SortedKeyBased + FGRecordBuf FGRecordBuf FGRecordBuf FGRecordBuf + (extends KeyBased) + + HOW WE PICK KeyBased (in DefaultFileGroupRecordBufferLoader): + !isSkipMerge && !sortOutputs() && !(useRecordPosition() && baseFilePresent) + → falls through all checks → DEFAULT branch → new KeyBasedFileGroupRecordBuffer(...) +``` + +``` + «abstract» HoodieLogBlock + │ + ├──── «abstract» HoodieDataBlock + │ │ + │ ├── HoodieAvroDataBlock (AVRO_DATA_BLOCK) + │ ├── HoodieHFileDataBlock (HFILE_DATA_BLOCK) + │ ├── HoodieParquetDataBlock (PARQUET_DATA_BLOCK) + │ └── HoodieCDCDataBlock (CDC_DATA_BLOCK) + │ + ├──── HoodieDeleteBlock (DELETE_BLOCK) + ├──── HoodieCommandBlock (COMMAND_BLOCK — rollback markers) + └──── HoodieCorruptBlock (CORRUPT_BLOCK) +``` + +``` + «abstract» BaseHoodieLogRecordReader + │ + │ extends + ▼ + HoodieMergedLogRecordReader + (the only concrete subclass in this path — selected directly + via HoodieMergedLogRecordReader.newBuilder()...build() in + LogScanningRecordBufferLoader.scanLogFiles()) +``` + +### End-to-End Data Flow Summary + +``` + ① LogScanningRecordBufferLoader.scanLogFiles() + creates HoodieMergedLogRecordReader via builder + passes: storage, logFiles, bufferSize, instantRange, + partitionPath, recordBuffer, allowInflightInstants + + ② HoodieMergedLogRecordReader constructor → performScan() + timer.start() + calls: super.scanInternal(keySpecOpt, false) + + ③ BaseHoodieLogRecordReader.scanInternal() PASS 1 + creates: HoodieLogFormatReader(storage, logFiles, schema, ...) + reads: ALL blocks across all log files sequentially + builds: instantToBlocksMap{instant → [blocks]} + targetRollbackInstants{instant} + + ④ scanInternal() PASS 2 + iterates: instants in REVERSE chronological order + filters: rolled-back instants, out-of-range instants + resolves: compacted blocks (removes superseded instants) + accumulates: currentInstantLogBlocks deque + records: validBlockInstants + + ⑤ scanInternal() PASS 3 — processQueuedBlocksForInstant() + drains deque OLDEST→NEWEST (pollLast): + DataBlock → recordBuffer.processDataBlock(block, keySpec) + DeleteBlock → recordBuffer.processDeleteBlock(block) + + ⑥ KeyBasedFileGroupRecordBuffer.processDataBlock() + deserializes: HoodieDataBlock → Iterator engine records + for each record: + wraps: BufferedRecords.fromEngineRecord() → BufferedRecord + merges: bufferedRecordMerger.deltaMerge(new, existing) + stores: records.put(key, merged.toBinary()) + └→ ExternalSpillableMap: in-memory or spills to disk + + ⑦ KeyBasedFileGroupRecordBuffer.processDeleteBlock() + deserializes: HoodieDeleteBlock → DeleteRecord[] + for each delete: + merges: bufferedRecordMerger.deltaMerge(delete, existing) + stores: records.put(key, BufferedRecords.fromDeleteRecord(...)) + + ⑧ Back in performScan(): + totalTimeTaken = timer.end() + numMergedRecords = recordBuffer.getTotalLogRecords() + + ⑨ Back in LogScanningRecordBufferLoader: + populates HoodieReadStats from reader counters + returns: validBlockInstants → caller + recordBuffer is now fully populated with all log deltas +``` diff --git a/filegroup/howLogBlocksAreFiletered.md b/filegroup/howLogBlocksAreFiletered.md new file mode 100644 index 00000000..fdde7ca9 --- /dev/null +++ b/filegroup/howLogBlocksAreFiletered.md @@ -0,0 +1,782 @@ + +--- + +## How `scanInternal` Filters Blocks — Annotated Code + +Inside `scanInternal`, each block read from the log files goes through **4 gates** before it can enter `instantToBlocksMap`: + +```java +// BaseHoodieLogRecordReader.java, scanInternal(), Pass 1 + +while (logFormatReaderWrapper.hasNext()) { // iterates across ALL log files sequentially + HoodieLogBlock logBlock = logFormatReaderWrapper.next(); // read one block + final String instantTime = logBlock.getLogBlockHeader().get(INSTANT_TIME); // each block stamps its commit + + // ┌──────────────────────────────────────────────────┐ + // │ GATE 1: Corrupt blocks → skip silently │ + // └──────────────────────────────────────────────────┘ + if (logBlock.getBlockType().equals(CORRUPT_BLOCK)) { + continue; + } + + // ┌──────────────────────────────────────────────────────────────────────┐ + // │ GATE 2: Future blocks → skip │ + // │ Block's instant time > latestInstantTime (reader's high-watermark) │ + // │ These are from commits that haven't been acknowledged yet │ + // └──────────────────────────────────────────────────────────────────────┘ + if (logBlock.isDataOrDeleteBlock() + && compareTimestamps(instantTime, GREATER_THAN, this.latestInstantTime)) { + continue; + } + + // ┌──────────────────────────────────────────────────────────────────────┐ + // │ GATE 3: Inflight/uncommitted blocks (table version < 8 only) │ + // │ If the commit is still inflight OR not on the completed timeline, │ + // │ skip it — it's from a failed/incomplete write │ + // └──────────────────────────────────────────────────────────────────────┘ + if (logBlock.getBlockType() != COMMAND_BLOCK) { + if (this.tableVersion.lesserThan(HoodieTableVersion.EIGHT) && !allowInflightInstants) { + HoodieTimeline commitsTimeline = this.hoodieTableMetaClient.getCommitsTimeline(); + if (commitsTimeline.filterInflights().containsInstant(instantTime) + || !commitsTimeline.filterCompletedInstants().containsOrBeforeTimelineStarts(instantTime)) { + continue; // uncommitted block from failed write + } + } + + // ┌──────────────────────────────────────────────────────────────────┐ + // │ GATE 4: Instant range filter (for incremental queries) │ + // │ If an instantRange is configured, only process blocks within it │ + // └──────────────────────────────────────────────────────────────────┘ + if (instantRange.isPresent() && !instantRange.get().isInRange(instantTime)) { + continue; + } + } + + // ═══════════════════════════════════════════════════════ + // Block passed all gates — classify it + // ═══════════════════════════════════════════════════════ + switch (logBlock.getBlockType()) { + case AVRO_DATA_BLOCK: + case HFILE_DATA_BLOCK: + case PARQUET_DATA_BLOCK: + case DELETE_BLOCK: + // Add to instantToBlocksMap, keyed by instantTime + // Track arrival order in orderedInstantsList + break; + case COMMAND_BLOCK: + // Rollback → remove the target instant from both maps + targetRollbackInstants.add(targetInstant); + orderedInstantsList.remove(targetInstant); + instantToBlocksMap.remove(targetInstant); + break; + } +} +``` + +--- + +## Visual: End-to-End Flow + +``` +┌─────────────────────────────────────────────────────────────────────────────--────┐ +│ ENGINE (Spark / Flink) │ + │ │ +│ FileSystemView ──selects──► FileSlice { │ +│ fileGroupId: (partition="p1", fileId="fg-001") │ +│ baseFile: "fg-001_001_0_123.parquet" │ +│ logFiles: [ │ +│ ".fg-001_20250101.log.1", │ +│ ".fg-001_20250102.log.1", │ +│ ".fg-001_20250103.log.1" │ +│ ] │ +│ } │ +└───────────────────────────────────────┬──────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────────────────── ─┐ +│ HoodieFileGroupReader.initRecordIterators() │ +│ │ +│ ┌─ makeBaseFileIterator() ──► lazy Parquet iterator (not consumed yet) │ +│ │ │ +│ └─ recordBufferLoader.getRecordBuffer() │ +│ │ │ +│ ▼ │ +│ InputSplit sorts log files: │ +│ .fg-001_20250101.log.1 (oldest) │ +│ .fg-001_20250102.log.1 │ +│ .fg-001_20250103.log.1 (newest) │ +│ │ +│ Passes them + latestInstantTime + instantRange to: │ +└───────────────────────────────────────┬──────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────────────────── ─┐ +│ HoodieMergedLogRecordReader → scanInternal() │ +│ │ +│ External inputs available: │ +│ • logFiles = [log1, log2, log3] (pre-sorted, one file group only) │ +│ • latestInstantTime = "20250103" (high-watermark) │ +│ • instantRange = Option.empty() (or [startTime, endTime]) │ +│ • allowInflight = false (skip uncommitted blocks) │ +│ • tableVersion = 6/7/8 (controls inflight check path) │ +│ • metaClient → timeline (for inflight/completed checks) │ +│ │ +│ HoodieLogFormatReader reads blocks sequentially across log files: │ +│ │ +│ log1: [B1(i=20250101)] │ +│ log2: [B2(i=20250102), B3(i=20250102)] │ +│ log3: [B4(i=20250103), R1(target=20250102)] ← rollback │ +│ │ +│ Pass 1: Forward scan with 4 gates │ +│ │ +│ B1 ──► GATE1:ok GATE2:ok GATE3:ok GATE4:ok ──► instantToBlocksMap │ +│ B2 ──► GATE1:ok GATE2:ok GATE3:ok GATE4:ok ──► instantToBlocksMap │ +│ B3 ──► GATE1:ok GATE2:ok GATE3:ok GATE4:ok ──► instantToBlocksMap │ +│ B4 ──► GATE1:ok GATE2:ok GATE3:ok GATE4:ok ──► instantToBlocksMap │ +│ R1 ──► COMMAND_BLOCK: removes "20250102" from maps │ +│ │ +│ instantToBlocksMap after pass 1: │ +│ "20250101" → [B1] │ +│ "20250103" → [B4] │ +│ orderedInstantsList: ["20250101", "20250103"] │ +│ │ +│ Pass 2: Reverse iteration → build deque │ +│ │ +│ i=1: "20250103" → addLast(B4) to deque │ +│ i=0: "20250101" → addLast(B1) to deque │ +│ deque: [B4, B1] │ +│ │ +│ processQueuedBlocksForInstant: pollLast() drains oldest→newest │ +│ │ +│ pollLast → B1 (i=20250101) → processDataBlock → records map updated │ +│ pollLast → B4 (i=20250103) → processDataBlock → records map updated │ +│ │ +└───────────────────────────────────────────────────────────────────────────-──────┘ +``` + +--- + +## Summary: Answers to Your Two Questions + +**Q1: How does it know which file group to process?** + +`scanInternal` doesn't pick a file group — it is **told**. The file group scoping happens entirely upstream: + +``` +Engine → FileSystemView → selects FileSlice (scoped to one fileGroupId) + → HoodieFileGroupReader.Builder.withFileSlice(fileSlice) + → InputSplit sorts & stores fileSlice.getLogFiles() + → LogScanningRecordBufferLoader passes inputSplit.getLogFiles() to the reader builder + → BaseHoodieLogRecordReader stores as this.logFiles + → scanInternal() reads exactly those files, nothing more +``` + +All log files in a `FileSlice` share the same `fileId` embedded in their filename (e.g., `.fg-001_20250101.log.1`). The `FileSlice` is a pre-scoped container — one file group, one partition. + +**Q2: How does it decide which log blocks to include?** + +Four gates, applied per-block during Pass 1 of `scanInternal`: + +| Gate | Input Used | What It Filters | +|---|---|---| +| **1. Corrupt** | Block type | Drops `CORRUPT_BLOCK` | +| **2. Future** | `latestInstantTime` | Drops blocks with `INSTANT_TIME > latestInstantTime` | +| **3. Inflight** | `metaClient.getCommitsTimeline()` + `allowInflightInstants` + `tableVersion` | Drops blocks from failed/inflight writes (table version < 8 only) | +| **4. Range** | `instantRange` | Drops blocks outside `[startTime, endTime]` (incremental queries) | + +After Pass 1, rollback command blocks retroactively remove their target instants from the surviving set. Pass 2 then handles log-compaction deduplication (replacing original blocks with their compacted equivalents). + + +👤 USER: +what is the semantic of instantToBlocksMap? we have covered how it is produced, now check it's consumer code to give the answer + + +🤖 ASSISTANT: +The consumer is Pass 2 of `scanInternal`. Let me re-read it with focus on how `instantToBlocksMap` is consumed: + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java + + +🤖 ASSISTANT: +The consumer code uses `instantToBlocksMap` in exactly **3 places**, all in Pass 2: + +```java +// USE 1 (line 338): Look up the blocks for the current instant being visited +List instantsBlocks = instantToBlocksMap.get(instantTime); + +// USE 2 (line 357): For a non-compacted instant, get its blocks to add to the processing deque +List logBlocks = instantToBlocksMap.get(instantTime); + +// USE 3 (line 369): For a compacted instant, get the COMPACTED block's blocks (not the original's) +List logBlocks = instantToBlocksMap.get(compactedFinalInstantTime); +``` + +## Semantic + +**`instantToBlocksMap` is a grouping of log blocks by the commit instant that produced them.** + +- **Key**: the commit instant time (e.g., `"20250101"`) — extracted from each block's `INSTANT_TIME` header +- **Value**: the ordered list of `HoodieLogBlock`s written by that commit, in the order they appeared in the log files + +Its purpose is to make log blocks **addressable by commit instant**, so that Pass 2 can: + +1. **Skip entire commits** — if an instant was compacted into another, skip the original's blocks entirely and use the compacted block's blocks instead (USE 3 at line 369 vs USE 2 at line 357) +2. **Retrieve the right replacement** — when a data block's instant has been compacted (e.g., `i1 → i3`), the map lets you jump directly to `instantToBlocksMap.get("i3")` to get the compacted block that subsumes it + +Without this grouping, you'd have a flat list of blocks and no way to "swap out" an original commit's blocks for their compacted replacement. The map is essentially an **indirection layer** that decouples "which instants to process" (`orderedInstantsList`) from "what blocks to actually read" (`instantToBlocksMap`). + +### Concrete example + +``` +Log files contain: B1(i1), B2(i2), CB(i3, compacted=[i1,i2]), B3(i4) + +instantToBlocksMap after Pass 1: + "i1" → [B1] + "i2" → [B2] + "i3" → [CB] ← compacted block with header COMPACTED_BLOCK_TIMES="i1,i2" + "i4" → [B3] + +orderedInstantsList: ["i1", "i2", "i3", "i4"] + +Pass 2 (reverse iteration): + i4: normal block, not compacted → add B3 to deque + i3: has COMPACTED_BLOCK_TIMES → updates map: i1→i3, i2→i3 (register replacement) + i2: compacted → looks up compactedFinalInstantTime = "i3" + → instantToBlocksMap.get("i3") → [CB] ← USE 3: fetches the COMPACTED block, not B2 + → adds CB to deque, marks i3 as included + i1: compacted → looks up "i3" → already included → skip + +Final deque: [B3, CB] (newest at head, oldest at tail) +pollLast order: CB → B3 (oldest commit first → newest commit last) +``` + +The key insight: `instantToBlocksMap.get(instantTime)` is called with **either the original instant or the compacted instant**, depending on context. The map serves as a lookup table where the compaction logic can substitute one set of blocks for another by simply changing which key it looks up. + +## Call Stack Tree + +``` +HoodieMergedLogRecordReader.() // :75 + └─ performScan() // :95 + └─ BaseHoodieLogRecordReader.scanInternal() // :192 + ├─ new HoodieLogFormatReader(storage, logFiles, …) // :203 + │ └─ logFiles.remove(0) → new HoodieLogFileReader(file₀) // :61-63 + │ + ├─ PHASE 1: Forward scan (while logFormatReaderWrapper.hasNext()) // :249 + │ ├─ HoodieLogFormatReader.hasNext() // :79 + │ │ └─ currentReader.hasNext() // delegates to HoodieLogFileReader + │ │ └─ if exhausted → logFiles.remove(0) // :87 ← advances to next log file + │ │ └─ new HoodieLogFileReader(nextFile) + │ ├─ HoodieLogFormatReader.next() → HoodieLogBlock // :101 + │ ├─ GATE 1-4 filtering // :259-281 + │ └─ switch(blockType) → populate maps // :284-316 + │ + ├─ PHASE 2: Reverse traversal of orderedInstantsList // :336 + │ └─ builds currentInstantLogBlocks (Deque) + │ + └─ processQueuedBlocksForInstant(…) // :383-386 +``` + +## Phase 1: Forward Scan — Annotated Code + +Given pre-sorted log files `[log1, log2, log3]`: + +```java +// ────────────────────────────────────────────────────────────────────── +// INITIALIZATION: HoodieLogFormatReader chains the sorted log files +// into one sequential block stream. +// ────────────────────────────────────────────────────────────────────── +// HoodieLogFormatReader.java:50-64 +HoodieLogFormatReader(storage, logFiles, …) { + // logFiles = [log1, log2, log3] (already sorted by caller) + this.logFiles = logFiles; // mutable list, acts as a queue + HoodieLogFile nextLogFile = logFiles.remove(0); // pops log1 + this.currentReader = new HoodieLogFileReader(storage, nextLogFile, …); + // remaining: logFiles = [log2, log3] +} + +// HoodieLogFormatReader.java:79-97 +// hasNext() auto-advances to the next log file when current is exhausted: +boolean hasNext() { + if (currentReader.hasNext()) return true; // still blocks in current file + else if (!logFiles.isEmpty()) { + HoodieLogFile nextLogFile = logFiles.remove(0); // pop next file + this.currentReader = new HoodieLogFileReader(…, nextLogFile, …); + return hasNext(); // recursive — keeps advancing until blocks found or list empty + } + return false; +} +``` + +```java +// ────────────────────────────────────────────────────────────────────── +// BaseHoodieLogRecordReader.java:232-239 — fresh data structures +// ────────────────────────────────────────────────────────────────────── +Set targetRollbackInstants = new HashSet<>(); +Map> instantToBlocksMap = new HashMap<>(); +List orderedInstantsList = new ArrayList<>(); + +// ────────────────────────────────────────────────────────────────────── +// BaseHoodieLogRecordReader.java:249-317 — FORWARD SCAN +// Reads blocks sequentially: log1:[B1] → log2:[B2,B3] → log3:[B4,R1] +// ────────────────────────────────────────────────────────────────────── +while (logFormatReaderWrapper.hasNext()) { + HoodieLogBlock logBlock = logFormatReaderWrapper.next(); + String instantTime = logBlock.getLogBlockHeader().get(INSTANT_TIME); + + // ════════ GATE 1: Corrupt block ════════ (:259-262) + if (logBlock.getBlockType() == CORRUPT_BLOCK) { + continue; // silently skip + } + + // ════════ GATE 2: Future block ════════ (:264-267) + // Block's instant > latestInstantTime ("20250103") → skip + if (logBlock.isDataOrDeleteBlock() + && compareTimestamps(instantTime, GREATER_THAN, this.latestInstantTime)) { + continue; + } + + // ════════ GATE 3: Inflight / uncommitted (table version < 8 only) ════════ (:269-277) + if (logBlock.getBlockType() != COMMAND_BLOCK) { + if (tableVersion < V8 && !allowInflightInstants) { + HoodieTimeline commits = metaClient.getCommitsTimeline(); + if (commits.filterInflights().containsInstant(instantTime) + || !commits.filterCompletedInstants() + .containsOrBeforeTimelineStarts(instantTime)) { + continue; // uncommitted / unknown instant → skip + } + } + + // ════════ GATE 4: Instant range ════════ (:278-281) + if (instantRange.isPresent() && !instantRange.get().isInRange(instantTime)) { + continue; // outside [startTime, endTime] window + } + } + + // ──────── Block passed all gates → classify & store ──────── (:284-316) + switch (logBlock.getBlockType()) { + + case AVRO_DATA_BLOCK: + case PARQUET_DATA_BLOCK: + case HFILE_DATA_BLOCK: + case DELETE_BLOCK: + List blocks = + instantToBlocksMap.getOrDefault(instantTime, new ArrayList<>()); + if (blocks.isEmpty()) { + // First block for this instant → record arrival order + orderedInstantsList.add(instantTime); + } + blocks.add(logBlock); + instantToBlocksMap.put(instantTime, blocks); + break; + + case COMMAND_BLOCK: + HoodieCommandBlock cmd = (HoodieCommandBlock) logBlock; + if (cmd.getType() == ROLLBACK_BLOCK) { + String target = logBlock.getLogBlockHeader().get(TARGET_INSTANT_TIME); + targetRollbackInstants.add(target); + // ★ Nuke the rolled-back instant from BOTH structures + orderedInstantsList.remove(target); + instantToBlocksMap.remove(target); + } + break; + } +} +``` + +### Concrete Example Trace + +``` +logFiles (sorted): [log1, log2, log3] + +log1: [B1(i=20250101)] +log2: [B2(i=20250102), B3(i=20250102)] +log3: [B4(i=20250103), R1(target=20250102)] + + Block │ Gates │ Action │ orderedInstantsList │ instantToBlocksMap +───────┼────────────┼──────────────────────────────────┼──────────────────────────┼────────────────────────── + B1 │ all pass │ new instant → add to both │ ["20250101"] │ {101→[B1]} + B2 │ all pass │ new instant → add to both │ ["20250101","20250102"] │ {101→[B1], 102→[B2]} + B3 │ all pass │ existing instant → append block │ ["20250101","20250102"] │ {101→[B1], 102→[B2,B3]} + B4 │ all pass │ new instant → add to both │ [..,"20250102","20250103"]│ {101→[B1], 102→[B2,B3], 103→[B4]} + R1 │ COMMAND │ remove("20250102") from both │ ["20250101","20250103"] │ {101→[B1], 103→[B4]} +``` + +**Key design points:** + +- `orderedInstantsList` tracks **arrival order** (first-block-seen order), not sorted timestamp order — though in practice they tend to coincide since log files are sorted. +- `instantToBlocksMap` is a multi-map: one instant can have many blocks (e.g. B2+B3 both at `20250102`). +- Rollback blocks bypass Gates 2-4 (only Gate 1 applies) because the gate checks are wrapped in `if (logBlock.getBlockType() != COMMAND_BLOCK)` at line 269. +- `List.remove(Object)` on `orderedInstantsList` is an O(n) scan — fine because the list is small (one entry per commit instant in the file group). + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +what are the invariants between +currentInstantLogBlocks, instantTimesIncluded and validBlockInstants + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here's the Phase 2 loop again for reference (lines 326–375), then the invariants: + +```java +Set instantTimesIncluded = new HashSet<>(); // :326 +Map blockTimeToCompactionBlockTimeMap = …; // :330 + +for (int i = orderedInstantsList.size() - 1; i >= 0; i--) { // :336 (reverse) + String instantTime = orderedInstantsList.get(i); + List instantsBlocks = instantToBlocksMap.get(instantTime); + HoodieLogBlock firstBlock = instantsBlocks.get(0); + + if (firstBlock has COMPACTED_BLOCK_TIMES header) { // :345 + // This IS a compacted block — update the mapping, but don't enqueue anything yet + update blockTimeToCompactionBlockTimeMap; + } else { + String compacted = blockTimeToCompactionBlockTimeMap.get(instantTime); + + if (compacted == null) { // :355 — not compacted + reverse(blocks); blocks → currentInstantLogBlocks.addLast(…) + instantTimesIncluded.add(instantTime); + validBlockInstants.add(instantTime); + + } else if (instantTimesIncluded.contains(compacted)) { // :365 — compacted & already included + continue; // deduplicate + + } else { // :368 — compacted & not yet included + blocks of compacted instant → currentInstantLogBlocks.addLast(…) + instantTimesIncluded.add(compactedFinalInstantTime); + validBlockInstants.add(compactedFinalInstantTime); + } + } +} +``` + +## The three structures and their roles + +| Structure | Type | Scope | Purpose | +|---|---|---|---| +| `currentInstantLogBlocks` | `Deque` | instance field | The final ordered queue of blocks to be merged by `processQueuedBlocksForInstant` | +| `instantTimesIncluded` | `Set` | local to Phase 2 | Dedup guard — prevents the same compacted instant from being enqueued twice | +| `validBlockInstants` | `List` | instance field | Records which instant times actually contributed blocks, exposed via getter for downstream consumers | + +## Invariants + +**Invariant 1: `instantTimesIncluded` and `validBlockInstants` contain exactly the same instant strings.** + +Every code path that adds to one adds to the other in the same `if` branch: +- Line 360–361: `instantTimesIncluded.add(instantTime)` + `validBlockInstants.add(instantTime)` +- Line 372–373: `instantTimesIncluded.add(compactedFinalInstantTime)` + `validBlockInstants.add(compactedFinalInstantTime)` + +They differ only in data structure — `Set` vs `List` — because they serve different purposes: one for O(1) membership checks during the loop, the other for ordered output after the loop. + +**Invariant 2: Every instant in `instantTimesIncluded`/`validBlockInstants` has its blocks in `currentInstantLogBlocks`, and vice versa.** + +A block is only added to `currentInstantLogBlocks` (lines 359, 371) in the same branch that adds the instant to the other two structures. There is no path that enqueues blocks without recording the instant, and no path that records an instant without enqueuing its blocks. + +**Invariant 3: Each instant appears in `instantTimesIncluded` at most once (dedup).** + +The compaction path at line 365 checks `instantTimesIncluded.contains(compactedFinalInstantTime)` before enqueuing. This prevents a scenario like: + +``` +orderedInstantsList: [B1(i1), B2(i2), M1(i3, compacted=[i1,i2])] +``` + +When iterating in reverse: M1 updates the mapping (i1→i3, i2→i3). Then encountering i2 would enqueue i3's blocks. Without the guard, encountering i1 would enqueue i3's blocks again. The `instantTimesIncluded` set prevents this double-enqueue. + +**Invariant 4: `currentInstantLogBlocks` is ordered latest-first (reverse chronological).** + +The deque is **latest-first, earliest-last** — the exact opposite of what you described. Here's the proof: + +## 1. Filling: reverse iteration + `addLast` + +```java +// :336 — iterates orderedInstantsList from END to START (latest → earliest) +for (int i = orderedInstantsList.size() - 1; i >= 0; i--) { + String instantTime = orderedInstantsList.get(i); + List logBlocks = instantToBlocksMap.get(instantTime); + + // :358-359 — reverses blocks within the instant, then appends to TAIL + Collections.reverse(logBlocks); + logBlocks.forEach(block -> currentInstantLogBlocks.addLast(block)); +} +``` + +## 2. Draining: `pollLast` + +```java +// :431-432 — consumes from the TAIL +// poll the element at the bottom of the stack since that's the order it was inserted +HoodieLogBlock lastBlock = logBlocks.pollLast(); +``` + +## Concrete trace + +``` +orderedInstantsList = ["20250101", "20250103"] +instantToBlocksMap = { "20250101"→[B1], "20250103"→[B4] } + +Reverse iteration: + i=1 instant="20250103" blocks=[B4] → reverse → [B4] → addLast + deque: [ B4 ] + head tail + + i=0 instant="20250101" blocks=[B1] → reverse → [B1] → addLast + deque: [ B4, B1 ] + head tail +``` + +The deque now has **latest at head, earliest at tail**. + +`processQueuedBlocksForInstant` calls `pollLast()` in a loop, so it drains **tail-first**: + +``` +pollLast → B1 (earliest instant, processed first) +pollLast → B4 (latest instant, processed second) +``` + +## Multi-block instant + +``` +orderedInstantsList = ["20250101", "20250102"] +instantToBlocksMap = { "20250101"→[B1], "20250102"→[B2, B3] } + (B2 was seen before B3 in forward scan) + +Reverse iteration: + i=1 instant="20250102" blocks=[B2,B3] → reverse → [B3,B2] → addLast + deque: [ B3, B2 ] + + i=0 instant="20250101" blocks=[B1] → reverse → [B1] → addLast + deque: [ B3, B2, B1 ] + +pollLast order: B1 → B2 → B3 +``` + +## Summary + +| Property | Value | +|---|---| +| Contains all valid (non-rolled-back, gate-passing) blocks? | Yes | +| Head of deque | Latest instant, last block within that instant | +| Tail of deque | Earliest instant, first block within that instant | +| `pollLast` drain order | **Earliest-instant-first, and within an instant, earliest-block-first** (i.e. original forward-scan order) | + +The `Collections.reverse()` + `addLast` during reverse iteration is a trick to reconstruct forward chronological order at the tail. The deque is used as a stack-like structure where the "bottom" (tail) holds the oldest data — and `pollLast` drains it in exactly the original log-file order. + +**Invariant 5: Compacted block instants replace their constituent instants.** + +If i1 and i2 were compacted into M1(i3), then `validBlockInstants` will contain `i3` (not i1 or i2), and `currentInstantLogBlocks` will contain the blocks from `instantToBlocksMap.get(i3)`. The original instants i1/i2 are effectively "absorbed" — their iteration in the loop hits the `continue` at line 365/366. + + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +cool, could you also summarize the invariant for orderedInstantsList and instantToBlocksMap from pass 1 + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +## Pass 1 Invariants for `orderedInstantsList` and `instantToBlocksMap` + +**Invariant 1: `orderedInstantsList` contains exactly the keys of `instantToBlocksMap`.** + +Every mutation site maintains this: + +```java +// :289-295 — ADD: both structures updated together +if (logBlocksList.isEmpty()) { + orderedInstantsList.add(instantTime); // add key to list +} +logBlocksList.add(logBlock); +instantToBlocksMap.put(instantTime, logBlocksList); // add key to map + +// :307-309 — REMOVE: both structures updated together +orderedInstantsList.remove(targetInstantForCommandBlock); +instantToBlocksMap.remove(targetInstantForCommandBlock); +``` + +There is no code path that touches one without the other. + +**Invariant 2: `orderedInstantsList` has no duplicates.** + +The `add` to the list only fires when `logBlocksList.isEmpty()` (:290) — i.e. only on the **first** block seen for a given instant. Subsequent blocks for the same instant hit the `getOrDefault` path, find a non-empty list, and skip the list add. + +**Invariant 3: `orderedInstantsList` preserves first-seen arrival order.** + +Instants appear in the order their first block was encountered in the forward scan. Since `HoodieLogFormatReader` drains log files in sorted order and blocks within a file are sequential, this is effectively **log-file order → block-position-within-file order**. + +**Invariant 4: `instantToBlocksMap` values preserve per-instant block encounter order.** + +For a given instant, blocks are appended via `logBlocksList.add(logBlock)` (:294) in the order the forward scan sees them. If an instant spans multiple log files (e.g. B2 in log2, B5 in log3, same instant), the list reflects that cross-file order. + +**Invariant 5: Rolled-back instants are fully purged — no tombstones.** + +A rollback doesn't mark entries as invalid; it removes them entirely from both structures (:308-309). Downstream code (Phase 2) never needs to check `targetRollbackInstants` to filter — the maps are already clean. + +**Invariant 6: Every value list in `instantToBlocksMap` is non-empty.** + +An entry is created only when a block is added (:294-295). Removal is all-or-nothing via rollback (:309). There is no path that empties a list without removing the key. (Phase 2 relies on this — line 340 throws if it finds an empty list.) + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +also for any ordered data structure within the 2, how the order relates to block position within log files plus the log file order in the input log file list, cover that invariant as well + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +## Ordering Invariant: How `orderedInstantsList` and `instantToBlocksMap` value lists relate to physical log file / block position + +### Setup: the physical read order + +`HoodieLogFormatReader` yields blocks in a strict sequence determined by two axes: + +1. **Log file order**: files are consumed left-to-right from the input `logFiles` list (which the caller pre-sorts) +2. **Block position within a file**: blocks are read sequentially from byte offset 0 to EOF + +This gives a total order over all blocks. Call it the **physical scan order**: + +``` +logFiles = [log1, log2, log3] + +physical scan order: + log1:B1 < log2:B2 < log2:B3 < log3:B4 < log3:R1 + ─────── ───────────────────── ───────────────────── + file 0 file 1 file 2 +``` + +### Invariant for `orderedInstantsList`: first-seen order tracks physical scan order + +An instant is added to `orderedInstantsList` when its **first** gate-passing block is encountered (:290-292). Since the forward scan reads blocks in physical scan order, the list order is: + +> **`orderedInstantsList[i]` appeared (first gate-passing block) earlier in physical scan order than `orderedInstantsList[j]` for all `i < j`.** + +This is *not* the same as timestamp-sorted order. A counterexample: + +``` +log1: [B1(i=20250103)] ← written by a late-arriving writer +log2: [B2(i=20250101)] ← written by an earlier writer, but to a later log file + +orderedInstantsList = ["20250103", "20250101"] + ↑ first-seen ↑ second-seen +``` + +In practice this is rare because log files are typically ordered by base instant, but the code does **not** sort by timestamp — it relies purely on physical encounter order. + +### Invariant for `instantToBlocksMap` value lists: strict physical scan order within an instant + +Blocks for the same instant are appended via `logBlocksList.add(logBlock)` (:294) as they appear in the forward scan. So: + +> **For a given instant `t`, `instantToBlocksMap.get(t)` is ordered by physical scan order — earlier file first, and within the same file, earlier byte offset first.** + +Evidence — there is only one code path that appends: + +```java +// :289-295 — the ONLY mutation that adds blocks +List logBlocksList = instantToBlocksMap.getOrDefault(instantTime, new ArrayList<>()); +if (logBlocksList.isEmpty()) { + orderedInstantsList.add(instantTime); +} +logBlocksList.add(logBlock); // ← ArrayList.add = append at tail +instantToBlocksMap.put(instantTime, logBlocksList); +``` + +`ArrayList.add` appends at the tail. The forward scan visits blocks in physical scan order. So the list is physical scan order. + +Concrete example with one instant spanning two files: + +``` +log1: [B1(i=20250102)] +log2: [B2(i=20250102), B3(i=20250102)] + +instantToBlocksMap.get("20250102") = [B1, B2, B3] + ↑ ↑ ↑ + log1 log2 log2 + (physical scan order preserved) +``` + +### How Phase 2 uses these ordering guarantees + +Phase 2 reverses both: + +```java +// iterate instants latest-first (reverse of orderedInstantsList) +for (int i = orderedInstantsList.size() - 1; i >= 0; i--) { + List logBlocks = instantToBlocksMap.get(instantTime); + Collections.reverse(logBlocks); // ← reverse physical order within instant + logBlocks.forEach(block -> currentInstantLogBlocks.addLast(block)); +} +``` + +Then `processQueuedBlocksForInstant` calls `pollLast()`, which re-reverses the drain order back to physical scan order. The double reversal (once in Phase 2 fill, once in `pollLast` drain) means **blocks are ultimately processed in their original physical scan order** — earliest file, earliest position first. + + + +👤 USER: +fgClassDesign + + +👤 USER: +org.apache.hudi.common.table.read.buffer.FileGroupRecordBufferLoader#getRecordBuffer, what are 2 items in the pair stands for + + +🤖 ASSISTANT: +Looking at `DefaultFileGroupRecordBufferLoader.getRecordBuffer()` at line 81: + +```java +return Pair.of(recordBuffer, scanLogFiles(readerContext, storage, inputSplit, hoodieTableMetaClient, props, + readerParameters, readStats, recordBuffer)); +``` + +And `LogScanningRecordBufferLoader.scanLogFiles()` at line 60: + +```java +return logRecordReader.getValidBlockInstants(); +``` + +So the pair is: + +| Position | Type | What it is | +|---|---|---| +| **Left** | `HoodieFileGroupRecordBuffer` | The record buffer (e.g., `KeyBasedFileGroupRecordBuffer`) — already populated with all log records after `scanLogFiles` returns | +| **Right** | `List` | `validBlockInstants` — the list of commit instant times from log blocks that were actually processed (excluding rolled-back, corrupt, or filtered-out blocks) | + +The `validBlockInstants` list is collected during `scanInternal` Pass 2 (`BaseHoodieLogRecordReader.java:361,371`), where each non-compacted or final-compacted instant gets added. It's then used by log-compaction to know which instants were read, enabling optimized log-block scans in subsequent operations. You can see the consumer back in `HoodieFileGroupReader.initRecordIterators()` at line 137: + +```java +validBlockInstants = initializationResult.getRight(); +``` + + +👤 USER: +Where is the base file content? + + +🤖 ASSISTANT: +The base file content is **not** in the pair. It's separate. + +Looking at `HoodieFileGroupReader.initRecordIterators()` (line 128): + +```java +private void initRecordIterators() throws IOException { + ClosableIterator iter = makeBaseFileIterator(); // ← base file iterator created here + if (inputSplit.hasNoRecordsToMerge()) { + this.baseFileIterator = new CloseableMappingIterator<>(...); // no logs → just use base file directly + } else { + this.baseFileIterator = iter; + Pair, List> initializationResult = + recordBufferLoader.getRecordBuffer(...); // ← log scan happens here, returns buffer + instants + recordBuffer = initializationResult.getLeft(); + validBlockInstants = initializationResult.getRight(); + recordBuffer.setBaseFileIterator(baseFileIterator); // ← base file iterator injected into buffer + } +} +``` + +The base file content lives in `this.baseFileIterator` — a lazy Parquet iterator created by `makeBaseFileIterator()` at line 129. It's **not** part of the `getRecordBuffer` return pair at all. Instead, it's handed to the buffer separately via `recordBuffer.setBaseFileIterator(baseFileIterator)` at line 138. + +The base file is only consumed later, lazily, when the consumer calls `recordBuffer.hasNext()` → `doHasNext()` → `baseFileIterator.next()` during Phase B. + +So to summarize where each piece of data lives after `initRecordIterators()`: + +| Data | Where | When consumed | +|---|---|---| +| Log file records | `recordBuffer.records` map (already populated) | Phase B (merge with base) + Phase C (log-only inserts) | +| Base file records | `baseFileIterator` (lazy, untouched) | Phase B, one at a time via `hasNext()`/`next()` | +| Valid block instants | `validBlockInstants` list | Used by log-compaction later | + diff --git a/filegroup/inputOfFgReader b/filegroup/inputOfFgReader new file mode 100644 index 00000000..7f975e7f --- /dev/null +++ b/filegroup/inputOfFgReader @@ -0,0 +1,115 @@ +╭─── Claude Code v2.1.92 ─────────────────────────────────────────────────────────────────────────────────────╮ +│ │ Tips for getting started │ +│ Welcome back Davis! │ Run /init to create a CLAUDE.md file with instructions for Claude │ +│ │ ───────────────────────────────────────────────────────────────── │ +│ ▗ ▗ ▖ ▖ │ Recent activity │ +│ │ No recent activity │ +│ ▘▘ ▝▝ │ │ +│ │ │ +│ Sonnet 4.6 · Claude Team · Onehouse │ │ +│ ~/hudi-oss │ │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + +❯ how is HoodieFileGroupReader got constructed for MOR table read. What are all the input it get. show me raw code + + Searched for 2 patterns, read 1 file (ctrl+o to expand) + +⏺ Now let me look at how it's constructed in the Spark path for MOR reads: + + Read 1 file (ctrl+o to expand) + +⏺ Here's the full picture of how HoodieFileGroupReader is constructed for MOR reads. + + --- + Constructor (private, via Builder) + + HoodieFileGroupReader.java:91-123 + private HoodieFileGroupReader(HoodieReaderContext readerContext, HoodieStorage storage, String tablePath, + String latestCommitTime, HoodieSchema dataSchema, HoodieSchema requestedSchema, + Option internalSchemaOpt, HoodieTableMetaClient hoodieTableMetaClient, + TypedProperties props, ReaderParameters readerParameters, InputSplit inputSplit, + Option> updateCallback, + FileGroupRecordBufferLoader recordBufferLoader) + + --- + Builder fields (all inputs) + + HoodieFileGroupReader.java:363-384 + public static class Builder { + private HoodieReaderContext readerContext; // engine-specific (Spark InternalRow, Flink RowData) + private HoodieStorage storage; // derived from metaClient + storageConf + private String tablePath; // base path + private String latestCommitTime; // query timestamp / end instant + private HoodieSchema dataSchema; // full table schema + private HoodieSchema requestedSchema; // projected schema (required cols only) + private Option internalSchemaOpt = Option.empty(); // schema evolution + private HoodieTableMetaClient hoodieTableMetaClient; + private TypedProperties props; // merged hoodie + spark configs + private Option baseFileOption; // parquet/orc base file (can be absent for log-only) + private Stream logFiles; // log files in the file slice (MOR delta logs) + private String partitionPath; + private long start = 0; // byte offset into base file (for splitting) + private long length = Long.MAX_VALUE; // byte length to read from base file + private Iterator recordIterator; // alternative: pre-built record stream + private boolean shouldUseRecordPosition = false; // use parquet row-index for faster merge + private boolean allowInflightInstants = false; + private boolean emitDelete; // whether to emit delete records + private boolean sortOutput = false; + private Option> fileGroupUpdateCallback = Option.empty(); + private FileGroupRecordBufferLoader recordBufferLoader; // how log records are loaded into buffer + } + + The InputSplit wraps: baseFileOption + logFiles (or recordIterator) + partitionPath + start + length. + + --- + Spark MOR construction site + + HoodieFileGroupReaderBasedFileFormat.scala:262-284 + val readerContext = new SparkFileFormatInternalRowReaderContext( + fileGroupBaseFileReader.value, filters, requiredFilters, storageConf, metaClient.getTableConfig) + readerContext.setEnableLogicalTimestampFieldRepair(...) + + val reader = HoodieFileGroupReader.newBuilder() + .withReaderContext(readerContext) // SparkFileFormatInternalRowReaderContext + .withHoodieTableMetaClient(metaClient) // built on driver, broadcast implicitly + .withLatestCommitTime(queryTimestamp) // snapshot commit time + .withFileSlice(fileSlice) // baseFile + logFiles + partitionPath + .withDataSchema(dataSchema) // full table avro schema + .withRequestedSchema(requestedSchema) // pruned projection schema + .withInternalSchema(internalSchemaOpt) // schema evolution (may be empty) + .withProps(props) // table config props + spark options + memory config + .withStart(file.start) // PartitionedFile split offset + .withLength(baseFileLength) // base file byte length + .withShouldUseRecordPosition(shouldUseRecordPosition) + .build() + + props at line 264-266 is assembled as: + val props = metaClient.getTableConfig.getProps // table-level hoodie configs + options.foreach(kv => props.setProperty(kv._1, kv._2)) // spark datasource options + props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction)) + + --- + Key things that happen inside the constructor + + ┌────────────────────────────────────────────────────┬────────────────────────────────────────────────────────────────────────────────────┐ + │ Step │ What │ + ├────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤ + │ InputSplit built │ wraps base file + log files + partition path + byte range │ + ├────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤ + │ readerContext.setHasLogFiles(...) │ MOR flag set from whether inputSplit.hasLogFiles() │ + ├────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤ + │ readerContext.setShouldMergeUseRecordPosition(...) │ only true when: shouldUseRecordPosition && hasLogFiles && isParquetBaseFile && │ + │ │ !skipMerge │ + ├────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤ + │ readerContext.setSchemaHandler(...) │ either ParquetRowIndexBasedSchemaHandler or FileGroupReaderSchemaHandler │ + ├────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤ + │ orderingFieldNames │ derived from mergeMode + table config (precombine fields) │ + └────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────────────────────┘ + +✻ Sautéed for 45s + +─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +❯  +─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + ? for shortcuts · hold Space to speak + diff --git a/filegroup/logFileBasics_b574f1d8.md b/filegroup/logFileBasics_b574f1d8.md new file mode 100644 index 00000000..8f783b74 --- /dev/null +++ b/filegroup/logFileBasics_b574f1d8.md @@ -0,0 +1,1115 @@ + +👤 USER: +introduce me the file group for table version 9. I only care parquet as base file avro as log file. +- log file types, what do they stand for, what are the special properties each one contains, also what kind of information can be extracted from the file - schema?commit time?data? +- log file name and base file name what are the information they contain. +- how do they relate to notions of file group and file slice +- would a file slice contain both committed and uncommited code change? If so, how they are differentiated. +- for committed changes within a file slice, how do we know which changes comes from which instant time / completion time +please read relevant tests, file group reader code and other places extensively to give a comprehensive intro covering these questions. I expect raw code pieces with annotation to serve as support of every claim you make. + + +🤖 ASSISTANT: + 🤖 Agent: Explore log file types and properties — I need a comprehensive understanding of Hudi log file types for table version 9 (Parquet base, Avro log). Research these questions thoroughly: + +1. What are the different log block types? Find the enum + + +🤖 ASSISTANT: + 🤖 Agent: Explore file naming conventions — I need to understand Hudi file naming conventions for table version 9 (TV9). Research: + +1. Base file (Parquet) naming - what information is encoded in the file name? +2. Log file (Avro) naming - what i + + +🤖 ASSISTANT: + 🤖 Agent: Explore file group and file slice — I need to understand the relationship between file groups, file slices, base files, and log files in Hudi table version 9. Research: + +1. What is a HoodieFileGroup? What does it contain? +2. What is a F + + +🤖 ASSISTANT: + 🤖 Agent: Explore file group reader and tests — I need to understand how Hudi reads file groups in table version 9, specifically the record buffer and merge logic. Research: + +1. How does the file group reader work? What's the reading flow? +2. How a + + +🤖 ASSISTANT: +Now let me read the key source files to get precise code snippets for the response. + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/KeyBasedFileGroupRecordBuffer.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/KeyBasedFileGroupRecordBuffer.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java + + +🤖 ASSISTANT: +Now let me read the test file that the user has modified, and check the log file scanner's handling of COMMAND_BLOCK for rollback. + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestKeyBasedFileGroupRecordBuffer.java + + +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-oss/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java + + +🤖 ASSISTANT: +Now I have all the information I need. Let me compose the comprehensive answer. + +--- + +# File Groups in Hudi Table Version 9 (Parquet Base + Avro Log) + +## 1. Log Block Types + +Defined in `HoodieLogBlock.java:187-219`: + +```java +public enum HoodieLogBlockType { + COMMAND_BLOCK(":command", HoodieTableVersion.ONE), + DELETE_BLOCK(":delete", HoodieTableVersion.ONE), + CORRUPT_BLOCK(":corrupted", HoodieTableVersion.ONE), + AVRO_DATA_BLOCK("avro", HoodieTableVersion.ONE), + HFILE_DATA_BLOCK("hfile", HoodieTableVersion.ONE), + PARQUET_DATA_BLOCK("parquet", HoodieTableVersion.FOUR), + CDC_DATA_BLOCK("cdc", HoodieTableVersion.SIX); +} +``` + +For TV9 with Parquet base + Avro log, the relevant block types are: + +| Block Type | What It Stores | Key Properties | +|---|---|---| +| **AVRO_DATA_BLOCK** | Upsert data records (binary Avro) | Schema in header, record positions bitmap, ordering values per record | +| **DELETE_BLOCK** | Array of `DeleteRecord` (key + ordering value) | Uses `HoodieDeleteRecordList` Avro format (v3) | +| **COMMAND_BLOCK** | No data content — metadata-only | `COMMAND_BLOCK_TYPE` header (currently only `ROLLBACK_BLOCK`), plus `TARGET_INSTANT_TIME` to identify which instant to rollback | +| **CORRUPT_BLOCK** | Raw corrupted bytes | Created when block size mismatch is detected during scanning | +| **CDC_DATA_BLOCK** | Change-data-capture records | Extends `HoodieAvroDataBlock`, appended to `.cdc` suffixed log files | + +### Header Metadata Each Block Can Carry + +Defined in `HoodieLogBlock.java:225-242`: + +```java +public enum HeaderMetadataType { + INSTANT_TIME(HoodieTableVersion.ONE), // commit instant that wrote this block + TARGET_INSTANT_TIME(HoodieTableVersion.ONE), // for COMMAND_BLOCK: the instant being rolled back + SCHEMA(HoodieTableVersion.ONE), // writer Avro schema JSON string + COMMAND_BLOCK_TYPE(HoodieTableVersion.ONE), // ordinal of HoodieCommandBlockTypeEnum + COMPACTED_BLOCK_TIMES(HoodieTableVersion.FIVE), // comma-separated instants merged by log-compaction + RECORD_POSITIONS(HoodieTableVersion.SIX), // Base64-encoded Roaring64NavigableMap bitmap + BLOCK_IDENTIFIER(HoodieTableVersion.SIX), // unique block ID + IS_PARTIAL(HoodieTableVersion.EIGHT), // "true" if partial update block + BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS(HoodieTableVersion.EIGHT); // base file instant for positions +} +``` + +**What can be extracted from a log file:** +- **Schema** — via `SCHEMA` header (writer Avro schema JSON string) +- **Commit time** — via `INSTANT_TIME` header (the instant that wrote the block) +- **Data records** — via `HoodieDataBlock.getEngineRecordIterator()` +- **Delete keys** — via `HoodieDeleteBlock.getRecordsToDelete()` +- **Record positions** — via `getRecordPositions()` (Roaring bitmap of positions in the base file this block updates) + +Evidence from `HoodieLogBlock.java:143-148`: +```java +public Roaring64NavigableMap getRecordPositions() throws IOException { + if (!logBlockHeader.containsKey(HeaderMetadataType.RECORD_POSITIONS)) { + return new Roaring64NavigableMap(); + } + return LogReaderUtils.decodeRecordPositionsHeader(logBlockHeader.get(HeaderMetadataType.RECORD_POSITIONS)); +} +``` + +--- + +## 2. File Name Anatomy + +### Base File (Parquet) + +Format from `FSUtils.java:120-121`: +```java +public static String makeBaseFileName(String instantTime, String writeToken, String fileId, String fileExtension) { + return String.format("%s_%s_%s%s", fileId, writeToken, instantTime, fileExtension); +} +``` + +Pattern: **`{fileId}_{writeToken}_{instantTime}.parquet`** + +Example: `136281f3-c24e-423b-a65a-95dbfbddce1d_1-0-1_20240101120000.parquet` + +- **fileId**: UUID identifying the file group +- **writeToken**: `{taskPartitionId}-{stageId}-{taskAttemptId}` (from `FSUtils.java:116-117`) +- **instantTime**: the commit timestamp when this base file was created + +Extraction via `FSUtils.java:133-142`: +```java +public static String getCommitTime(String fullFileName) { + if (isLogFile(fullFileName)) { + return fullFileName.split("_")[1].split("\\.", 2)[0]; // 2nd segment for logs + } + return fullFileName.split("_")[2].split("\\.", 2)[0]; // 3rd segment for base files +} +``` + +### Log File (Avro) + +Pattern from `FSUtils.java:81-82`: +```java +public static final Pattern LOG_FILE_PATTERN = + Pattern.compile("^\\.([^._]+)_([^.]*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(\\.cdc)?)?$"); +``` + +Format: **`.{fileId}_{deltaCommitTime}.log.{logVersion}_{writeToken}`** + +Example: `.136281f3-c24e-423b-a65a-95dbfbddce1d_20240101130000.log.1_1-0-1` + +Parsed in `HoodieLogFile.java:111-122`: +```java +private void parseFieldsFromPath() { + Matcher matcher = LOG_FILE_PATTERN.matcher(getPath().getName()); + this.fileId = matcher.group(1); // same fileId as its base file + this.deltaCommitTime = matcher.group(2); // instant that wrote this log + this.fileExtension = matcher.group(3); // "log" or "archive" + this.logVersion = Integer.parseInt(matcher.group(4)); // version number (monotonically increasing) + this.logWriteToken = matcher.group(6); // writeToken for disambiguation + this.suffix = matcher.group(10); // ".cdc" for CDC log files, else empty +} +``` + +Key difference: log files start with `.` prefix and encode `deltaCommitTime` as the 2nd segment (not 3rd like base files). + +--- + +## 3. File Group and File Slice Relationship + +### HoodieFileGroup + +From `HoodieFileGroup.java:42-47`: +```java +/** + * A set of data/base files + set of log files, that make up a unit for all operations. + */ +public class HoodieFileGroup implements Serializable { + private final HoodieFileGroupId fileGroupId; // (partitionPath, fileId) + private final TreeMap fileSlices; // keyed by baseInstantTime, reverse-sorted + private final HoodieTimeline timeline; + private final Option lastInstant; // high watermark +} +``` + +A **file group** is identified by `(partitionPath, fileId)`. It's a container of **file slices** ordered by time. + +### FileSlice + +From `FileSlice.java:37-43`: +```java +/** + * Within a file group, a slice is a combination of data file written at a commit time + * and list of log files, containing changes to the data file from that commit time. + */ +public class FileSlice implements Serializable { + private final HoodieFileGroupId fileGroupId; + private final String baseInstantTime; // the instant that created this slice + private HoodieBaseFile baseFile; // 0 or 1 parquet file + private final TreeSet logFiles; // 0..N log files, reverse-sorted by version +} +``` + +### How they relate — visually: + +``` +FileGroup (fileId = "abc-123", partition = "2024/01/01") + │ + ├── FileSlice (baseInstantTime = "t3") ← latest after compaction + │ ├── baseFile: abc-123_1-0-1_t3.parquet + │ ├── logFile: .abc-123_t4.log.1_1-0-1 + │ └── logFile: .abc-123_t5.log.2_1-0-1 + │ + └── FileSlice (baseInstantTime = "t1") ← older (kept for time-travel) + └── baseFile: abc-123_1-0-1_t1.parquet +``` + +**How a log file is assigned to a file slice** — `HoodieFileGroup.java:129-148`: + +```java +public String getBaseInstantTime(CompletionTimeQueryView completionTimeQueryView, HoodieLogFile logFile) { + if (fileSlices.isEmpty()) { + return logFile.getDeltaCommitTime(); // no base file yet + } + Option completionTimeOpt = completionTimeQueryView + .getCompletionTime(fileSlices.firstKey(), logFile.getDeltaCommitTime()); + if (completionTimeOpt.isPresent()) { + // find the largest base instant time <= log's completion time + for (String commitTime : fileSlices.keySet()) { + if (compareTimestamps(completionTimeOpt.get(), GREATER_THAN_OR_EQUALS, commitTime)) { + return commitTime; + } + } + return logFile.getDeltaCommitTime(); + } + // pending log file → belongs to latest file slice + return fileSlices.firstKey(); +} +``` + +The `CompletionTimeQueryView` resolves the log file's `deltaCommitTime` to an actual completion time, then finds the most recent base file slice that was created before that completion time. + +--- + +## 4. Committed vs Uncommitted Changes Within a File Slice + +### At the file-slice level + +A file slice is considered committed based on `HoodieFileGroup.java:158-164`: + +```java +private boolean isFileSliceCommitted(FileSlice slice) { + if (!compareTimestamps(slice.getBaseInstantTime(), LESSER_THAN_OR_EQUALS, + lastInstant.get().requestedTime())) { + return false; + } + return timeline.containsOrBeforeTimelineStarts(slice.getBaseInstantTime()); +} +``` + +- `getAllFileSlices()` returns **only committed** slices (line 183-188) +- `getAllFileSlicesIncludingInflight()` returns **all** slices including pending ones (line 169-171) + +### At the log-block level (within a file slice) + +A single file slice can contain log files from both committed and inflight instants. The **log record scanner** filters them during read. + +From `AbstractHoodieLogRecordScanner.java:309-333`: + +```java +final String instantTime = logBlock.getLogBlockHeader().get(INSTANT_TIME); + +// Skip blocks from future instants (beyond the reader's high watermark) +if (logBlock.isDataOrDeleteBlock() + && compareTimestamps(logBlock.getLogBlockHeader().get(INSTANT_TIME), + GREATER_THAN, this.latestInstantTime)) { + continue; +} + +// For table version < 8: explicitly filter uncommitted blocks +if (this.tableVersion.lesserThan(HoodieTableVersion.EIGHT) && !allowInflightInstants) { + if (!getOrCreateCompletedInstantsTimeline().containsOrBeforeTimelineStarts(instantTime) + || getOrCreateInflightInstantsTimeline().containsInstant(instantTime)) { + // hit an uncommitted block from a failed write — skip it + continue; + } +} + +// Optional instant range filter +if (instantRange.isPresent() && !instantRange.get().isInRange(instantTime)) { + continue; +} +``` + +For **TV9 (>= EIGHT)**: the `tableVersion.lesserThan(HoodieTableVersion.EIGHT)` check is `false`, so uncommitted blocks are **not** filtered by this path. Instead, the pending log file is simply assigned to the latest file slice by `getBaseInstantTime()` returning `fileSlices.firstKey()` (the "pending" branch at line 147). + +### Rollback of uncommitted changes + +Failed writes are cleaned up via `COMMAND_BLOCK` with `ROLLBACK_BLOCK` type. From `AbstractHoodieLogRecordScanner.java:349-363`: + +```java +case COMMAND_BLOCK: + HoodieCommandBlock commandBlock = (HoodieCommandBlock) logBlock; + if (commandBlock.getType().equals(HoodieCommandBlockTypeEnum.ROLLBACK_BLOCK)) { + String targetInstantForCommandBlock = + logBlock.getLogBlockHeader().get(TARGET_INSTANT_TIME); + targetRollbackInstants.add(targetInstantForCommandBlock); + orderedInstantsList.remove(targetInstantForCommandBlock); + instantToBlocksMap.remove(targetInstantForCommandBlock); + } +``` + +So a rollback block with `TARGET_INSTANT_TIME = "t5"` causes all data/delete blocks from instant "t5" to be removed from the processing queue. + +--- + +## 5. Tracking Which Instant Each Change Comes From + +### Blocks are grouped by instant time + +The scanner groups blocks into `instantToBlocksMap` keyed by their `INSTANT_TIME` header (`AbstractHoodieLogRecordScanner.java:336-347`): + +```java +case AVRO_DATA_BLOCK: +case PARQUET_DATA_BLOCK: +case DELETE_BLOCK: + List logBlocksList = + instantToBlocksMap.getOrDefault(instantTime, new ArrayList<>()); + if (logBlocksList.isEmpty()) { + orderedInstantsList.add(instantTime); // track arrival order + } + logBlocksList.add(logBlock); + instantToBlocksMap.put(instantTime, logBlocksList); +``` + +### Log-compaction tracking + +When a log-compacted block is encountered, the `COMPACTED_BLOCK_TIMES` header tells us which original instants were merged into it. From `AbstractHoodieLogRecordScanner.java:396-425`: + +```java +if (firstBlock.getLogBlockHeader().containsKey(COMPACTED_BLOCK_TIMES)) { + // Map each original instant to the compacted block's instant + Arrays.stream(firstBlock.getLogBlockHeader().get(COMPACTED_BLOCK_TIMES).split(",")) + .forEach(originalInstant -> { + String finalInstant = blockTimeToCompactionBlockTimeMap + .getOrDefault(instantTime, instantTime); + blockTimeToCompactionBlockTimeMap.put(originalInstant, finalInstant); + }); +} else { + String compactedFinalInstantTime = blockTimeToCompactionBlockTimeMap.get(instantTime); + if (compactedFinalInstantTime == null) { + // Not compacted: add blocks directly to processing queue + logBlocks.forEach(block -> currentInstantLogBlocks.addLast(block)); + validBlockInstants.add(instantTime); // ← tracked for later reference + } +} +``` + +### At the merge level — instant time is NOT used for merging + +Within the record buffer, merging is driven by **ordering values** (e.g., `ts` field), **not** instant times. The instant time decides *which blocks are read*; the ordering value decides *which record wins*. + +Evidence from `KeyBasedFileGroupRecordBuffer.java:105-109`: + +```java +public void processNextDataRecord(BufferedRecord record, Serializable recordKey) throws IOException { + BufferedRecord existingRecord = records.get(recordKey); + // deltaMerge uses ordering values, not instant times + bufferedRecordMerger.deltaMerge(record, existingRecord) + .ifPresent(bufferedRecord -> records.put(recordKey, bufferedRecord.toBinary(...))); +} +``` + +And for the final base-vs-log merge (`FileGroupRecordBuffer.java:238-249`): + +```java +protected boolean hasNextBaseRecord(T baseRecord, BufferedRecord logRecordInfo) throws IOException { + if (logRecordInfo != null) { + BufferedRecord baseRecordInfo = BufferedRecords.fromEngineRecord(baseRecord, ...); + BufferedRecord mergeResult = bufferedRecordMerger.finalMerge(baseRecordInfo, logRecordInfo); + nextRecord = updateProcessor.processUpdate(...); + return nextRecord != null; + } + // No log record for this key — return base record as-is + nextRecord = bufferedRecordConverter.convert(readerContext.getRecordContext().seal(baseRecord)); + return true; +} +``` + +The three-phase output from `doHasNext()` (`KeyBasedFileGroupRecordBuffer.java:142-154`): + +```java +protected boolean doHasNext() throws IOException { + // Phase A+B: iterate base file, merge with log map matches (updates) or return as-is + while (baseFileIterator.hasNext()) { + if (hasNextBaseRecord(baseFileIterator.next())) { + return true; + } + } + // Phase C: remaining log-only records (inserts from log) + return hasNextLogRecord(); +} +``` + +### Concrete example from the test + +From `TestKeyBasedFileGroupRecordBuffer.java:78-109`: + +```java +// Base file: Log block (single block, records processed in order): +// (rk=1, counter=1, ts=1) (rk=1, counter=2, ts=1) ← ts equal → stored +// (rk=2, counter=1, ts=1) (rk=2, counter=1, ts=2) ← ts=2 > none → stored +// (rk=3, counter=1, ts=1) (rk=2, counter=1, ts=0) ← ts=0 < ts=2 → REJECTED +// (rk=3, counter=1, ts=2) ← ts=2 > none → stored +// (rk=3, counter=3, ts=1) ← DELETE ts=1 < ts=2 → REJECTED +``` + +The test shows how in EVENT_TIME_ORDERING mode, the ordering field (`ts`) determines merge outcomes — not the position within the log block or the instant time. + +--- + +## Summary + +| Concept | What It Is | +|---|---| +| **File Group** | All files sharing the same `(partitionPath, fileId)` — the unit of all reads/writes | +| **File Slice** | One base file + its associated log files, anchored at a `baseInstantTime` | +| **Base File** | Parquet file — compact snapshot at a point in time. Name encodes `fileId`, `writeToken`, `instantTime` | +| **Log File** | Append-only delta file (Avro blocks). Name encodes `fileId`, `deltaCommitTime`, `logVersion`, `writeToken` | +| **Log Block** | Atomic unit within a log file — carries `INSTANT_TIME`, `SCHEMA`, record data, positions | +| **Committed vs uncommitted** | Determined at scan time: TV<8 checks timeline; TV9 uses rollback blocks + latestInstantTime watermark | +| **Instant attribution** | Each log block's `INSTANT_TIME` header identifies its source commit; log-compacted blocks carry `COMPACTED_BLOCK_TIMES` listing merged instants | +| **Merge winner** | Decided by ordering field values (EVENT_TIME) or write order (COMMIT_TIME) — **not** by instant time | + + +👤 USER: +Format: .{fileId}_{deltaCommitTime}.log.{logVersion}_{writeToken} +is deltacommit time of a log file the instant time that write the file? is it instant time or completion time in v9 + + +🤖 ASSISTANT: +Good question. Let me trace exactly how the log file name is constructed. + +Now I have the full picture. The `instantTime` passed to the writer is the **requested time** (the time the commit was initiated), not the completion time. Let me verify this is also what goes into the log file name. + +The chain is clear: +1. `BaseActionExecutor.instantTime` = requested time (from `createNewInstantTime()`) +2. `HoodieWriteHandle.instantTime` = same requested time (passed through constructor) +3. `HoodieLogFormat.WriterBuilder.withInstantTime(instantTime)` = same requested time +4. `FSUtils.makeLogFileName(..., instantTime, ...)` = same requested time → becomes `deltaCommitTime` in the file name + +--- + +**Yes, `deltaCommitTime` in the log file name is the instant's *requested time* (a.k.a. instant time), not the completion time.** + +Here's the evidence chain: + +1. The write handle receives `instantTime` which is the **requested time** — the time generated when the commit is first initiated: + + `BaseActionExecutor.java:63`: + ```java + this.instantTime = instantTime; // this is the requested time from createNewInstantTime() + ``` + +2. It flows directly into the log format writer: + + `HoodieWriteHandle.java:291-294`: + ```java + return HoodieLogFormat.newWriterBuilder() + .onParentPath(...) + .withFileId(fileId) + .withInstantTime(instantTime) // ← still the requested time + ``` + +3. Which uses it as the file name's `deltaCommitTime`: + + `HoodieLogFormat.java:281`: + ```java + FSUtils.makeLogFileName(logFileId, fileExtension, instantTime, logVersion, logWriteToken) + // ^^^^^^^^^^^ + // this becomes deltaCommitTime in the file name + ``` + +4. The **completion time** is only assigned later, when the commit is finalized (transition from INFLIGHT to COMPLETED state). It's stored in the timeline metadata (`HoodieInstant.completionTime`), not in the file name. + +This means for a log file `.abc-123_20240101130000.log.1_1-0-1`: +- `20240101130000` = the **requested time** — when the write operation was initiated +- The **completion time** (when it actually committed) is only known from the timeline, not from the file name +- `CompletionTimeQueryView.getCompletionTime(baseInstant, deltaCommitTime)` resolves the requested time → completion time mapping at read time + + +👤 USER: +HoodieFileGroup + + From HoodieFileGroup.java:42-47: + /** + * A set of data/base files + set of log files, that make up a unit for all operations. + */ + public class HoodieFileGroup implements Serializable { + private final HoodieFileGroupId fileGroupId; // (partitionPath, fileId) + private final TreeMap fileSlices; // keyed by baseInstantTime, reverse-sorted + private final HoodieTimeline timeline; + private final Option lastInstant; // high watermark + } + + A file group is identified by (partitionPath, fileId). It's a container of file slices ordered by time. + + "file slices ordered by time." request time or completion time? + + +🤖 ASSISTANT: +**Requested time.** + +The TreeMap key comes from `HoodieBaseFile.getCommitTime()`, which is parsed from the base file name. + +From `HoodieFileGroup.java:108-113`: +```java +public void addBaseFile(HoodieBaseFile dataFile) { + if (!fileSlices.containsKey(dataFile.getCommitTime())) { + fileSlices.put(dataFile.getCommitTime(), new FileSlice(fileGroupId, dataFile.getCommitTime())); + } + fileSlices.get(dataFile.getCommitTime()).setBaseFile(dataFile); +} +``` + +And `dataFile.getCommitTime()` is extracted from the 3rd segment of `{fileId}_{writeToken}_{instantTime}.parquet` via `FSUtils.getCommitTime()` (`FSUtils.java:138`): + +```java +return fullFileName.split("_")[2].split("\\.", 2)[0]; +``` + +That `instantTime` in the base file name is the requested time, as we traced earlier. So the TreeMap is keyed by **requested time**, and file slices are ordered by requested time. + + +👤 USER: +FileSlice + + From FileSlice.java:37-43: + /** + * Within a file group, a slice is a combination of data file written at a commit time + * and list of log files, containing changes to the data file from that commit time. + */ + public class FileSlice implements Serializable { + private final HoodieFileGroupId fileGroupId; + private final String baseInstantTime; // the instant that created this slice + private HoodieBaseFile baseFile; // 0 or 1 parquet file + private final TreeSet logFiles; // 0..N log files, reverse-sorted by version + } + +0..N log files, reverse-sorted by version, what is this version, how it is generated, who would consume this version, what's the effect it can have + +🤖 ASSISTANT: +Now I have the complete picture. Here's the answer: + +--- + +The `logVersion` is the `.{N}` portion of the log file name (e.g., `.abc_t1.log.1`, `.abc_t1.log.2`). + +### How it's generated + +It starts at `LOGFILE_BASE_VERSION = 1` and increments on **rollover**. Rollover happens when a log file exceeds a size threshold, or when a file-already-exists conflict occurs. + +From `HoodieLogFormatWriter.java:206-218`: +```java +private void rolloverIfNeeded() throws IOException { + if (getCurrentSize() > sizeThreshold) { + log.info("CurrentSize {} has reached threshold {}. Rolling over to the next version", + getCurrentSize(), sizeThreshold); + rollOver(); + } +} + +private void rollOver() throws IOException { + closeStream(); + this.logFile = logFile.rollOver(rolloverLogWriteToken); // version + 1 + this.closed = false; +} +``` + +And `rollOver()` on `HoodieLogFile` (`HoodieLogFile.java:181-188`): +```java +public HoodieLogFile rollOver(String logWriteToken) { + return new HoodieLogFile(new StoragePath(path.getParent(), + FSUtils.makeLogFileName(fileId, extension, deltaCommitTime, logVersion + 1, logWriteToken))); +} +``` + +Rollover also happens on **write-token collision** — when two tasks try to write the same log file simultaneously (`HoodieLogFormatWriter.java:97-116`): +```java +if (storage.exists(logFile.getPath())) { + rollOver(); // file already exists → bump version +} +// ... +} catch (FileAlreadyExistsException ignored) { + rollOver(); // race condition → bump version +} catch (RemoteException re) { + if (re.getClassName().contentEquals(AlreadyBeingCreatedException.class.getName())) { + rollOver(); // another task writing → bump version + } +} +``` + +**For TV >= 8**, the initial version always starts at base version 1 (no storage listing needed): + +```java +// HoodieLogFormat.java:247-251 +boolean useBaseVersion = tableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT) + && logWriteToken != null; +if (useBaseVersion) { + logVersion = HoodieLogFile.LOGFILE_BASE_VERSION; // always 1 +} +``` + +This works because TV8+ uses distinct `writeToken` per task, so version conflicts are resolved by rollover at write time rather than pre-computing the next version from storage (which was the expensive pre-TV8 approach). + +### Who consumes it — the sort order + +The `LogFileComparator` uses version as a **secondary sort key** (after `deltaCommitTime`). From `HoodieLogFile.java:214-240`: + +```java +public int compare(HoodieLogFile o1, HoodieLogFile o2) { + String deltaCommitTime1 = o1.getDeltaCommitTime(); + String deltaCommitTime2 = o2.getDeltaCommitTime(); + + if (deltaCommitTime1.equals(deltaCommitTime2)) { + if (o1.getLogVersion() == o2.getLogVersion()) { + int compareWriteToken = getWriteTokenComparator() + .compare(o1.getLogWriteToken(), o2.getLogWriteToken()); + if (compareWriteToken == 0) { + return o1.getSuffix().compareTo(o2.getSuffix()); + } + return compareWriteToken; // 3rd tiebreaker: writeToken + } + return Integer.compare(o1.getLogVersion(), o2.getLogVersion()); // 2nd: logVersion + } + return deltaCommitTime1.compareTo(deltaCommitTime2); // 1st: deltaCommitTime +} +``` + +The `FileSlice` stores log files in a `TreeSet` with the **reverse** comparator (`HoodieLogFile.getReverseLogFileComparator()`), so within a file slice, log files are ordered **highest version first**. + +### Effect + +The version establishes **write order within the same instant**. When a single write task produces more data than the size threshold, it rolls over to version 2, 3, etc. The log scanner then processes blocks in version order (ascending via the forward comparator), ensuring blocks are replayed in the order they were written. This matters because block processing order affects the merge result in `COMMIT_TIME_ORDERING` mode (last-writer-wins). + + +👤 USER: +how does filterUncommittedFiles work? + +🤖 ASSISTANT: +Here's how it works end-to-end: + +### `filterUncommittedFiles` (line 602) + +```java +private Stream filterUncommittedFiles(FileSlice fileSlice, boolean includeEmptyFileSlice) { + Option committedBaseFile = fileSlice.getBaseFile().isPresent() + && completionTimeQueryView.isCompleted(fileSlice.getBaseInstantTime()) + ? fileSlice.getBaseFile() : Option.empty(); + + List committedLogFiles = fileSlice.getLogFiles() + .filter(logFile -> completionTimeQueryView.isCompleted(logFile.getDeltaCommitTime())) + .collect(Collectors.toList()); + + // ... rebuild FileSlice with only committed files +} +``` + +It delegates to `completionTimeQueryView.isCompleted(requestedTime)`. For TV9, this is `CompletionTimeQueryViewV2`. + +### `CompletionTimeQueryViewV2.isCompleted()` (line 100-103) + +```java +public boolean isCompleted(String instantTime) { + return this.instantTimeToCompletionTimeMap.containsKey(instantTime) + || isArchived(instantTime); +} +``` + +Two checks: +1. Is the requested time in the `instantTimeToCompletionTimeMap`? +2. Or is it archived (older than the first active instant)? + +### How the map gets populated — `load()` (line 296-301) + +```java +private void load() { + this.metaClient.getActiveTimeline() + .filterCompletedInstants().getInstantsAsStream() + .forEach(instant -> setCompletionTime(instant.requestedTime(), instant.getCompletionTime())); +} +``` + +**This is where the "completed" semantics live.** It iterates the active timeline filtered to **completed instants only** (`filterCompletedInstants()`), and builds a map of `requestedTime -> completionTime`. Only instants that have transitioned to the COMPLETED state get into this map. + +### `setCompletionTime()` (line 308-314) + +```java +private void setCompletionTime(String beginInstantTime, String completionTime) { + if (completionTime == null) { + completionTime = beginInstantTime; // meta-server fallback + } + this.instantTimeToCompletionTimeMap.putIfAbsent(beginInstantTime, completionTime); +} +``` + +### `getCompletionTime()` with lazy loading (line 166-177) + +```java +public Option getCompletionTime(String instantTime) { + String completionTime = this.instantTimeToCompletionTimeMap.get(instantTime); + if (completionTime != null) { + return Option.of(completionTime); + } + if (compareTimestamps(instantTime, GREATER_THAN_OR_EQUALS, this.cursorInstant)) { + return Option.empty(); // instant is still pending + } + // Older than cursor → might be in archived timeline, lazy-load it + loadCompletionTimeIncrementally(instantTime); + return Option.ofNullable(this.instantTimeToCompletionTimeMap.get(instantTime)); +} +``` + +Three-tier lookup: +1. **In-memory map** — populated from active timeline's completed instants +2. **Cursor check** — if requested time >= cursor (first active instant), it's pending → return empty +3. **Archived timeline** — lazy-load from archive files if older than cursor + +### Summary + +``` +filterUncommittedFiles(fileSlice) + │ + ├── completionTimeQueryView.isCompleted("20240101130000") // log file's deltaCommitTime (= requested time) + │ │ + │ ├── instantTimeToCompletionTimeMap.containsKey("20240101130000") + │ │ │ + │ │ └── populated by: activeTimeline.filterCompletedInstants() ← HERE is the COMPLETED state check + │ │ .forEach(instant -> map.put(instant.requestedTime(), instant.completionTime())) + │ │ + │ └── OR isArchived("20240101130000") + │ └── instantTime < firstNonSavepointCommit (archived = definitely completed) + │ + └── if not completed → log file excluded from FileSlice +``` + +The map is keyed by **requested time** but only populated from instants that have `state == COMPLETED`. So `map.containsKey(requestedTime)` effectively answers "has this instant completed?" without needing to compare completion times — the mere presence in the map is proof of completion. + +👤 USER: +now with all these corrections, please answer the question of log block v.s. log file again with all the details we covered so far + +🤖 ASSISTANT: +## Log File vs Log Block + +### Relationship: 1 log file : N blocks, all from the same instant + +A log file is a physical file on storage. Log blocks are the logical units inside it, written sequentially with a binary framing protocol. + +**All blocks within a single log file share the same `INSTANT_TIME`**, because: + +1. Each instant creates its own writer with its own `instantTime` as the file name's `deltaCommitTime`: + + ```java + // HoodieWriteHandle.java:294 + .withInstantTime(instantTime) // → becomes deltaCommitTime in file name + ``` + +2. The writer never appends to an existing file — it always creates new: + + ```java + // HoodieLogFormatWriter.java:98-99 + if (storage.exists(logFile.getPath())) { + rollOver(); // bump version, never append + } + // line 222-223 + this.output = storage.create(this.logFile.getPath(), false, ...); // false = create-only + ``` + +3. The block header `INSTANT_TIME` is set once per writer session: + + ```java + // HoodieAppendHandle.java:484 + header.put(HeaderMetadataType.INSTANT_TIME, instantTime); // same for all blocks + ``` + +### What block types can coexist in one log file + +Since a log file belongs to one instant, and an instant performs one type of action: + +| Combination | Can coexist? | Why | +|---|---|---| +| DATA_BLOCK + DELETE_BLOCK | Yes | Same instant writes both upserts and deletes in one `appendBlocks()` call | +| DATA_BLOCK + COMMAND_BLOCK | No | COMMAND_BLOCK is written by a rollback instant, which is a separate instant with its own log file | +| DATA_BLOCK + DATA_BLOCK | Yes | Same instant, multiple blocks due to size threshold rollover within a file, or multiple flushes | + +Evidence for DATA + DELETE coexistence — `HoodieAppendHandle.java:482-520`: + +```java +protected void appendDataAndDeleteBlocks(Map header, boolean appendDeleteBlocks) { + header.put(HeaderMetadataType.INSTANT_TIME, instantTime); + List blocks = new ArrayList<>(2); + + if (recordList.size() > 0) { + blocks.add(getDataBlock(header, ...)); // DATA_BLOCK + } + if (appendDeleteBlocks && deleteRecordList.size() > 0) { + blocks.add(new HoodieDeleteBlock(deleteRecordList, header)); // DELETE_BLOCK + } + + writer.appendBlocks(blocks); // both written to same log file +} +``` + +### One instant can produce multiple log files + +When a single write exceeds the size threshold, the writer rolls over to a new version: + +```java +// HoodieLogFormatWriter.java:206-211 +private void rolloverIfNeeded() throws IOException { + if (getCurrentSize() > sizeThreshold) { + rollOver(); // closes current file, bumps logVersion + } +} + +// HoodieLogFile.java:181-188 +public HoodieLogFile rollOver(String logWriteToken) { + return new HoodieLogFile(new StoragePath(path.getParent(), + FSUtils.makeLogFileName(fileId, extension, deltaCommitTime, logVersion + 1, logWriteToken))); +} +``` + +This produces multiple files with the same `deltaCommitTime` but incrementing `logVersion`: + +``` +.abc_t1.log.1_1-0-1 ← blocks until size threshold +.abc_t1.log.2_1-0-1 ← overflow blocks, same instant t1 +``` + +Also for TV8+, concurrent tasks writing to the same file group produce different files with the same `deltaCommitTime` but different `writeToken`: + +``` +.abc_t1.log.1_1-0-1 ← task partition 1 +.abc_t1.log.1_2-0-1 ← task partition 2 +``` + +### On-disk binary layout of a log file + +``` +Log File: .{fileId}_{deltaCommitTime}.log.{version}_{writeToken} +┌────────────────────────────────────────-─────────┐ +│ Block 1: AVRO_DATA_BLOCK (INSTANT_TIME = t1) │ +│ ┌─────────────────────────────────────────────┐ │ +│ │ MAGIC (6 bytes): '#HUDI#' │ │ +│ │ BLOCK SIZE (8 bytes, long) │ │ +│ │ LOG FORMAT VERSION (4 bytes, int) = 3 │ │ +│ │ BLOCK TYPE ORDINAL (4 bytes, int) = 3 │ │ +│ │ HEADER METADATA: │ │ +│ │ count(4) + for each: │ │ +│ │ ordinal(4) + size(4) + value(UTF-8) │ │ +│ │ includes: INSTANT_TIME, SCHEMA, │ │ +│ │ RECORD_POSITIONS, IS_PARTIAL, etc. │ │ +│ │ CONTENT SIZE (8 bytes, long) │ │ +│ │ CONTENT: Avro binary encoded records │ │ +│ │ FOOTER METADATA (currently empty) │ │ +│ │ TOTAL BLOCK LENGTH (8 bytes, reverse ptr) │ │ +│ └─────────────────────────────────────────────┘ │ +├─────────────────────────────────-────────────────┤ +│ Block 2: DELETE_BLOCK (INSTANT_TIME = t1) │ +│ ┌─────────────────────────────────────────────┐ │ +│ │ MAGIC: '#HUDI#' │ │ +│ │ ... same framing ... │ │ +│ │ BLOCK TYPE ORDINAL = 1 │ │ +│ │ CONTENT: HoodieDeleteRecordList (Avro v3) │ │ +│ │ each: recordKey + partitionPath + │ │ +│ │ orderingValue │ │ +│ └─────────────────────────────────────────────┘ │ +└───────────────────────────────────────────────-──┘ +``` + +### Consequence for filtering + +Since all blocks in a log file share the same instant, `filterUncommittedFiles` filtering at the **log file level** is equivalent to filtering at the block level: + +```java +// AbstractTableFileSystemView.java:604 +List committedLogFiles = fileSlice.getLogFiles() + .filter(logFile -> completionTimeQueryView.isCompleted(logFile.getDeltaCommitTime())) + .collect(Collectors.toList()); +``` + +If `deltaCommitTime` is not completed → the entire file (all blocks inside) is uncommitted. No block-level ambiguity exists. + +### Summary + +``` +Instant t1 ──writes──> .abc_t1.log.1_1-0-1 [DATA_BLOCK(t1), DELETE_BLOCK(t1)] + └──> .abc_t1.log.2_1-0-1 [DATA_BLOCK(t1)] (size rollover) + +Instant t2 ──writes──> .abc_t2.log.1_2-0-1 [DATA_BLOCK(t2)] + +Rollback ──writes──> .abc_t3.log.1_1-0-1 [COMMAND_BLOCK(t3, TARGET=t1)] +``` + + +👤 USER: +when will we have "compacted log block"? who generates that, what does it compact, what is the relation between what is been compacted and the output file? are they in the same file slice? will this cause a record being duplicated in 2 places and how would readers handle it properly? +Please give a comprehensive intro on this part. + + +🤖 ASSISTANT: + 🤖 Agent: Explore log compaction end to end — I need a comprehensive understanding of "log compaction" in Hudi (not regular compaction which merges log files into a new base file — this is specifically LOG compaction which merges log blocks into + +--- + +## What is Log Compaction? + +**Regular compaction** merges log files + base file → new base file. **Log compaction** merges log blocks → new compacted log block. It stays within the log layer — no new base file is produced. + +## Who Generates It? + +`FileGroupReaderBasedAppendHandle.doAppend()` (line 76-104): + +```java +public void doAppend() { + // 1. Collect log files from the compaction operation + Stream logFiles = operation.getDeltaFileNames().stream().map(logFileName -> + new HoodieLogFile(new StoragePath(FSUtils.constructAbsolutePath( + config.getBasePath(), operation.getPartitionPath()), logFileName))); + + // 2. Read all log records via HoodieFileGroupReader (no base file) + try (HoodieFileGroupReader fileGroupReader = HoodieFileGroupReader.newBuilder() + .withLogFiles(logFiles) + .withBaseFileOption(Option.empty()) // ← no base file, logs only + .withEmitDelete(true) // ← preserve deletes + .build()) { + + // 3. Get merged records from logs + recordItr = new CloseableMappingIterator<>( + fileGroupReader.getLogRecordsOnly(), ...); + + // 4. Stamp the output block with which instants were compacted + header.put(HeaderMetadataType.COMPACTED_BLOCK_TIMES, + StringUtils.join(fileGroupReader.getValidBlockInstants(), ",")); + + // 5. Write to new log file via parent class (HoodieAppendHandle) + super.doAppend(); + } +} +``` + +The key line is **line 97-98**: the compacted block's header carries `COMPACTED_BLOCK_TIMES = "t1,t2,t3"` — the comma-separated list of original instant times whose blocks were merged. + +## What Does It Compact? + +Input: multiple log files from the **same file slice** (same fileId, same partition). +Output: a **new** log file in the **same file slice**, with `deltaCommitTime` = the log compaction instant time. + +``` +BEFORE log compaction: +FileSlice (baseInstantTime = t0) + ├── base: abc_1-0-1_t0.parquet + ├── log: .abc_t1.log.1_1-0-1 ← blocks from instant t1 + ├── log: .abc_t2.log.1_2-0-1 ← blocks from instant t2 + └── log: .abc_t3.log.1_3-0-1 ← blocks from instant t3 + +AFTER log compaction (instant t4): +FileSlice (baseInstantTime = t0) + ├── base: abc_1-0-1_t0.parquet + ├── log: .abc_t1.log.1_1-0-1 ← STILL EXISTS (not deleted) + ├── log: .abc_t2.log.1_2-0-1 ← STILL EXISTS + ├── log: .abc_t3.log.1_3-0-1 ← STILL EXISTS + └── log: .abc_t4.log.1_4-0-1 ← NEW compacted log file + COMPACTED_BLOCK_TIMES = "t1,t2,t3" +``` + +**Yes, the original log files remain on disk.** The same records exist in both the old log files (t1, t2, t3) and the new compacted log file (t4). This is where the duplication problem arises. + +## How the Reader Avoids Double-Counting + +The scanner in `AbstractHoodieLogRecordScanner.scanInternal()` uses a two-pass algorithm. + +**Pass 1 (forward, lines 302-368):** Read all blocks from all log files, group by `INSTANT_TIME`: + +``` +orderedInstantsList = [t1, t2, t3, t4] +instantToBlocksMap = { + t1 → [dataBlock from .abc_t1.log], + t2 → [dataBlock from .abc_t2.log], + t3 → [dataBlock from .abc_t3.log], + t4 → [compactedBlock from .abc_t4.log] ← has COMPACTED_BLOCK_TIMES="t1,t2,t3" +} +``` + +**Pass 2 (reverse, lines 387-426):** Iterate newest-to-oldest, building a dedup map: + +```java +// Iterate: t4, t3, t2, t1 + +// t4: has COMPACTED_BLOCK_TIMES header → build the map +// blockTimeToCompactionBlockTimeMap = {t1→t4, t2→t4, t3→t4} +// (t4 itself is NOT added to the processing queue yet) + +// t3: lookup in map → compactedFinalInstantTime = t4 +// t4 not yet in instantTimesIncluded → ADD t4's blocks to queue +// instantTimesIncluded = {t4} + +// t2: lookup in map → compactedFinalInstantTime = t4 +// t4 already in instantTimesIncluded → SKIP + +// t1: lookup in map → compactedFinalInstantTime = t4 +// t4 already in instantTimesIncluded → SKIP +``` + +**Result:** Only t4's compacted block is processed. t1, t2, t3 blocks are skipped entirely. + +The code with annotations (lines 395-425): + +```java +if (firstBlock.getLogBlockHeader().containsKey(COMPACTED_BLOCK_TIMES)) { + // COMPACTED BLOCK: don't process it yet, just record the mapping + Arrays.stream(firstBlock.getLogBlockHeader().get(COMPACTED_BLOCK_TIMES).split(",")) + .forEach(originalInstant -> { + String finalInstant = blockTimeToCompactionBlockTimeMap + .getOrDefault(instantTime, instantTime); + blockTimeToCompactionBlockTimeMap.put(originalInstant, finalInstant); + // t1 → t4, t2 → t4, t3 → t4 + }); +} else { + String compactedFinalInstantTime = blockTimeToCompactionBlockTimeMap.get(instantTime); + if (compactedFinalInstantTime == null) { + // NOT compacted — process normally + logBlocks.forEach(block -> currentInstantLogBlocks.addLast(block)); + validBlockInstants.add(instantTime); + } else if (!instantTimesIncluded.contains(compactedFinalInstantTime)) { + // First time seeing this compacted group — add the COMPACTED block instead + logBlocks = instantToBlocksMap.get(compactedFinalInstantTime); // ← t4's blocks + logBlocks.forEach(block -> currentInstantLogBlocks.addLast(block)); + instantTimesIncluded.add(compactedFinalInstantTime); + } + // else: compacted block already included — skip this original block +} +``` + +The reverse iteration order is important: the compacted block (t4, newest) is encountered first, which sets up the map. Then when older originals (t3, t2, t1) are encountered, the map redirects them to t4's already-included compacted block. + +### Chained log compaction + +The `getOrDefault(instantTime, instantTime)` on line 400 handles chaining — if t4 itself was later compacted into t7: + +```java +String finalInstant = blockTimeToCompactionBlockTimeMap.getOrDefault(instantTime, instantTime); +// If t4→t7 already in map, then t1→t7 (not t1→t4) +``` + +This ensures that even with multiple rounds of log compaction, each original block maps to the most recent compacted block. + diff --git a/filegroup/recordContext/dd.md b/filegroup/recordContext/dd.md new file mode 100644 index 00000000..fedfb269 --- /dev/null +++ b/filegroup/recordContext/dd.md @@ -0,0 +1,824 @@ +# hudi internal RecordContext + +The recordContext is the engine-specific "glue" that lets the engine-agnostic FileGroupRecordBuffer and BufferedRecordMerger infrastructure manipulate Spark InternalRow objects without knowing anything about Spark's type system. + + +## Call Stack Tree: Record Types Through HoodieFileGroupReader + +``` +HoodieFileGroupReader.getClosableIterator() → ClosableIterator +HoodieFileGroupReader.getClosableHoodieRecordIterator() → ClosableIterator> +HoodieFileGroupReader.getClosableKeyIterator() → ClosableIterator +HoodieFileGroupReader.getClosableBufferedRecordIterator() → ClosableIterator> + │ + └─ getBufferedRecordIterator(mode) + │ + └─ initRecordIterators() + │ + ├─── [BRANCH A: no merge needed — base file only] + │ makeBaseFileIterator() + │ │ + │ └─ readerContext.getFileRecordIterator(...) + │ └─ reads parquet file + │ TYPE: ClosableIterator ← raw engine record (e.g. InternalRow) + │ │ + │ └─ [if no log files] CloseableMappingIterator + │ └─ RecordContext.seal(T) + │ TYPE: T (copied/frozen) + │ + └─── [BRANCH B: MOR — base + log files] + │ + ├─── [1. BASE FILE reading] + │ makeBaseFileIterator() + │ └─ readerContext.getFileRecordIterator(...) + │ TYPE: ClosableIterator ← raw engine record + │ + └─── [2. LOG FILE scanning] + LogScanningRecordBufferLoader.scanLogFiles() + └─ HoodieMergedLogRecordReader.build() + └─ [for each log block] + │ + ├─── [DATA BLOCK] + │ HoodieDataBlock.getEngineRecordIterator(readerContext) + │ └─ deserializeRecords(readerContext, content) + │ TYPE: ClosableIterator ← raw engine record + │ │ + │ └─ [schema evolution transformer applied] + │ RecordContext.projectRecord(from, to, renames) + │ TYPE: T (projected) + │ │ + │ KeyBasedFileGroupRecordBuffer.processDataBlock() + │ └─ RecordContext.isDeleteRecord(T, deleteContext) + │ BufferedRecords.fromEngineRecord(T, schema, ...) + │ └─ RecordContext.getRecordKey(T) + │ RecordContext.encodeAvroSchema(schema) + │ RecordContext.getOrderingValue(T, schema, fields) + │ TYPE: BufferedRecord ← Hudi internal record + │ │ + │ └─ bufferedRecordMerger.deltaMerge(new, existing) + │ TYPE: BufferedRecord (merged) + │ │ + │ └─ BufferedRecord.toBinary(recordContext) + │ RecordContext.toBinaryRow(schema, T) + │ RecordContext.seal(T) + │ stored in: + │ ExternalSpillableMap> + │ + └─── [DELETE BLOCK] + HoodieDeleteBlock.getRecordsToDelete() + TYPE: DeleteRecord[] ← Hudi delete record + │ + └─ BufferedRecords.fromDeleteRecord(DeleteRecord, ...) + RecordContext.getOrderingValue(DeleteRecord) + TYPE: BufferedRecord ← isDelete=true, record=null + │ + └─ stored in: + ExternalSpillableMap> +``` + +--- + +``` +[3. MERGE PHASE — doHasNext() drives the merge] + │ + KeyBasedFileGroupRecordBuffer.doHasNext() + │ + ├─── [base record has matching log record] + │ baseFileIterator.next() + │ TYPE: T ← raw engine record from base file + │ │ + │ └─ BufferedRecords.fromEngineRecord(T, schema, ...) + │ TYPE: BufferedRecord ← wraps base record + │ │ + │ └─ bufferedRecordMerger.finalMerge(base: BufferedRecord, log: BufferedRecord) + │ │ + │ ├─ [event/commit time ordering] + │ │ TYPE: BufferedRecord ← higher ordering value wins + │ │ + │ └─ [custom merger / payload merger] + │ RecordContext.decodeAvroSchema(schemaId) + │ RecordContext.convertToAvroRecord(T, schema) + │ → merge via HoodieRecordMerger (in Avro space) + │ RecordContext.convertAvroRecord(IndexedRecord) + │ TYPE: BufferedRecord ← merged result + │ │ + │ └─ UpdateProcessor.processUpdate(key, base, merged, isDelete) + │ │ + │ ├─ [delete] RecordContext.getDeleteRow(key) + │ │ TYPE: BufferedRecord (isDelete=true, T=delete row) + │ │ + │ └─ [non-delete] BufferedRecord.seal(recordContext) + │ RecordContext.seal(T) + │ TYPE: BufferedRecord ← final merged, sealed + │ + └─── [log-only record — no base file match] + records iterator → BufferedRecord + └─ UpdateProcessor.processUpdate(key, null, logRecord, isDelete) + TYPE: BufferedRecord ← insert or delete +``` + +--- + +``` +[4. OUTPUT — HoodieFileGroupReader.next()] + │ + BufferedRecord + │ + └─ outputConverter.apply(T) [if schema projection needed] + RecordContext.projectRecord(from, to) + TYPE: BufferedRecord (projected T inside) + │ + ├─ getClosableIterator() + │ BufferedRecord::getRecord + │ TYPE: T ← raw engine record (InternalRow) + │ + ├─ getClosableHoodieRecordIterator() + │ RecordContext.constructFinalHoodieRecord(BufferedRecord) + │ └─ RecordContext.constructHoodieRecord(BufferedRecord, partitionPath) + │ TYPE: HoodieRecord ← HoodieSparkRecord or HoodieEmptyRecord + │ + ├─ getClosableKeyIterator() + │ BufferedRecord::getRecordKey + │ TYPE: String ← record key only + │ + └─ getClosableBufferedRecordIterator() + TYPE: BufferedRecord ← raw internal record with metadata +``` + + +# Object Construction + + + HoodieFileGroupReaderBasedFileFormat.buildReaderWithPartitionValues() :227 + │ [Spark scan layer — one readerContext per task] + │ + └─ new SparkFileFormatInternalRowReaderContext( + fileGroupBaseFileReader, + filters, + requiredFilters, + storageConf, + metaClient.getTableConfig() ← HoodieTableConfig from .hoodie/hoodie.properties + ) + │ + └─ SparkFileFormatInternalRowReaderContext.() :63 + └─ extends BaseSparkInternalRowReaderContext( + storageConf, + tableConfig, + new SparkFileFormatInternalRecordContext(tableConfig) ← RecordContext built here + ) + │ + ├─ [RecordContext construction path] + │ SparkFileFormatInternalRecordContext.(tableConfig) + │ └─ extends BaseSparkInternalRecordContext(tableConfig) + │ └─ BaseSparkInternalRecordContext.(tableConfig) :54 + │ └─ super(tableConfig, new DefaultJavaTypeConverter()) + │ │ + │ └─ RecordContext.(tableConfig, typeConverter) :70 + │ │ + │ └─ tableConfig.populateMetaFields() :72 + │ │ + │ ├─ true → recordKeyExtractor = metadataKeyExtractor() + │ │ reads _hoodie_record_key meta field directly + │ │ getValue(record, schema, RECORD_KEY_METADATA_FIELD) + │ │ + │ └─ false → recordKeyExtractor = virtualKeyExtractor( + │ tableConfig.getRecordKeyFields()) + │ computes key from user-defined key fields + + +--- +RecordContext Construction Chain + +Starting from your buildReaderWithPartitionValues code: + +Step 1: SparkFileFormatInternalRowReaderContext instantiation +``` +val readerContext = new SparkFileFormatInternalRowReaderContext( + fileGroupBaseFileReader.value, filters, requiredFilters, storageConf, metaClient.getTableConfig) +``` +The constructor at SparkFileFormatInternalRowReaderContext.scala:63-68 passes the recordContext up to the superclass: +``` +class SparkFileFormatInternalRowReaderContext(...) +extends BaseSparkInternalRowReaderContext( + storageConfiguration, tableConfig, + SparkFileFormatInternalRecordContext.apply(tableConfig)) // <-- recordContext created here +``` +Step 2: SparkFileFormatInternalRecordContext.apply(tableConfig) + +At SparkFileFormatInternalRecordContext.scala:74: +``` +def apply(tableConfig: HoodieTableConfig): SparkFileFormatInternalRecordContext = + new BaseSparkInternalRecordContext(tableConfig) with SparkFileFormatInternalRecordContext +``` +This creates a BaseSparkInternalRecordContext mixed with the SparkFileFormatInternalRecordContext trait. The trait adds: +- supportsParquetRowIndex → true if Spark >= 3.5 +- convertAvroRecord / convertToAvroRecord → Avro ↔ InternalRow via Spark deserializers + +Step 3: BaseSparkInternalRecordContext(tableConfig) + +At BaseSparkInternalRecordContext.java:58-59: +``` +protected BaseSparkInternalRecordContext(HoodieTableConfig tableConfig) { + super(tableConfig, new DefaultJavaTypeConverter()); +} +``` +Step 4: RecordContext(tableConfig, typeConverter) + +At RecordContext.java:71-75: +``` +protected RecordContext(HoodieTableConfig tableConfig, JavaTypeConverter typeConverter) { + this.typeConverter = typeConverter; + this.recordKeyExtractor = tableConfig.populateMetaFields() + ? metadataKeyExtractor() // reads _hoodie_record_key meta field + : virtualKeyExtractor(tableConfig.getRecordKeyFields()); // reads actual PK columns +} +``` +This is where the key-extraction strategy is wired: either reading _hoodie_record_key from position 2 in the row (meta fields mode) or constructing the key from configured record key columns (virtual keys mode). + +Step 5: recordContext is stored in HoodieReaderContext + +BaseSparkInternalRowReaderContext calls super(storageConfig, tableConfig, Option.empty(), Option.empty(), recordContext) which flows to HoodieReaderContext:101-107: +``` +protected HoodieReaderContext(..., RecordContext recordContext) { + ... + this.recordContext = recordContext; +} +``` +Step 6: Late initialization — ordering value converter + +When the HoodieFileGroupReader constructor calls readerContext.setSchemaHandler(...) at line 119-121, BaseSparkInternalRowReaderContext.setSchemaHandler() (line 96-103) +fires: + +``` +@Override +public void setSchemaHandler(FileGroupReaderSchemaHandler schemaHandler) { + super.setSchemaHandler(schemaHandler); + List orderingFieldNames = HoodieRecordUtils.getOrderingFieldNames(getMergeMode(), tableConfig); + Schema schema = schemaHandler.getRequiredSchema(); + ((BaseSparkInternalRecordContext) recordContext).initOrderingValueConverter(schema, orderingFieldNames); +} +``` + +This initializes OrderingValueEngineTypeConverter inside recordContext. For commit-time ordering, orderingFieldNames is empty, so this converter is effectively a no-op (it will never be called because ordering values are always the default 0). + +Step 7: Partition path set + +Back in HoodieFileGroupReader constructor at line 106: + +`readerContext.getRecordContext().setPartitionPath(inputSplit.getPartitionPath());` + +This lets recordContext stamp partition path onto HoodieRecord objects it constructs later + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +in hoodie file group reader we have reader context which contains recordContext. + could you walk through consumers within the fg reader and see what are the set of APIs that actually got consumed + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here's a walkthrough of the `RecordContext` API surface and what's actually consumed: + +## RecordContext - Full API (31 methods) + +Defined in `hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java` + +## Consumed APIs by Layer + +### Directly in `HoodieFileGroupReader` (6 methods) + +| Method | Line | Purpose | +|--------|------|---------| +| `setPartitionPath()` | :104 | Set partition path from input split | +| `supportsParquetRowIndex()` | :118 | Pick schema handler type | +| `seal()` | :132 | Seal records from base file iterator | +| `convertValueToEngineType()` | :200 | Convert partition field values | +| `constructFinalHoodieRecord()` | :309 | Build final `HoodieRecord` for output iterator | + +### Via `BufferedRecords` / `BufferedRecordConverter` + +- `extractDataFromRecord()` +- `getRecordKey()` +- `encodeAvroSchema()` +- `getOrderingValue()` (multiple overloads) +- `convertOrderingValueToEngineType()` + +### Via `UpdateProcessor` + +- `getDeleteRow()` +- `decodeAvroSchema()` +- `convertToAvroRecord()` +- `convertAvroRecord()` +- `seal()` (again, via `BufferedRecord.seal()`) + +### Via `PartialUpdateHandler` + +- `getValue()` +- `constructEngineRecord()` +- `getTypeConverter()` + +### Via `KeyBasedFileGroupRecordBuffer` + +- `isDeleteRecord()` + +### Via `HoodieReaderContext` (instant range filtering) + +- `getMetaFieldValue()` + +--- + +## Summary + +Of the 31 defined methods, roughly **17-18 distinct methods** are actually exercised across the FG reader pipeline. The remaining ones (e.g., `projectRecord`, `toBinaryRow`, `getSchemaFromBufferRecord`, `extractRecordPosition`) are called only by engine-specific subclasses or are utility helpers used outside the core FG reader flow. + +The FG reader itself is thin — it delegates almost all heavy lifting to `BufferedRecords`, `UpdateProcessor`, `PartialUpdateHandler`, and `BufferedRecordConverter`, which is where most of the `RecordContext` surface area is consumed. + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +for each of the API, show me call stack tree starting from hoodie file group reader API + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here are the call stack trees rooted at `HoodieFileGroupReader` for each consumed `RecordContext` API: + +--- + +## 1. `setPartitionPath()` +``` +HoodieFileGroupReader.() :104 + -> readerContext.getRecordContext() + -> RecordContext.setPartitionPath(partitionPath) +``` + +## 2. `supportsParquetRowIndex()` +``` +HoodieFileGroupReader.() :118 + -> readerContext.getRecordContext() + -> RecordContext.supportsParquetRowIndex() +``` + +## 3. `seal()` + +**Path A — base file only (no merge)** +``` +HoodieFileGroupReader.initRecordIterators() :132 + -> CloseableMappingIterator.apply() + -> readerContext.getRecordContext() + -> RecordContext.seal(rec) +``` + +**Path B — buffer hasNext** +``` +FileGroupRecordBuffer.hasNextBaseRecord() :244 + -> readerContext.getRecordContext() + -> RecordContext.seal(baseRecord) +``` + +**Path C — unmerged buffer** +``` +UnmergedFileGroupRecordBuffer.doHasNext() :70,88 + -> readerContext.getRecordContext() + -> RecordContext.seal(next) +``` + +**Path D — via BufferedRecord** +``` +BufferedRecord.seal() :100 + -> RecordContext.seal(record) +``` + +**Path E — UpdateProcessor non-delete** +``` +UpdateProcessor.StandardUpdateProcessor.handleNonDeletes() :118 + -> BufferedRecord.seal(readerContext.getRecordContext()) + -> RecordContext.seal(record) +``` + +## 4. `convertValueToEngineType()` + +**Path A — bootstrap partition values** +``` +HoodieFileGroupReader.makeBootstrapBaseFileIterator() :200 + -> readerContext.getRecordContext() + -> RecordContext.convertValueToEngineType(partitionValues[i]) +``` + +**Path B — via convertOrderingValueToEngineType (see #10)** + +## 5. `constructFinalHoodieRecord()` +``` +HoodieFileGroupReader.getClosableHoodieRecordIterator() :309 + -> CloseableMappingIterator.apply() + -> readerContext.getRecordContext() + -> RecordContext.constructFinalHoodieRecord(bufferedRecord) +``` + +## 6. `extractDataFromRecord()` + +**Path A** +``` +BufferedRecords.fromHoodieRecord() :44 + -> RecordContext.extractDataFromRecord(record, schema, props) +``` + +**Path B** +``` +StreamingFileGroupRecordBufferLoader.loadRecords() :89 + -> RecordContext.extractDataFromRecord(hoodieRecord, schema, props) +``` + +## 7. `getRecordKey()` + +**Path A — from engine record via converter** +``` +HoodieFileGroupReader.initRecordIterators() + -> BufferedRecordConverter.createConverter() :56 + -> RecordContext.getRecordKey(record, readerSchema) +``` + +**Path B — from HoodieRecord** +``` +BufferedRecords.fromHoodieRecord() :45 + -> RecordContext.getRecordKey(data, schema) +``` + +**Path C — from base record in key-based buffer** +``` +KeyBasedFileGroupRecordBuffer.hasNextBaseRecord() :138 + -> readerContext.getRecordContext() + -> RecordContext.getRecordKey(baseRecord, readerSchema) +``` + +## 8. `encodeAvroSchema()` + +``` +BufferedRecords.fromHoodieRecord() :46 + -> RecordContext.encodeAvroSchema(schema) + +BufferedRecords.fromEngineRecord() [overload 2] :58 + -> RecordContext.encodeAvroSchema(schema) + +BufferedRecords.fromEngineRecord() [overload 3] :64 + -> RecordContext.encodeAvroSchema(schema) +``` + +## 9. `getOrderingValue(T, Schema, List)` +``` +BufferedRecords.fromEngineRecord() [overload 2] :59 + -> RecordContext.getOrderingValue(record, schema, orderingFieldNames) + +BufferedRecords.fromEngineRecord() [overload 3] :65 + -> RecordContext.getOrderingValue(record, schema, orderingFieldNames) +``` + +## 10. `getOrderingValue(DeleteRecord)` +``` +BufferedRecords.fromDeleteRecord() [overload 1] :70 + -> RecordContext.getOrderingValue(deleteRecord) + +BufferedRecords.fromDeleteRecord() [overload 2] :75 + -> RecordContext.getOrderingValue(deleteRecord) +``` + +## 11. `convertOrderingValueToEngineType()` +``` +BufferedRecords.fromHoodieRecord() :48 + -> RecordContext.convertOrderingValueToEngineType(orderingValue) + -> RecordContext.convertValueToEngineType(value) [for ArrayComparable] +``` + +## 12. `getDeleteRow()` +``` +UpdateProcessor.StandardUpdateProcessor.processUpdate() :96 + -> readerContext.getRecordContext() + -> RecordContext.getDeleteRow(recordKey) +``` + +## 13. `decodeAvroSchema()` + +**Path A — payload update** +``` +UpdateProcessor.PayloadUpdateProcessor.handleNonDeletes() :137 + -> readerContext.getRecordContext() + -> RecordContext.decodeAvroSchema(mergedRecord.getSchemaId()) +``` + +**Path B — merger factory** +``` +BufferedRecordMergerFactory.[merger impl] :528 + -> RecordContext.getSchemaFromBufferRecord(bufferedRecord) + -> RecordContext.decodeAvroSchema(record.getSchemaId()) +``` + +## 14. `convertToAvroRecord()` + +**Path A — payload update** +``` +UpdateProcessor.PayloadUpdateProcessor.handleNonDeletes() :138 + -> readerContext.getRecordContext() + -> RecordContext.convertToAvroRecord(mergedRecord.getRecord(), schema) +``` + +**Path B — merger factory** +``` +BufferedRecordMergerFactory.[merger impl] :529 + -> RecordContext.convertToAvroRecord(bufferedRecord.getRecord(), schema) +``` + +## 15. `convertAvroRecord()` + +**Path A — payload update rewrite** +``` +UpdateProcessor.PayloadUpdateProcessor.handleNonDeletes() :147 + -> readerContext.getRecordContext() + -> RecordContext.convertAvroRecord(rewrittenRecord.getData()) +``` + +**Path B — custom payload merger** +``` +BufferedRecordMergerFactory.CustomPayloadRecordMerger.deltaMerge() :480 + -> RecordContext.convertAvroRecord(mergedRecord.toIndexedRecord(...).getData()) +``` + +**Path C — custom record merger** +``` +BufferedRecordMergerFactory.CustomRecordMerger.finalMerge() :505 + -> RecordContext.convertAvroRecord(indexedRecord) +``` + +## 16. `getValue()` +``` +PartialUpdateHandler.reconcileDefaultValues() :120,122 + -> RecordContext.getValue(high.getRecord(), highSchema, fieldName) + -> RecordContext.getValue(low.getRecord(), lowSchema, fieldName) + +PartialUpdateHandler.reconcileMarkerValues() :164,168 + -> RecordContext.getValue(high.getRecord(), highSchema, fieldName) + -> RecordContext.getValue(low.getRecord(), lowSchema, fieldName) +``` + +## 17. `constructEngineRecord()` +``` +PartialUpdateHandler.reconcileDefaultValues() :131 + -> RecordContext.constructEngineRecord(newSchema, fieldVals) + +PartialUpdateHandler.reconcileMarkerValues() :177 + -> RecordContext.constructEngineRecord(newSchema, fieldVals) +``` + +## 18. `getTypeConverter()` +``` +PartialUpdateHandler.reconcileMarkerValues() :167 + -> RecordContext.getTypeConverter() + -> JavaTypeConverter.castToString(newValue) +``` + +## 19. `isDeleteRecord()` + +**Path A** +``` +KeyBasedFileGroupRecordBuffer.processDataBlock() :99 + -> readerContext.getRecordContext() + -> RecordContext.isDeleteRecord(nextRecord, deleteContext) +``` + +**Path B** +``` +PositionBasedFileGroupRecordBuffer.processDataBlock() :149 + -> readerContext.getRecordContext() + -> RecordContext.isDeleteRecord(evolvedNextRecord, deleteContext) +``` + +**Path C** +``` +StreamingFileGroupRecordBufferLoader.loadRecords() :100 + -> RecordContext.isDeleteRecord(data, deleteContext) +``` + +## 20. `getMetaFieldValue()` — called internally by `isDeleteRecord` +``` +RecordContext.isDeleteRecord(record, deleteContext) + -> RecordContext.isDeleteHoodieOperation(record, schema) :280 + -> RecordContext.getMetaFieldValue(record, hoodieOperationPos) +``` + +--- + +**Key structural insight**: `HoodieFileGroupReader` itself touches only 5 APIs directly (`setPartitionPath`, `supportsParquetRowIndex`, `seal`, `convertValueToEngineType`, `constructFinalHoodieRecord`). The remaining 15 APIs are reached through two delegation chains: +- **Record buffering path**: `BufferedRecords` / `BufferedRecordConverter` / buffer impls +- **Merge/update path**: `UpdateProcessor` → `BufferedRecordMergerFactory` → `PartialUpdateHandler` + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +to construct a reader context for spark, what are the input parameters + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🔍 Grep: pattern="class.*ReaderContext.*extends HoodieReaderContext" path=. + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 📖 Read: /Users/zhanyeha/hudi-internal/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRowReaderContext.java + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🔍 Grep: pattern="class BaseSparkInternalRecordContext" path=. + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🔍 Grep: pattern="class.*extends.*BaseSparkInternalRowReaderContext" path=. + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Now I have the full picture. Here's the answer: + +--- + +## Constructor chain for Spark reader context + +The concrete class is `SparkFileFormatInternalRowReaderContext`, which requires **5 parameters**: + +```scala +// SparkFileFormatInternalRowReaderContext.scala:58-63 +new SparkFileFormatInternalRowReaderContext( + baseFileReader: SparkColumnarFileReader, // Spark parquet reader (PartitionedFile → Iterator[InternalRow]) + filters: Seq[Filter], // pushdown filters (optional, may be pushed into reader) + requiredFilters: Seq[Filter], // filters always applied, even during merging + storageConfiguration: StorageConfiguration[_], + tableConfig: HoodieTableConfig +) +``` + +Internally its constructor immediately builds a `SparkFileFormatInternalRecordContext(tableConfig)` and passes everything up the chain: + +``` +SparkFileFormatInternalRowReaderContext(baseFileReader, filters, requiredFilters, storageConfig, tableConfig) + -> BaseSparkInternalRowReaderContext(storageConfig, tableConfig, new SparkFileFormatInternalRecordContext(tableConfig)) + -> HoodieReaderContext(storageConfig, tableConfig, Option.empty(), Option.empty(), recordContext) + +SparkFileFormatInternalRecordContext(tableConfig) + -> BaseSparkInternalRecordContext(tableConfig) + -> RecordContext(tableConfig, new DefaultJavaTypeConverter()) +``` + +## What each parameter drives + +| Parameter | Where used | +|-----------|-----------| +| `baseFileReader` | `getFileRecordIterator()` — reads base parquet files and log blocks | +| `filters` | Applied when no log files and no bootstrap merge needed | +| `requiredFilters` | Always applied; combined with `filters` into `allFilters` and `bootstrapSafeFilters` | +| `storageConfiguration` | Stored in `HoodieReaderContext`, passed to storage I/O | +| `tableConfig` | Drives `baseFileFormat`, record key extraction strategy (`populateMetaFields` vs virtual keys), merge mode, table version | + +## What `tableConfig` determines inside `RecordContext` + +At `RecordContext.` (:70-74): +- If `tableConfig.populateMetaFields()` → uses `metadataKeyExtractor()` (reads `_hoodie_record_key` meta field directly) +- Otherwise → uses `virtualKeyExtractor(tableConfig.getRecordKeyFields())` (computes key from user-defined key fields) + +So the **minimum required inputs** to construct a working Spark reader context are: +1. A `SparkColumnarFileReader` (the Spark parquet reader callback) +2. `StorageConfiguration` (Hadoop conf wrapper) +3. `HoodieTableConfig` (from `.hoodie/hoodie.properties`) +4. Optionally: pushdown `Filter`s + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +could you tell me how the record context is related to the record type returned by hoodie file group reader + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Now I have the full picture. Here's the relationship: + +--- + +## How RecordContext relates to the output record type + +`HoodieFileGroupReader` exposes **three public iterator methods**, and the output type of each is determined by the `IteratorMode` passed to `getBufferedRecordIterator()`. `RecordContext` sits at the boundary where the internal `BufferedRecord` gets projected into the caller-facing type. + +``` +HoodieFileGroupReader + │ + ▼ +getBufferedRecordIterator(mode) ─── always produces ClosableIterator> + │ + ├─ getClosableIterator() → ClosableIterator + │ maps: BufferedRecord::getRecord ← raw engine record, RecordContext not involved + │ + ├─ getClosableHoodieRecordIterator() → ClosableIterator> + │ maps: recordContext.constructFinalHoodieRecord(bufferedRecord) ← RecordContext involved + │ + └─ getClosableKeyIterator() → ClosableIterator + maps: BufferedRecord::getRecordKey ← RecordContext not involved +``` + +--- + +## The central role: `constructFinalHoodieRecord` + +This is the only RecordContext method that bridges `BufferedRecord` → `HoodieRecord`. Looking at `RecordContext.java`: + +```java +// RecordContext.java:147 +public HoodieRecord constructFinalHoodieRecord(BufferedRecord bufferedRecord) { + if (readerContext.getIteratorMode() == IteratorMode.HOODIE_RECORD) { + return constructHoodieRecord(bufferedRecord); // abstract — engine specific + } + ... +} +``` + +And in `BaseSparkInternalRecordContext.java:81-95`, Spark's implementation of `constructHoodieRecord`: + +```java +// wraps InternalRow into HoodieSparkRecord (or HoodieEmptyRecord for deletes) +HoodieKey hoodieKey = new HoodieKey(bufferedRecord.getRecordKey(), partitionPath); +if (bufferedRecord.isDelete()) { + return new HoodieEmptyRecord<>(hoodieKey, ...); +} +return new HoodieSparkRecord(hoodieKey, row, cachedSchema, false, ...); +``` + +--- + +## The internal currency: `BufferedRecord` + +Everything inside the reader pipeline — buffering, merging, log replay — works in terms of `BufferedRecord`, not raw `T` or `HoodieRecord`. `BufferedRecord` carries: + +| Field | Source | +|-------|--------| +| `record: T` | Raw engine record (e.g. `InternalRow`) | +| `recordKey: String` | Extracted via `RecordContext.getRecordKey()` | +| `orderingValue` | Extracted via `RecordContext.getOrderingValue()` | +| `schemaId` | Encoded via `RecordContext.encodeAvroSchema()` | +| `isDelete` | Flagged via `RecordContext.isDeleteRecord()` | + +`RecordContext` is what **populates** the `BufferedRecord` on ingestion and **unwraps** it into the output type on egress. + +--- + +## Summary diagram + +``` +File / Log + │ + │ getFileRecordIterator() → raw T (InternalRow) + ▼ +RecordContext.seal(T) ← copy/freeze raw record +RecordContext.getRecordKey(T) ┐ +RecordContext.encodeAvroSchema() │ BufferedRecordConverter wraps T +RecordContext.getOrderingValue(T) │ into BufferedRecord +RecordContext.isDeleteRecord(T) ┘ + │ + │ BufferedRecord (internal currency throughout merge pipeline) + │ + ├── merge / update path (UpdateProcessor, PartialUpdateHandler, etc.) + │ RecordContext.convertToAvroRecord() + │ RecordContext.convertAvroRecord() + │ RecordContext.constructEngineRecord() + │ ... + │ + ▼ +RecordContext.constructFinalHoodieRecord(BufferedRecord) ← egress + │ + ├── getClosableHoodieRecordIterator() → HoodieRecord (HoodieSparkRecord / HoodieEmptyRecord) + ├── getClosableIterator() → T (raw InternalRow, bypasses RecordContext) + └── getClosableKeyIterator() → String (bypasses RecordContext) +``` + +**Key point**: `RecordContext` is the type adapter between the engine's native record (`T`) and Hudi's record model. It is invoked at ingestion to enrich raw engine records into `BufferedRecord`, throughout the merge pipeline to transform them, and at egress only when the caller wants `HoodieRecord` — the raw-`T` and key-only paths skip it entirely. + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +could you tell me how the record type that hoodie file group reader returns and what are the intermediate + format it uses at various stages to store the records extracted from log file and base file. Show me in a + call stack tree so I know both the stage and the record type + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 📂 Glob: hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/*.java + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Now I have everything I need. Here is the full call stack tree showing both stage and record type at each level: + +--- + +--- + +## Summary: record type at each stage + +| Stage | Record type | Notes | +|-------|------------|-------| +| Base file read | `T` (`InternalRow`) | Raw from parquet reader | +| Log data block deserialization | `T` | Deserialized by `HoodieDataBlock.deserializeRecords()` | +| Log data block after schema evolution | `T` (projected) | `RecordContext.projectRecord()` applied | +| Log delete block | `DeleteRecord` | Avro-free; key + ordering value only | +| After wrapping for buffer | `BufferedRecord` | Enriched with key, ordering, schemaId, delete flag | +| In-buffer storage (delta merge done) | `BufferedRecord` (binary form) | `toBinary()` converts inner `T` to `UnsafeRow` for spillable map | +| After base+log final merge | `BufferedRecord` | Merged result; inner `T` is result of merger | +| After `UpdateProcessor` | `BufferedRecord` (sealed) | `seal()` copies/freezes the inner `T` | +| Output — `getClosableIterator` | `T` | Inner engine record unwrapped | +| Output — `getClosableHoodieRecordIterator` | `HoodieRecord` | Wrapped in `HoodieSparkRecord` / `HoodieEmptyRecord` | +| Output — `getClosableKeyIterator` | `String` | Only the record key | +| Output — `getClosableBufferedRecordIterator` | `BufferedRecord` | Full internal representation | diff --git a/filegroup/recordContext/task1/prompt b/filegroup/recordContext/task1/prompt new file mode 100644 index 00000000..2f5dd078 --- /dev/null +++ b/filegroup/recordContext/task1/prompt @@ -0,0 +1,52 @@ +Here is how hudi-internal contruct the record context and inject into the fg reader + + HoodieFileGroupReaderBasedFileFormat.buildReaderWithPartitionValues() :227 + │ [Spark scan layer — one readerContext per task] + │ + └─ new SparkFileFormatInternalRowReaderContext( + fileGroupBaseFileReader, + filters, + requiredFilters, + storageConf, + metaClient.getTableConfig() ← HoodieTableConfig from .hoodie/hoodie.properties + ) + │ + └─ SparkFileFormatInternalRowReaderContext.() :63 + └─ extends BaseSparkInternalRowReaderContext( + storageConf, + tableConfig, + new SparkFileFormatInternalRecordContext(tableConfig) ← RecordContext built here + ) + │ + ├─ [RecordContext construction path] + │ SparkFileFormatInternalRecordContext.(tableConfig) + │ └─ extends BaseSparkInternalRecordContext(tableConfig) + │ └─ BaseSparkInternalRecordContext.(tableConfig) :54 + │ └─ super(tableConfig, new DefaultJavaTypeConverter()) + │ │ + │ └─ RecordContext.(tableConfig, typeConverter) :70 + │ │ + │ └─ tableConfig.populateMetaFields() :72 + │ │ + │ ├─ true → recordKeyExtractor = metadataKeyExtractor() + │ │ reads _hoodie_record_key meta field directly + │ │ getValue(record, schema, RECORD_KEY_METADATA_FIELD) + │ │ + │ └─ false → recordKeyExtractor = virtualKeyExtractor( + │ tableConfig.getRecordKeyFields()) + │ computes key from user-defined key fields + └─ [ReaderContext construction path] + BaseSparkInternalRowReaderContext.(storageConf, tableConfig, recordContext) :54 + └─ super(storageConf, tableConfig, Option.empty(), Option.empty(), recordContext) + │ + └─ HoodieReaderContext.() :98 + this.recordContext = recordContext ← stores the already-built + +We should restrict the construction part of the record context to take the same set of variables (seems to be tableConfig only) and do the same injection into the fg reader + +==== + + B1: isDeleteRecord() never called during data block processing + + - Java: processDataBlock:99 calls recordContext.isDeleteRecord(nextRecord, deleteContext) for every data record + - hudi-rs: process_data_block() (key_based.rs:172-196) never calls it — soft-delete records (_hoodie_is_deleted=true) are ingested as normal records diff --git a/filegroup/sparkOssMorReadFlow.md b/filegroup/sparkOssMorReadFlow.md new file mode 100644 index 00000000..e7f99ec2 --- /dev/null +++ b/filegroup/sparkOssMorReadFlow.md @@ -0,0 +1,393 @@ +# Analyze phase +Analyzer rule `class FindDataSourceTable(sparkSession: SparkSession) extends Rule[LogicalPlan]` +``` + def resolveRelation(checkFilesExist: Boolean = true): BaseRelation = { + val relation = (providingInstance(), userSpecifiedSchema) match { + // TODO: Throw when too much is given. + case (dataSource: SchemaRelationProvider, Some(schema)) => + dataSource.createRelation(sparkSession.sqlContext, caseInsensitiveOptions, schema) +``` +-> `org.apache.hudi.DefaultSource#createRelation`. It also decides RO v.s. RT. +``` + + def createRelation(sqlContext: SQLContext, + metaClient: HoodieTableMetaClient, + schema: StructType, + parameters: Map[String, String]): BaseRelation = { + case (MERGE_ON_READ, QUERY_TYPE_SNAPSHOT_OPT_VAL, false) => + if (isNotMetadataTable) { + new HoodieMergeOnReadSnapshotHadoopFsRelationFactory( + sqlContext, metaClient, parameters, userSchema, isBootstrap = false).build() + } else { + new MergeOnReadSnapshotRelation(sqlContext, parameters, metaClient, userSchema) + } +``` +# Physical planning phase + +## Create file source scan exec +``` +case class FileSourceScanExec( + @transient override val relation: HadoopFsRelation, + override val output: Seq[Attribute], + override val requiredSchema: StructType, + override val partitionFilters: Seq[Expression], + override val optionalBucketSet: Option[BitSet], + override val optionalNumCoalescedBuckets: Option[Int], + override val dataFilters: Seq[Expression], + override val tableIdentifier: Option[TableIdentifier], + override val disableBucketedScan: Boolean = false) + extends FileSourceScanLike { +``` +### what base files to read: +org.apache.spark.sql.execution.FileSourceScanLike#dynamicallySelectedPartitions + +Internally hoodie file index first gets all file slices, then use this func to extract base file / 1 log file to build spark `FileStatus` object +``` + def convertFileSliceToPartitionDirectory(partitionValues: InternalRow, + fileSlice: FileSlice, + options: Map[String, String]): PartitionDirectory = { + val logFileEstimationFraction = options.getOrElse(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key(), + HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue()).toDouble + // 1. Generate a delegate file for file slice, which spark uses to optimize rdd partition parallelism based on data such as file size + // - For file slice only has base file, we directly use the base file size as delegate file size + // - For file slice has log file, we estimate the delegate file size based on the log file size and option(base file) size + val estimationFileSize = fileSlice.getTotalFileSizeAsParquetFormat(logFileEstimationFraction) + val fileInfo = if (fileSlice.getBaseFile.isPresent) { + fileSlice.getBaseFile.get().getPathInfo + } else { + fileSlice.getLogFiles.findAny().get().getPathInfo + } + // create a delegate file status based on the file size estimation + val delegateFile = new FileStatus(estimationFileSize, fileInfo.isDirectory, 0, fileInfo.getBlockSize, fileInfo.getModificationTime, new Path(fileInfo.getPath.toUri)) + + // 2. Generate a partition directory based on the delegate file and partition values + if (fileSlice.hasLogFiles || fileSlice.hasBootstrapBase) { + // should read as file slice, so we need to create a mapping from fileId to file slice + sparkAdapter.getSparkPartitionedFileUtils.newPartitionDirectory( + sparkAdapter.createPartitionFileSliceMapping(partitionValues, Map(fileSlice.getFileId -> fileSlice)), Seq(delegateFile)) + } else { + sparkAdapter.getSparkPartitionedFileUtils.newPartitionDirectory(partitionValues, Seq(delegateFile)) + } + } +``` + +### MOR how to figure out file group based on the parquet files + +relation.fileFormat.buildReaderWithPartitionValues - offloaded to the HoodieMergeOnReadSnapshotHadoopFsRelation: + +``` + lazy val inputRDD: RDD[InternalRow] = { + val options = relation.options + + (FileFormat.OPTION_RETURNING_BATCH -> supportsColumnar.toString) + val readFile: (PartitionedFile) => Iterator[InternalRow] = + relation.fileFormat.buildReaderWithPartitionValues( <-------------- returns func consuming individual `FileStatus` from `PartitionDirectory` + sparkSession = relation.sparkSession, + dataSchema = relation.dataSchema, + partitionSchema = relation.partitionSchema, + requiredSchema = requiredSchema, + filters = pushedDownFilters, + options = options, + hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options)) + + val readRDD = if (bucketedScan) { + createBucketedReadRDD(relation.bucketSpec.get, readFile, dynamicallySelectedPartitions) + } else { + createReadRDD(readFile, dynamicallySelectedPartitions) + } + sendDriverMetrics() + readRDD + } +``` + +buildReaderWithPartitionValues method return this reader func, where given parquet file path, extract file group id and use that to figure out all log files +Inside that we build file group reader as well. +``` + (file: PartitionedFile) => { + // executor + val storageConf = new HadoopStorageConfiguration(broadcastedStorageConf.value.value) + val iter = file.partitionValues match { + // Snapshot or incremental queries. + case fileSliceMapping: HoodiePartitionFileSliceMapping => + val fileGroupName = FSUtils.getFileIdFromFilePath(sparkAdapter + .getSparkPartitionedFileUtils.getPathFromPartitionedFile(file)) + fileSliceMapping.getSlice(fileGroupName) match { + case Some(fileSlice) if !isCount && (requiredSchema.nonEmpty || fileSlice.getLogFiles.findAny().isPresent) => + val readerContext = new SparkFileFormatInternalRowReaderContext(fileGroupBaseFileReader.value, filters, requiredFilters, storageConf, metaClient.getTableConfig) + readerContext.setEnableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true)) + val props = metaClient.getTableConfig.getProps + options.foreach(kv => props.setProperty(kv._1, kv._2)) + props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction)) + val baseFileLength = if (fileSlice.getBaseFile.isPresent) { + fileSlice.getBaseFile.get.getFileSize + } else { + 0 + } + val reader = HoodieFileGroupReader.newBuilder() + .withReaderContext(readerContext) + .withHoodieTableMetaClient(metaClient) + .withLatestCommitTime(queryTimestamp) + .withFileSlice(fileSlice) + .withDataSchema(dataSchema) + .withRequestedSchema(requestedSchema) + .withInternalSchema(internalSchemaOpt) + .withProps(props) + .withStart(file.start) + .withLength(baseFileLength) + .withShouldUseRecordPosition(shouldUseRecordPosition) + .build() + // Append partition values to rows and project to output schema + appendPartitionAndProject( + reader.getClosableIterator, + requestedStructType, + remainingPartitionSchema, + outputSchema, + fileSliceMapping.getPartitionValues, + fixedPartitionIndexes) +``` + +### RDD parallelism (file spliting) +For MOR it is not splitable, 1 parquet file belongs to 1 spark task. 1 file group has 1 parquet file. So 1 file group is handled by 1 spark task at a time. +``` + override def isSplitable(sparkSession: SparkSession, + options: Map[String, String], + path: Path): Boolean = { + // NOTE: When we have and only the base file that needs to be read with normal reading mode, + // we can consider the current format to be equivalent to `org.apache.spark.sql.execution.datasources.parquet.ParquetFormat`. + // Naturally, we can maintain the same `isSplitable` logic as the upper-level format. + // This will enable us to take advantage of spark's file splitting capability. + // For overly large single files, we can use multiple concurrent tasks to read them, thereby reducing the overall job reading time consumption + val superSplitable = super.isSplitable(sparkSession, options, path) + val splitable = !isMOR && !isIncremental && !isBootstrap && superSplitable + logInfo(s"isSplitable: $splitable, super.isSplitable: $superSplitable, isMOR: $isMOR, isIncremental: $isIncremental, isBootstrap: $isBootstrap") + splitable + } +``` + +# Chat + ~/hudi-oss 17df0307-624c-4a31-9d42-c108efee6f9b +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +def convertFileSliceToPartitionDirectory(partitionValues: InternalRow, + fileSlice: FileSlice, + options: Map[String, String]): PartitionDirectory = { + val logFileEstimationFraction = options.getOrElse(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key(), + HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue()).toDouble + // 1. Generate a delegate file for file slice, which spark uses to optimize rdd partition parallelism based on data such as file size + // - For file slice only has base file, we directly use the base file size as delegate file size + // - For file slice has log file, we estimate the delegate file size based on the log file size and option(base file) size + val estimationFileSize = fileSlice.getTotalFileSizeAsParquetFormat(logFileEstimationFraction) + val fileInfo = if (fileSlice.getBaseFile.isPresent) { + fileSlice.getBaseFile.get().getPathInfo + } else { + fileSlice.getLogFiles.findAny().get().getPathInfo + } + // create a delegate file status based on the file size estimation + val delegateFile = new FileStatus(estimationFileSize, fileInfo.isDirectory, 0, fileInfo.getBlockSize, fileInfo.getModificationTime, new Path(fileInfo.getPath.toUri)) + + // 2. Generate a partition directory based on the delegate file and partition values + if (fileSlice.hasLogFiles || fileSlice.hasBootstrapBase) { + // should read as file slice, so we need to create a mapping from fileId to file slice + sparkAdapter.getSparkPartitionedFileUtils.newPartitionDirectory( + sparkAdapter.createPartitionFileSliceMapping(partitionValues, Map(fileSlice.getFileId -> fileSlice)), Seq(delegateFile)) + } else { + sparkAdapter.getSparkPartitionedFileUtils.newPartitionDirectory(partitionValues, Seq(delegateFile)) + } + } + +could you explain for log file only file slice, eventually what does createPartitionFileSliceMapping do and how eventually it is consumed by + + +buildReaderWithPartitionValues method return this reader func, where given parquet file path, extract file group id and use that to figure out all log files +Inside that we build file group reader as well. + +hudi/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedFileFormat.scala +``` + (file: PartitionedFile) => { + // executor + val storageConf = new HadoopStorageConfiguration(broadcastedStorageConf.value.value) + val iter = file.partitionValues match { + // Snapshot or incremental queries. + case fileSliceMapping: HoodiePartitionFileSliceMapping => + val fileGroupName = FSUtils.getFileIdFromFilePath(sparkAdapter + .getSparkPartitionedFileUtils.getPathFromPartitionedFile(file)) + fileSliceMapping.getSlice(fileGroupName) match { + case Some(fileSlice) if !isCount && (requiredSchema.nonEmpty || fileSlice.getLogFiles.findAny().isPresent) => + val readerContext = new SparkFileFormatInternalRowReaderContext(fileGroupBaseFileReader.value, filters, requiredFilters, storageConf, metaClient.getTableConfig) + readerContext.setEnableLogicalTimestampFieldRepair(storageConf.getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true)) + val props = metaClient.getTableConfig.getProps + options.foreach(kv => props.setProperty(kv._1, kv._2)) + props.put(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE.key(), String.valueOf(maxMemoryPerCompaction)) + val baseFileLength = if (fileSlice.getBaseFile.isPresent) { + fileSlice.getBaseFile.get.getFileSize + } else { + 0 + } + val reader = HoodieFileGroupReader.newBuilder() + .withReaderContext(readerContext) + .withHoodieTableMetaClient(metaClient) + .withLatestCommitTime(queryTimestamp) + .withFileSlice(fileSlice) + .withDataSchema(dataSchema) + .withRequestedSchema(requestedSchema) + .withInternalSchema(internalSchemaOpt) + .withProps(props) + .withStart(file.start) + .withLength(baseFileLength) + .withShouldUseRecordPosition(shouldUseRecordPosition) + .build() + // Append partition values to rows and project to output schema + appendPartitionAndProject( + reader.getClosableIterator, + requestedStructType, + remainingPartitionSchema, + outputSchema, + fileSliceMapping.getPartitionValues, + fixedPartitionIndexes) + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Let me trace the key pieces. + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🤖 Agent: Trace createPartitionFileSliceMapping + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here's the full picture for **log-file-only file slices**: + +## 1. `createPartitionFileSliceMapping` — what it does + +It's simple: it wraps a `Map[String, FileSlice]` (keyed by **file group ID**) into a special `InternalRow` subclass (`HoodiePartitionFileSliceMapping`). The mapping is created with exactly one entry: + +```scala +Map(fileSlice.getFileId -> fileSlice) +``` + +The `getSlice(fileId)` method is just `slices.get(fileId)` — a HashMap lookup. + +## 2. The trick — smuggling the FileSlice through Spark's partition machinery + +Spark's `PartitionDirectory` holds `(partitionValues: InternalRow, files: Seq[FileStatus])`. Normally `partitionValues` is just partition column values. Hudi **replaces** it with `HoodiePartitionFileSliceMapping`, which extends `InternalRow` so Spark accepts it, but also carries the fileId→FileSlice map. + +The `delegateFile` (a `FileStatus`) is just a stand-in so Spark can do its split/parallelism planning based on estimated file size. For a log-only slice, the delegate file path is the **log file path**. + +## 3. How it's consumed at read time + +When `buildReaderWithPartitionValues` returns the reader function: + +1. Spark calls `(file: PartitionedFile) => ...` with a `PartitionedFile` derived from the delegate `FileStatus` +2. The code pattern-matches `file.partitionValues` as `HoodiePartitionFileSliceMapping` — this tells it "this needs file-group-level reading" +3. It extracts the file group ID from the delegate file path: + ```scala + val fileGroupName = FSUtils.getFileIdFromFilePath(getPathFromPartitionedFile(file)) + ``` + For a log file like `.f-0_20250101010101.log.0_0-0-0`, `getFileIdFromLogPath` uses a regex to extract `f-0` (group 1 of the log file pattern) +4. It looks up the FileSlice: `fileSliceMapping.getSlice(fileGroupName)` → `Some(fileSlice)` +5. It builds a `HoodieFileGroupReader` with that FileSlice, which knows about all the log files (and no base file), and reads/merges them + +## Key insight + +The entire mechanism is a way to **pass the FileSlice through Spark's file-based planning infrastructure** without Spark knowing about it. Spark sees a normal `PartitionDirectory` with a file and partition values. Hudi's reader function then unpacks the hidden FileSlice mapping from the "partition values" to get the actual log files it needs to read. + +──────────────────────────────────────────────────────────────────────────────── +👤 USER: +do you know at the time we build file group reader, how these input are populated? + + Input 2: latestInstantTime — High-watermark for block filtering + + // BaseHoodieLogRecordReader.java:144 + this.latestInstantTime = readerContext.getLatestCommitTime(); + + Set by: + HoodieFileGroupReader constructor [HoodieFileGroupReader.java:113] + └─ readerContext.setLatestCommitTime(latestCommitTime) + + Input 3: instantRange — Optional incremental range filter + + // BaseHoodieLogRecordReader constructor, stored as this.instantRange + // LogScanningRecordBufferLoader.java:47 + .withInstantRange(readerContext.getInstantRange()) + + Input 4: allowInflightInstants — Whether to include uncommitted blocks + + // LogScanningRecordBufferLoader.java:50 + .withAllowInflightInstants(readerParameters.allowInflightInstants()) + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: + 🤖 Agent: Trace file group reader inputs + +──────────────────────────────────────────────────────────────────────────────── +🤖 ASSISTANT: +Here's how each is populated in the Spark snapshot read path: + +--- + +### Input 2: `latestCommitTime` (→ `latestInstantTime` in log reader) + +**Value:** `queryTimestamp` from `HoodieFileGroupReaderBasedFileFormat`'s constructor. + +**Origin** (`HoodieHadoopFsRelationFactory.scala`): +```scala +private lazy val queryTimestamp: Option[String] = + specifiedQueryTimestamp.orElse(toScalaOption(timeline.lastInstant()).map(_.requestedTime)) +``` + +- If `TIME_TRAVEL_AS_OF_INSTANT` is set in read options → uses that formatted timestamp +- Otherwise → **last completed instant on the timeline** (e.g., `"20240101120000"`) + +This flows through: +1. Builder: `.withLatestCommitTime(queryTimestamp)` +2. Constructor: `readerContext.setLatestCommitTime(latestCommitTime)` (`HoodieFileGroupReader.java:113`) +3. Log reader: `this.latestInstantTime = readerContext.getLatestCommitTime()` + +**Effect:** The log scanner uses this as a high-watermark — it **skips log blocks whose instant time is greater than this**, so you get a point-in-time consistent read. + +--- + +### Input 3: `instantRange` + +**Value:** `Option.empty()` for Spark snapshot reads. + +**Why:** `BaseSparkInternalRowReaderContext` hardcodes it: +```java +// BaseSparkInternalRowReaderContext.java +super(storageConfiguration, tableConfig, Option.empty(), Option.empty(), recordContext); +// ^^^^^^^^^^^^^^ ← instantRangeOpt +``` + +No code in the Spark snapshot path ever calls `setInstantRange()`. So: +- `readerContext.getInstantRange()` returns empty +- The log scanner gets an empty range → **no instant range filtering** +- Base file records also skip the `applyInstantRangeFilter` path + +This parameter becomes relevant for **incremental/CDC queries** where a custom reader context provides an actual range like `[commitA, commitB]`. + +--- + +### Input 4: `allowInflightInstants` + +**Value:** `false` (default). + +**Why:** The builder declares: +```java +// HoodieFileGroupReader.java, Builder +private boolean allowInflightInstants = false; +``` + +`HoodieFileGroupReaderBasedFileFormat` never calls `.withAllowInflightInstants(true)`, so it stays `false`. + +**Effect:** The log scanner will **skip blocks from uncommitted/inflight instants**. Only blocks from completed commits (up to `latestInstantTime`) are processed. This prevents dirty reads of in-progress writes. + +--- + +### Summary for snapshot reads + +| Input | Value | Effect | +|---|---|---| +| `latestCommitTime` | Last completed instant (or time-travel target) | High-watermark: skip log blocks newer than this | +| `instantRange` | `empty` | No range filtering — read all blocks up to the watermark | +| `allowInflightInstants` | `false` | Skip uncommitted/inflight blocks | + +Together these ensure a **consistent snapshot**: read everything committed up to the watermark, ignore anything newer or uncommitted. diff --git a/python/src/internal.rs b/python/src/internal.rs index ca5081c2..6b2d1270 100644 --- a/python/src/internal.rs +++ b/python/src/internal.rs @@ -29,7 +29,6 @@ use datafusion::error::DataFusionError; use hudi::error::CoreError; use hudi::file_group::FileGroup; use hudi::file_group::file_slice::FileSlice; -use hudi::file_group::reader::FileGroupReader; use hudi::storage::error::StorageError; use hudi::table::Table; use hudi::table::builder::TableBuilder; @@ -70,92 +69,6 @@ impl From for PyErr { } } -#[cfg(not(tarpaulin_include))] -#[derive(Clone, Debug)] -#[pyclass] -pub struct HudiFileGroupReader { - inner: FileGroupReader, -} - -#[cfg(not(tarpaulin_include))] -#[pymethods] -impl HudiFileGroupReader { - #[new] - #[pyo3(signature = (base_uri, options=None))] - fn new_with_options( - py: Python, - base_uri: &str, - options: Option>, - ) -> PyResult { - let inner = py.detach(|| { - rt().block_on(FileGroupReader::new_with_options( - base_uri, - options.unwrap_or_default(), - )) - .map_err(PythonError::from) - })?; - Ok(HudiFileGroupReader { inner }) - } - - fn read_file_slice_by_base_file_path( - &self, - relative_path: &str, - py: Python, - ) -> PyResult> { - py.detach(|| { - rt().block_on(self.inner.read_file_slice_by_base_file_path(relative_path)) - .map_err(PythonError::from) - })? - .to_pyarrow(py) - .map(|b| b.unbind()) - } - - fn read_file_slice(&self, file_slice: &HudiFileSlice, py: Python) -> PyResult> { - let mut file_group = FileGroup::new_with_base_file_name( - &file_slice.base_file_name, - &file_slice.partition_path, - ) - .map_err(PythonError::from)?; - let log_file_names = &file_slice.log_file_names; - file_group - .add_log_files_from_names(log_file_names) - .map_err(PythonError::from)?; - let (_, file_slice) = file_group - .file_slices - .iter() - .next() - .ok_or_else(|| { - CoreError::FileGroup(format!( - "Failed to get file slice from file group: {file_group:?}" - )) - }) - .map_err(PythonError::from)?; - py.detach(|| { - rt().block_on(self.inner.read_file_slice(file_slice)) - .map_err(PythonError::from) - })? - .to_pyarrow(py) - .map(|b| b.unbind()) - } - - fn read_file_slice_from_paths( - &self, - base_file_path: &str, - log_file_paths: Vec, - py: Python, - ) -> PyResult> { - py.detach(|| { - rt().block_on( - self.inner - .read_file_slice_from_paths(base_file_path, log_file_paths), - ) - .map_err(PythonError::from) - })? - .to_pyarrow(py) - .map(|b| b.unbind()) - } -} - #[cfg(not(tarpaulin_include))] #[derive(Clone, Debug)] #[pyclass] @@ -221,8 +134,15 @@ impl From<&FileSlice> for HudiFileSlice { let file_id = f.file_id().to_string(); let partition_path = f.partition_path.to_string(); let creation_instant_time = f.creation_instant_time().to_string(); - let base_file_name = f.base_file.file_name(); - let file_metadata = f.base_file.file_metadata.clone().unwrap_or_default(); + let base_file_name = f + .base_file + .as_ref() + .map_or(String::new(), |bf| bf.file_name()); + let file_metadata = f + .base_file + .as_ref() + .and_then(|bf| bf.file_metadata.clone()) + .unwrap_or_default(); let base_file_size = file_metadata.size; let base_file_byte_size = file_metadata.byte_size; let log_file_names = f.log_files.iter().map(|l| l.file_name()).collect(); @@ -479,18 +399,6 @@ impl HudiTable { }) } - #[pyo3(signature = (options=None))] - fn create_file_group_reader_with_options( - &self, - options: Option>, - ) -> PyResult { - let fg_reader = self - .inner - .create_file_group_reader_with_options(options.unwrap_or_default()) - .map_err(PythonError::from)?; - Ok(HudiFileGroupReader { inner: fg_reader }) - } - #[pyo3(signature = (filters=None))] fn read_snapshot( &self, diff --git a/python/src/lib.rs b/python/src/lib.rs index 7588ec89..0d594f45 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -31,8 +31,7 @@ mod testing_internal; fn _internal(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("__version__", env!("CARGO_PKG_VERSION"))?; - use internal::{HudiFileGroupReader, HudiFileSlice, HudiInstant, HudiTable, HudiTimeline}; - m.add_class::()?; + use internal::{HudiFileSlice, HudiInstant, HudiTable, HudiTimeline}; m.add_class::()?; m.add_class::()?; m.add_class::()?;