Storages: introduce trim min-max index for DATE/DATETIME/TIMESTAMP rough set filtering

.gitignore

@@ -92,3 +92,11 @@ CoverageReport
 # installation and sysroot path
 release-linux-llvm/tiflash
 release-linux-llvm/build-release
+release-linux-llvm/build-release-columnar
+release-linux-llvm/.cargo/
+
+
+gtest_10x_workdir/
+second_opinion*
+
+license-eye

dbms/src/Common/FailPoint.cpp

@@ -107,6 +107,7 @@ namespace DB
     M(force_set_mocked_s3_object_mtime)                      \
     M(force_stop_background_checkpoint_upload)               \
     M(force_schema_sync_diff_fail)                           \
+    M(force_get_dropped_table_info_in_schema_sync)           \
     M(exception_after_large_write_exceed)                    \
     M(proactive_flush_force_set_type)                        \
     M(exception_when_fetch_disagg_pages)                     \

dbms/src/Common/TiFlashMetrics.h

@@ -365,6 +365,38 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
       "Bucketed histogram of rough set filter rate",                                                                                \
       Histogram,                                                                                                                    \
       F(type_dtfile_pack, {{"type", "dtfile_pack"}}, EqualWidthBuckets{0, 6, 20}))                                                  \
+    M(tiflash_storage_rough_set_pack_count,                                                                                         \
+      "Total number of packs before and after query rough set filtering",                                                           \
+      Counter,                                                                                                                      \
+      F(stage_query_input, {"stage", "query_input"}),                                                                               \
+      F(stage_query_filtered, {"stage", "query_filtered"}),                                                                         \
+      F(stage_query_remaining, {"stage", "query_remaining"}))                                                                       \
+    M(tiflash_storage_trim_minmax_select_count,                                                                                     \
+      "Total number of trim min-max selection results",                                                                             \
+      Counter,                                                                                                                      \
+      F(result_used, {"result", "used"}),                                                                                           \
+      F(result_fallback_disabled, {"result", "fallback_disabled"}),                                                                 \
+      F(result_fallback_non_meta_v2, {"result", "fallback_non_meta_v2"}),                                                           \
+      F(result_fallback_column_missing, {"result", "fallback_column_missing"}),                                                     \
+      F(result_fallback_no_meta, {"result", "fallback_no_meta"}),                                                                   \
+      F(result_fallback_unsupported_version, {"result", "fallback_unsupported_version"}),                                           \
+      F(result_fallback_metadata_mismatch, {"result", "fallback_metadata_mismatch"}),                                               \
+      F(result_fallback_index_missing, {"result", "fallback_index_missing"}),                                                       \
+      F(result_fallback_unsupported_expression, {"result", "fallback_unsupported_expression"}),                                     \
+      F(result_fallback_predicate_outside_range, {"result", "fallback_predicate_outside_range"}),                                   \
+      F(result_fallback_invalid_pack_marks, {"result", "fallback_invalid_pack_marks"}))                                             \
+    M(tiflash_storage_trim_minmax_rough_check_pack_count,                                                                           \
+      "Total number of packs by trim min-max rough check result",                                                                   \
+      Counter,                                                                                                                      \
+      F(result_none, {"result", "none"}),                                                                                           \
+      F(result_some, {"result", "some"}),                                                                                           \
+      F(result_all, {"result", "all"}),                                                                                             \
+      F(result_all_null, {"result", "all_null"}))                                                                                   \
+    M(tiflash_storage_trim_minmax_correction_pack_count,                                                                            \
+      "Total number of packs whose trim min-max rough check result is conservatively corrected",                                    \
+      Counter,                                                                                                                      \
+      F(type_none_to_some, {"type", "none_to_some"}),                                                                               \
+      F(type_all_to_some, {"type", "all_to_some"}))                                                                                 \
     M(tiflash_disaggregated_object_lock_request_count,                                                                              \
       "Total number of S3 object lock/delete request",                                                                              \
       Counter,                                                                                                                      \

dbms/src/Interpreters/Settings.h

@@ -174,6 +174,7 @@ struct Settings
     M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.")                                                                          \
     M(SettingBool, dt_enable_logical_split, false, "Enable logical split or not in DeltaTree Engine.")                                                                                                                                  \
     M(SettingBool, dt_enable_rough_set_filter, true, "Whether to parse where expression as Rough Set Index filter or not.")                                                                                                             \
+    M(SettingBool, dt_enable_trim_minmax, true, "Whether to generate and use trim min-max index for DATE/DATETIME/TIMESTAMP Rough Set filtering.")                                                                                     \
     M(SettingBool, dt_enable_relevant_place, false, "Enable relevant place or not in DeltaTree Engine.")                                                                                                                                \
     M(SettingBool, dt_enable_skippable_place, true, "Enable skippable place or not in DeltaTree Engine.")                                                                                                                               \
     M(SettingBool, dt_enable_stable_column_cache, true, "Enable column cache for StorageDeltaMerge.")                                                                                                                                   \

dbms/src/Storages/DeltaMerge/File/ColumnStat.h

@@ -20,6 +20,8 @@
 #include <Storages/DeltaMerge/dtpb/dmfile.pb.h>
 #include <Storages/KVStore/Types.h>
 
+#include <optional>
+
 namespace DB::DM
 {
 struct ColumnStat
@@ -42,6 +44,9 @@ struct ColumnStat
 
     std::vector<dtpb::VectorIndexFileProps> vector_index;
 
+    // Optional trim min-max metadata. Independent from vector_index / local-index lifecycle.
+    std::optional<dtpb::TrimMinMaxIndexProps> trim_minmax_index{};
+
 #ifndef NDEBUG
     // This field is only used for testing
     String additional_data_for_test{};
@@ -68,6 +73,9 @@ struct ColumnStat
             pb_idx->CopyFrom(vec_idx);
         }
 
+        if (trim_minmax_index.has_value())
+            *stat.mutable_trim_minmax_index() = *trim_minmax_index;
+
 #ifndef NDEBUG
         stat.set_additional_data_for_test(additional_data_for_test);
 #endif
@@ -104,6 +112,13 @@ struct ColumnStat
             vector_index.emplace_back(pb_idx);
         }
 
+        // Soft-load only. Structural validation and fallback happen at selection time so a
+        // corrupt / unsupported trim meta never fails DMFile open.
+        if (proto.has_trim_minmax_index())
+            trim_minmax_index = proto.trim_minmax_index();
+        else
+            trim_minmax_index.reset();
+
 #ifndef NDEBUG
         additional_data_for_test = proto.additional_data_for_test();
 #endif
@@ -185,6 +200,7 @@ readText(ColumnStats & column_sats, DMFileFormat::Version ver, ReadBuffer & buf)
                 .serialized_bytes = serialized_bytes,
                 // ... here ignore some fields with default initializers
                 .vector_index = {},
+                .trim_minmax_index = {},
 #ifndef NDEBUG
                 .additional_data_for_test = {},
 #endif

dbms/src/Storages/DeltaMerge/File/DMFile.cpp

@@ -191,6 +191,12 @@ String DMFile::colIndexCacheKey(const FileNameBase & file_name_base) const
     return colIndexPath(file_name_base);
 }
 
+String DMFile::colTrimIndexCacheKey(const FileNameBase & file_name_base) const
+{
+    // Distinct from ordinary `.idx` cache key; path already ends with `.trim.idx`.
+    return colTrimIndexPath(file_name_base);
+}
+
 String DMFile::colMarkCacheKey(const FileNameBase & file_name_base) const
 {
     return colMarkPath(file_name_base);
@@ -284,6 +290,11 @@ EncryptionPath DMFile::encryptionIndexPath(const FileNameBase & file_name_base)
     return EncryptionPath(encryptionBasePath(), file_name_base + details::INDEX_FILE_SUFFIX, keyspaceId());
 }
 
+EncryptionPath DMFile::encryptionTrimIndexPath(const FileNameBase & file_name_base) const
+{
+    return EncryptionPath(encryptionBasePath(), file_name_base + details::TRIM_INDEX_FILE_SUFFIX, keyspaceId());
+}
+
 EncryptionPath DMFile::encryptionMarkPath(const FileNameBase & file_name_base) const
 {
     return EncryptionPath(encryptionBasePath(), file_name_base + details::MARK_FILE_SUFFIX, keyspaceId());

dbms/src/Storages/DeltaMerge/File/DMFile.h

@@ -287,19 +287,25 @@ class DMFile : private boost::noncopyable
     {
         return subFilePath(colIndexFileName(file_name_base));
     }
+    String colTrimIndexPath(const FileNameBase & file_name_base) const
+    {
+        return subFilePath(colTrimIndexFileName(file_name_base));
+    }
     String colMarkPath(const FileNameBase & file_name_base) const
     {
         return subFilePath(colMarkFileName(file_name_base));
     }
 
     String colIndexCacheKey(const FileNameBase & file_name_base) const;
+    String colTrimIndexCacheKey(const FileNameBase & file_name_base) const;
     String colMarkCacheKey(const FileNameBase & file_name_base) const;
 
     bool isColIndexExist(const ColId & col_id) const;
 
     String encryptionBasePath() const;
     EncryptionPath encryptionDataPath(const FileNameBase & file_name_base) const;
     EncryptionPath encryptionIndexPath(const FileNameBase & file_name_base) const;
+    EncryptionPath encryptionTrimIndexPath(const FileNameBase & file_name_base) const;
     EncryptionPath encryptionMarkPath(const FileNameBase & file_name_base) const;
 
     static FileNameBase getFileNameBase(ColId col_id, const IDataType::SubstreamPath & substream = {})

dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp

@@ -71,7 +71,8 @@ DMFileBlockInputStreamPtr DMFileBlockInputStreamBuilder::build(
             read_limiter,
             scan_context,
             tracing_id,
-            read_tag);
+            read_tag,
+            enable_trim_minmax);
     }
 
     bool enable_read_thread = SegmentReaderPoolManager::instance().isSegmentReader();
@@ -128,6 +129,7 @@ DMFileBlockInputStreamPtr createSimpleBlockInputStream(
 DMFileBlockInputStreamBuilder & DMFileBlockInputStreamBuilder::setFromSettings(const Settings & settings)
 {
     enable_column_cache = settings.dt_enable_stable_column_cache;
+    enable_trim_minmax = settings.dt_enable_trim_minmax;
     max_read_buffer_size = settings.max_read_buffer_size;
     max_sharing_column_bytes_for_all = settings.dt_max_sharing_column_bytes_for_all;
     return *this;
@@ -197,7 +199,8 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::tryBuildWithVectorIn
             read_limiter,
             scan_context,
             tracing_id,
-            ReadTag::Query);
+            ReadTag::Query,
+            enable_trim_minmax);
     }
 
     bool enable_read_thread = SegmentReaderPoolManager::instance().isSegmentReader();

dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h

@@ -222,10 +222,10 @@ class DMFileBlockInputStreamBuilder
     FileProviderPtr file_provider;
 
     // clean read
-
     bool enable_handle_clean_read = false;
     bool is_fast_scan = false;
     bool enable_del_clean_read = false;
+    bool enable_trim_minmax = false;
     UInt64 max_data_version = std::numeric_limits<UInt64>::max();
     // Rough set filter
     RSOperatorPtr rs_filter;
@@ -247,7 +247,7 @@ class DMFileBlockInputStreamBuilder
     DMFilePackFilterResultPtr pack_filter;
 
     VectorIndexCachePtr vector_index_cache;
-    // Note: Currently thie field is assigned only for Stable streams, not available for ColumnFileBig
+    // Note: Currently this field is assigned only for Stable streams, not available for ColumnFileBig
     std::optional<BitmapFilterView> bitmap_filter;
 
     // Note: column_cache_long_term is currently only filled when performing Vector Search.

dbms/src/Storages/DeltaMerge/File/DMFileBlockOutputStream.cpp

@@ -31,7 +31,9 @@ DMFileBlockOutputStream::DMFileBlockOutputStream(
                 context.getSettingsRef().dt_compression_method,
                 context.getSettingsRef().dt_compression_level),
             context.getSettingsRef().min_compress_block_size,
-            context.getSettingsRef().max_compress_block_size})
+            context.getSettingsRef().max_compress_block_size,
+            context.getSettingsRef().dt_enable_trim_minmax,
+        })
 {}
 
-} // namespace DB::DM
+} // namespace DB::DM

dbms/src/Storages/DeltaMerge/File/DMFileMeta.cpp

@@ -25,6 +25,7 @@
 namespace DB::ErrorCodes
 {
 extern const int BAD_ARGUMENTS;
+extern const int LOGICAL_ERROR;
 } // namespace DB::ErrorCodes
 
 namespace DB::DM
@@ -53,6 +54,10 @@ void DMFileMeta::initializeIndices()
     directory.list(sub_files);
     for (const auto & name : sub_files)
     {
+        // Skip trim min-max files. Their names also end with `.idx` (`*.trim.idx`),
+        // but the prefix is not a plain ColId and must not enter column_indices.
+        if (endsWith(name, details::TRIM_INDEX_FILE_SUFFIX))
+            continue;
         if (endsWith(name, details::INDEX_FILE_SUFFIX))
         {
             column_indices.insert(
@@ -405,6 +410,14 @@ UInt64 DMFileMeta::getFileSize(ColId col_id, const String & filename) const
 {
     auto itr = column_stats.find(col_id);
     RUNTIME_CHECK(itr != column_stats.end(), col_id);
+    // Trim index size is only available from MergedSubFileInfo, never from ColumnStat.
+    if (endsWith(filename, details::TRIM_INDEX_FILE_SUFFIX))
+    {
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "trim index size must be read from MergedSubFileInfo, filename={}",
+            filename);
+    }
     if (endsWith(filename, ".idx"))
     {
         return itr->second.index_bytes;

dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp

@@ -109,7 +109,7 @@ void DMFileMetaV2::parseColumnStat(std::string_view buffer)
         ColumnStat stat;
         stat.parseFromBuffer(rbuf);
         // Do not overwrite the ColumnStat if already exist, it may
-        // created by `ExteandColumnStat`
+        // created by `ExtendColumnStat`
         column_stats.emplace(stat.col_id, std::move(stat));
     }
 }
@@ -181,7 +181,7 @@ void DMFileMetaV2::finalize(
         writeSLPackStatToBuffer(tmp_buffer),
         writeSLPackPropertyToBuffer(tmp_buffer),
         writeExtendColumnStatToBuffer(tmp_buffer),
-        writeMergedSubFilePosotionsToBuffer(tmp_buffer),
+        writeMergedSubFilePositionsToBuffer(tmp_buffer),
     };
     writePODBinary(meta_block_handles, tmp_buffer);
     writeIntBinary(static_cast<UInt64>(meta_block_handles.size()), tmp_buffer);
@@ -255,7 +255,7 @@ DMFileMeta::BlockHandle DMFileMetaV2::writeExtendColumnStatToBuffer(WriteBuffer
     return BlockHandle{BlockType::ExtendColumnStat, offset, buffer.count() - offset};
 }
 
-DMFileMeta::BlockHandle DMFileMetaV2::writeMergedSubFilePosotionsToBuffer(WriteBuffer & buffer)
+DMFileMeta::BlockHandle DMFileMetaV2::writeMergedSubFilePositionsToBuffer(WriteBuffer & buffer)
 {
     auto offset = buffer.count();
 

dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.h

@@ -125,7 +125,7 @@ class DMFileMetaV2 : public DMFileMeta
     BlockHandle writeSLPackPropertyToBuffer(WriteBuffer & buffer) const;
     BlockHandle writeColumnStatToBuffer(WriteBuffer & buffer);
     BlockHandle writeExtendColumnStatToBuffer(WriteBuffer & buffer);
-    BlockHandle writeMergedSubFilePosotionsToBuffer(WriteBuffer & buffer);
+    BlockHandle writeMergedSubFilePositionsToBuffer(WriteBuffer & buffer);
 
     // read
     void parse(std::string_view buffer);