Fix TS_2DIFF float/double page compatibility

cpp/CMakeLists.txt

@@ -105,18 +105,37 @@ else ()
 endif ()
 
 message("CMAKE BUILD TYPE " ${CMAKE_BUILD_TYPE})
-# MSVC provides sensible per-configuration optimization flags by default; the
-# GCC-style flags below would be rejected by cl.exe, so skip them on MSVC.
-if (NOT MSVC)
-    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-        set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
-    elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
-        set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2")
-    elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
-        set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O2 -g")
-    elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
-        set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -ffunction-sections -fdata-sections -Os")
-        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
+# Keep optimization policy external by default (caller/toolchain/CMake defaults).
+set(TSFILE_OPTIMIZATION_FLAGS ""
+        CACHE STRING
+        "Optional extra optimization flags for tsfile-cpp (e.g. -O3). Empty means inherit caller defaults.")
+if (TSFILE_OPTIMIZATION_FLAGS)
+    # Apply after CMake defaults for each config so explicit optimization can
+    # override default -O flags in Release/RelWithDebInfo/Debug/MinSizeRel.
+    set(CMAKE_CXX_FLAGS_DEBUG
+            "${CMAKE_CXX_FLAGS_DEBUG} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS_RELEASE
+            "${CMAKE_CXX_FLAGS_RELEASE} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
+            "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS_MINSIZEREL
+            "${CMAKE_CXX_FLAGS_MINSIZEREL} ${TSFILE_OPTIMIZATION_FLAGS}")
+    message("cmake using: TSFILE_OPTIMIZATION_FLAGS=${TSFILE_OPTIMIZATION_FLAGS}")
+else ()
+    message("cmake using: TSFILE_OPTIMIZATION_FLAGS=<inherit>")
+    # MSVC provides sensible per-configuration optimization flags by default; the
+    # GCC-style flags below would be rejected by cl.exe, so skip them on MSVC.
+    if (NOT MSVC)
+        if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+            set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
+        elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
+            set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2")
+        elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+            set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O2 -g")
+        elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
+            set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -ffunction-sections -fdata-sections -Os")
+            set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
+        endif ()
     endif ()
 endif ()
 message("CMAKE DEBUG: CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
@@ -241,7 +260,6 @@ else ()
     set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11")
 endif ()
 add_subdirectory(third_party)
-set(CMAKE_CXX_FLAGS "${SAVED_CXX_FLAGS}")
 
 add_subdirectory(src)
 if (BUILD_TEST)

cpp/examples/CMakeLists.txt

@@ -22,6 +22,7 @@ message("Running in examples directory")
 
 if (NOT MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
 endif ()
 
 # TsFile include dir
@@ -39,13 +40,20 @@ include_directories(${PROJECT_SOURCE_DIR}/../third_party/antlr4-cpp-runtime-4/ru
 set(BUILD_TYPE "Release")
 include_directories(${SDK_INCLUDE_DIR})
 
-if (NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g")
+if (DEFINED TSFILE_OPTIMIZATION_FLAGS AND NOT "${TSFILE_OPTIMIZATION_FLAGS}" STREQUAL "")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TSFILE_OPTIMIZATION_FLAGS}")
+    message("examples using: TSFILE_OPTIMIZATION_FLAGS=${TSFILE_OPTIMIZATION_FLAGS}")
+else ()
+    message("examples using: TSFILE_OPTIMIZATION_FLAGS=<inherit>")
+    if (NOT MSVC)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g")
+    endif ()
 endif ()
 
 add_subdirectory(cpp_examples)
 add_subdirectory(c_examples)
 
 add_executable(examples examples.cc)
 target_link_libraries(examples cpp_examples_obj c_examples_obj)
-target_link_libraries(examples tsfile)
+target_link_libraries(examples tsfile)

cpp/examples/README.md

@@ -55,6 +55,14 @@ target_link_libraries(your_target ${TSFILE_LIB})
 
 Note: Set ${SDK_LIB} to your TSFile library directory.
 
+### Optional Optimization Control
+
+By default, `tsfile-cpp` inherits optimization settings from the caller/toolchain.
+If you want to override optimization for `tsfile-cpp`, pass
+`TSFILE_OPTIMIZATION_FLAGS` during configure:
+
+Leave `TSFILE_OPTIMIZATION_FLAGS` empty to keep inherited behavior.
+
 ## 3. Implementation Examples
 
 ### Directory Structure

cpp/src/common/allocator/byte_stream.h

@@ -347,6 +347,21 @@ class ByteStream {
 
     FORCE_INLINE uint32_t total_size() const { return total_size_.load(); }
     FORCE_INLINE uint32_t read_pos() const { return read_pos_; };
+    /**
+     * Seek the read cursor to an absolute offset. Re-anchors read_page_ for
+     * multi-page streams.
+     */
+    void set_read_pos(uint32_t pos) {
+        ASSERT(pos <= total_size());
+        read_pos_ = pos;
+        Page* p = head_.load();
+        uint32_t skipped = 0;
+        while (p != nullptr && skipped + page_size_ <= pos) {
+            skipped += page_size_;
+            p = p->next_.load();
+        }
+        read_page_ = p;
+    }
     FORCE_INLINE void wrapped_buf_advance_read_pos(uint32_t size) {
         if (size + read_pos_ > total_size_.load()) {
             read_pos_ = total_size_.load();

cpp/src/encoding/ts2diff_decoder.h

@@ -22,14 +22,116 @@
 
 #include <sys/types.h>
 
+#include <cmath>
 #include <cstddef>
+#include <vector>
 
 #include "common/allocator/alloc_base.h"
 #include "common/allocator/byte_stream.h"
 #include "decoder.h"
 #include "utils/util_define.h"
 
 namespace storage {
+
+namespace ts2diff_java_detail {
+
+// Java float/double TS_2DIFF overflow page markers.
+constexpr uint32_t FLAG_ORIGINAL_VALUE_OVERFLOW =
+    2147483646u;  // Integer.MAX_VALUE - 1
+constexpr uint32_t FLAG_SCALED_VALUE_OVERFLOW =
+    2147483647u;  // Integer.MAX_VALUE
+
+inline bool bitmap_marked(const std::vector<uint8_t>& bm, int idx) {
+    if (bm.empty()) {
+        return false;
+    }
+    size_t byte_idx = static_cast<size_t>(idx / 8);
+    if (byte_idx >= bm.size()) {
+        return false;
+    }
+    return (bm[byte_idx] & static_cast<uint8_t>(1u << (idx % 8))) != 0;
+}
+
+inline bool looks_like_ts2diff_header(common::ByteStream& in) {
+    int ret = common::E_OK;
+    uint32_t probe_mark = in.read_pos();
+    int32_t write_index = 0;
+    int32_t bit_width = 0;
+    if (RET_FAIL(common::SerializationUtil::read_i32(write_index, in)) ||
+        RET_FAIL(common::SerializationUtil::read_i32(bit_width, in))) {
+        in.set_read_pos(probe_mark);
+        return false;
+    }
+    in.set_read_pos(probe_mark);
+    if (write_index < 0 || write_index > 128) {
+        return false;
+    }
+    if (bit_width < 0 || bit_width > 64) {
+        return false;
+    }
+    return true;
+}
+
+inline int consume_float_double_ts2diff_prefix(
+    common::ByteStream& in, bool& is_legacy_raw, int& max_point_number,
+    std::vector<uint8_t>& underflow_bm, std::vector<uint8_t>& overflow_bm,
+    int& segment_size) {
+    int ret = common::E_OK;
+    is_legacy_raw = false;
+    max_point_number = 0;
+    underflow_bm.clear();
+    overflow_bm.clear();
+    segment_size = 0;
+    uint32_t mark = in.read_pos();
+    uint32_t tag = 0;
+    if (RET_FAIL(common::SerializationUtil::read_var_uint(tag, in))) {
+        return ret;
+    }
+    if (tag == FLAG_ORIGINAL_VALUE_OVERFLOW ||
+        tag == FLAG_SCALED_VALUE_OVERFLOW) {
+        uint32_t n = 0;
+        if (RET_FAIL(common::SerializationUtil::read_var_uint(n, in))) {
+            return ret;
+        }
+        segment_size = static_cast<int>(n);
+        int bm_len = segment_size / 8 + 1;
+        underflow_bm.resize(static_cast<size_t>(bm_len), 0);
+        uint32_t read_len = 0;
+        if (RET_FAIL(in.read_buf(underflow_bm.data(),
+                                 static_cast<uint32_t>(bm_len), read_len)) ||
+            read_len != static_cast<uint32_t>(bm_len)) {
+            return ret;
+        }
+        if (tag == FLAG_ORIGINAL_VALUE_OVERFLOW) {
+            overflow_bm.resize(static_cast<size_t>(bm_len), 0);
+            if (RET_FAIL(in.read_buf(overflow_bm.data(),
+                                     static_cast<uint32_t>(bm_len),
+                                     read_len)) ||
+                read_len != static_cast<uint32_t>(bm_len)) {
+                return ret;
+            }
+        }
+        uint32_t mpn = 0;
+        if (RET_FAIL(common::SerializationUtil::read_var_uint(mpn, in))) {
+            return ret;
+        }
+        max_point_number = static_cast<int>(mpn);
+        return common::E_OK;
+    }
+
+    // Distinguish Java maxPointNumber prefix from legacy raw C++ block.
+    max_point_number = static_cast<int>(tag);
+    if (!looks_like_ts2diff_header(in)) {
+        in.set_read_pos(mark);
+        is_legacy_raw = true;
+    } else {
+        segment_size = 0;
+    }
+    return common::E_OK;
+}
+
+}  // namespace ts2diff_java_detail
+
 template <typename T>
 class TS2DIFFDecoder : public Decoder {
    public:
@@ -174,6 +276,7 @@ inline int64_t TS2DIFFDecoder<int64_t>::decode(common::ByteStream& in) {
 
 class FloatTS2DIFFDecoder : public TS2DIFFDecoder<int32_t> {
    public:
+    FloatTS2DIFFDecoder() = default;
     float decode(common::ByteStream& in) {
         int32_t value_int = TS2DIFFDecoder<int32_t>::decode(in);
         return common::int_to_float(value_int);
@@ -184,10 +287,20 @@ class FloatTS2DIFFDecoder : public TS2DIFFDecoder<int32_t> {
     int read_int64(int64_t& ret_value, common::ByteStream& in);
     int read_float(float& ret_value, common::ByteStream& in);
     int read_double(double& ret_value, common::ByteStream& in);
+
+   private:
+    bool is_legacy_raw_{false};
+    int max_point_number_{0};
+    double max_point_value_{1.0};
+    int segment_pos_{0};
+    int segment_size_{0};
+    std::vector<uint8_t> underflow_bm_;
+    std::vector<uint8_t> overflow_bm_;
 };
 
 class DoubleTS2DIFFDecoder : public TS2DIFFDecoder<int64_t> {
    public:
+    DoubleTS2DIFFDecoder() = default;
     double decode(common::ByteStream& in) {
         int64_t value_long = TS2DIFFDecoder<int64_t>::decode(in);
         return common::long_to_double(value_long);
@@ -198,6 +311,15 @@ class DoubleTS2DIFFDecoder : public TS2DIFFDecoder<int64_t> {
     int read_int64(int64_t& ret_value, common::ByteStream& in);
     int read_float(float& ret_value, common::ByteStream& in);
     int read_double(double& ret_value, common::ByteStream& in);
+
+   private:
+    bool is_legacy_raw_{false};
+    int max_point_number_{0};
+    double max_point_value_{1.0};
+    int segment_pos_{0};
+    int segment_size_{0};
+    std::vector<uint8_t> underflow_bm_;
+    std::vector<uint8_t> overflow_bm_;
 };
 
 typedef TS2DIFFDecoder<int32_t> IntTS2DIFFDecoder;
@@ -295,7 +417,38 @@ FORCE_INLINE int FloatTS2DIFFDecoder::read_int64(int64_t& ret_value,
 }
 FORCE_INLINE int FloatTS2DIFFDecoder::read_float(float& ret_value,
                                                  common::ByteStream& in) {
-    ret_value = decode(in);
+    int ret = common::E_OK;
+    if (current_index_ == 0) {
+        if (RET_FAIL(ts2diff_java_detail::consume_float_double_ts2diff_prefix(
+                in, is_legacy_raw_, max_point_number_, underflow_bm_,
+                overflow_bm_, segment_size_))) {
+            return ret;
+        }
+        max_point_value_ =
+            max_point_number_ <= 0
+                ? 1.0
+                : std::pow(10.0, static_cast<double>(max_point_number_));
+        segment_pos_ = 0;
+    }
+    if (is_legacy_raw_) {
+        ret_value = decode(in);
+        return common::E_OK;
+    }
+    int32_t value_int = TS2DIFFDecoder<int32_t>::decode(in);
+    if (!overflow_bm_.empty() &&
+        ts2diff_java_detail::bitmap_marked(overflow_bm_, segment_pos_)) {
+        ret_value = common::int_to_float(value_int);
+    } else {
+        bool use_scaled = true;
+        if (!underflow_bm_.empty()) {
+            use_scaled =
+                ts2diff_java_detail::bitmap_marked(underflow_bm_, segment_pos_);
+        }
+        const double divisor = use_scaled ? max_point_value_ : 1.0;
+        ret_value =
+            static_cast<float>(static_cast<double>(value_int) / divisor);
+    }
+    segment_pos_++;
     return common::E_OK;
 }
 FORCE_INLINE int FloatTS2DIFFDecoder::read_double(double& ret_value,
@@ -325,7 +478,37 @@ FORCE_INLINE int DoubleTS2DIFFDecoder::read_float(float& ret_value,
 }
 FORCE_INLINE int DoubleTS2DIFFDecoder::read_double(double& ret_value,
                                                    common::ByteStream& in) {
-    ret_value = decode(in);
+    int ret = common::E_OK;
+    if (current_index_ == 0) {
+        if (RET_FAIL(ts2diff_java_detail::consume_float_double_ts2diff_prefix(
+                in, is_legacy_raw_, max_point_number_, underflow_bm_,
+                overflow_bm_, segment_size_))) {
+            return ret;
+        }
+        max_point_value_ =
+            max_point_number_ <= 0
+                ? 1.0
+                : std::pow(10.0, static_cast<double>(max_point_number_));
+        segment_pos_ = 0;
+    }
+    if (is_legacy_raw_) {
+        ret_value = decode(in);
+        return common::E_OK;
+    }
+    int64_t value_long = TS2DIFFDecoder<int64_t>::decode(in);
+    if (!overflow_bm_.empty() &&
+        ts2diff_java_detail::bitmap_marked(overflow_bm_, segment_pos_)) {
+        ret_value = common::long_to_double(value_long);
+    } else {
+        bool use_scaled = true;
+        if (!underflow_bm_.empty()) {
+            use_scaled =
+                ts2diff_java_detail::bitmap_marked(underflow_bm_, segment_pos_);
+        }
+        const double divisor = use_scaled ? max_point_value_ : 1.0;
+        ret_value = static_cast<double>(value_long) / divisor;
+    }
+    segment_pos_++;
     return common::E_OK;
 }