Keep integers on the python read path for fixed-shape ArrayXD columns with nulls

src/datasets/features/features.py

@@ -835,6 +835,13 @@ def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None
         numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only)
         if self.type.shape[0] is None and numpy_arr.dtype == object:
             return [arr.tolist() for arr in numpy_arr.tolist()]
+        elif self.type.shape[0] is not None and self.storage.null_count:
+            # For a fixed-shape array, to_numpy casts the whole column to float64 to
+            # hold np.nan in the null rows. On the python read path that silently
+            # corrupts every non-null value: integers become floats and values above
+            # 2**53 lose precision. The nested-list storage already carries the exact
+            # values and None for the null rows, so build the list from it directly.
+            return self.storage.to_pylist()
         else:
             return numpy_arr.tolist()
 

tests/features/test_array_xd.py

@@ -397,6 +397,28 @@ def test_array_xd_with_none():
     assert np.isnan(arr[1]) and np.isnan(arr[3])  # a single np.nan value - np.all not needed
 
 
+def test_array_xd_with_none_python_format_keeps_integers():
+    # Fixed-shape integer column with a null row: the python read path (to_dict /
+    # to_pylist) must keep the non-null values as integers and surface the null as
+    # None, not cast the whole column to float to make room for np.nan.
+    features = datasets.Features({"foo": datasets.Array2D(dtype="int64", shape=(1, 2))})
+    dataset = datasets.Dataset.from_dict({"foo": [[[10, 20]], [[30, 40]], None]}, features=features)
+    assert dataset.to_dict()["foo"] == [[[10, 20]], [[30, 40]], None]
+    assert all(isinstance(v, int) for row in (dataset[0]["foo"], dataset[1]["foo"]) for v in row[0])
+
+    # Integers above 2**53 are not representable as float64, so the old float cast
+    # silently altered them; the python path must round-trip them exactly.
+    big = 9007199254740993  # 2**53 + 1
+    features = datasets.Features({"foo": datasets.Array2D(dtype="int64", shape=(1, 1))})
+    dataset = datasets.Dataset.from_dict({"foo": [[[big]], None]}, features=features)
+    assert dataset.to_dict()["foo"] == [[[big]], None]
+
+    # The numpy format is unchanged: a fixed-shape column keeps its dense float64 +
+    # np.nan representation (numpy cannot hold both integers and a missing marker).
+    arr = NumpyArrowExtractor().extract_column(dataset._data)
+    assert arr.dtype == np.float64 and np.isnan(arr[1]).all()
+
+
 @pytest.mark.parametrize("seq_type", ["no_sequence", "sequence", "sequence_of_sequence"])
 @pytest.mark.parametrize(
     "dtype",