code coverage

Author: czaloom

src/valor_lite/cache/compute.py

@@ -167,7 +167,7 @@ def paginate_index(
     """
     Iterate through a paginated cache reader.
 
-    Note this function expects unqiue keys to be fragment-aligned and in ascending order.
+    Note this function expects unqiue keys to be fragment-aligned.
     """
     total = source.count_rows()
     limit = limit if limit else total
@@ -185,6 +185,7 @@ def paginate_index(
         if tbl.num_rows == 0:
             continue
 
+        # sort the unique keys as they may be out of order
         unique_values = pc.unique(tbl[column_key]).sort()  # type: ignore[reportAttributeAccessIssue]
         n_unique = len(unique_values)
         prev_idx = curr_idx

tests/object_detection/conftest.py

@@ -377,7 +377,7 @@ def basic_detections(
 
 
 @pytest.fixture
-def torchmetrics_detections(loader: Loader) -> Evaluator:
+def torchmetrics_detections() -> list[Detection[BoundingBox]]:
     """Creates a model called "test_model" with some predicted
     detections on the dataset "test_dataset". These predictions are taken
     from a torchmetrics unit test (see test_metrics.py)
@@ -520,7 +520,14 @@ def torchmetrics_detections(loader: Loader) -> Evaluator:
         )
         for idx, (gt, pd) in enumerate(zip(groundtruths, predictions))
     ]
-    loader.add_bounding_boxes(detections)
+    return detections
+
+
+@pytest.fixture
+def torchmetrics_evaluator(
+    loader: Loader, torchmetrics_detections: list[Detection[BoundingBox]]
+) -> Evaluator:
+    loader.add_bounding_boxes(torchmetrics_detections)
     return loader.finalize()
 
 

tests/object_detection/test_average_precision.py

@@ -249,12 +249,12 @@ def test_ap_basic_detections(basic_detections: Evaluator):
         assert m in actual_metrics
 
 
-def test_ap_using_torch_metrics_example(torchmetrics_detections: Evaluator):
+def test_ap_using_torch_metrics_example(torchmetrics_evaluator: Evaluator):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6
     assert evaluator.info.number_of_groundtruth_annotations == 20

tests/object_detection/test_average_recall.py

@@ -205,12 +205,12 @@ def test_ar_metrics_second_class(basic_detections_second_class: Evaluator):
         assert m in actual_metrics
 
 
-def test_ar_using_torch_metrics_example(torchmetrics_detections: Evaluator):
+def test_ar_using_torch_metrics_example(torchmetrics_evaluator: Evaluator):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
 
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6

tests/object_detection/test_confusion_matrix.py

@@ -322,13 +322,13 @@ def test_confusion_matrix(detections_for_detailed_counting: Evaluator):
 
 
 def test_confusion_matrix_using_torch_metrics_example(
-    torchmetrics_detections: Evaluator,
+    torchmetrics_evaluator: Evaluator,
 ):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6
     assert evaluator.info.number_of_groundtruth_annotations == 20

tests/object_detection/test_confusion_matrix_with_examples.py

@@ -838,13 +838,13 @@ def _filter_out_examples_and_zero_counts(cm: dict, hl: dict, mp: dict):
 
 
 def test_confusion_matrix_with_examples_using_torch_metrics_example(
-    torchmetrics_detections: Evaluator,
+    torchmetrics_evaluator: Evaluator,
 ):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6
     assert evaluator.info.number_of_groundtruth_annotations == 20

tests/object_detection/test_evaluator.py

@@ -4,7 +4,14 @@
 import numpy as np
 import pytest
 
-from valor_lite.object_detection import Evaluator, Metric, MetricType
+from valor_lite.object_detection import (
+    BoundingBox,
+    Detection,
+    Evaluator,
+    Loader,
+    Metric,
+    MetricType,
+)
 
 
 def test_evaluator_file_not_found(tmp_path: Path):
@@ -21,7 +28,7 @@ def test_evaluator_not_a_directory(tmp_path: Path):
         Evaluator.load(filepath)
 
 
-def test_evaluator_valid_thresholds(tmp_path: Path):
+def test_evaluator_valid_thresholds():
     eval = Evaluator(
         detailed_reader=None,  # type: ignore - testing
         ranked_reader=None,  # type: ignore - testing
@@ -41,17 +48,15 @@ def test_evaluator_valid_thresholds(tmp_path: Path):
         assert "score" in str(e)
 
 
-def test_info_using_torch_metrics_example(torchmetrics_detections: Evaluator):
+def test_info_using_torch_metrics_example(torchmetrics_evaluator: Evaluator):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
-
-    assert evaluator.info.number_of_datums == 4
-    assert evaluator.info.number_of_labels == 6
-    assert evaluator.info.number_of_groundtruth_annotations == 20
-    assert evaluator.info.number_of_prediction_annotations == 19
+    assert torchmetrics_evaluator.info.number_of_datums == 4
+    assert torchmetrics_evaluator.info.number_of_labels == 6
+    assert torchmetrics_evaluator.info.number_of_groundtruth_annotations == 20
+    assert torchmetrics_evaluator.info.number_of_prediction_annotations == 19
 
 
 def test_no_thresholds(detection_ranked_pair_ordering: Evaluator):
@@ -177,3 +182,20 @@ def test_output_types_dont_contain_numpy(basic_detections: Evaluator):
     for value in values:
         if isinstance(value, (np.generic, np.ndarray)):
             raise TypeError(f"Value `{value}` has type `{type(value)}`.")
+
+
+def test_evaluator_loading_using_torch_metrics_example(
+    tmp_path: Path, torchmetrics_detections: list[Detection[BoundingBox]]
+):
+    loader = Loader.persistent(path=tmp_path)
+    loader.add_bounding_boxes(torchmetrics_detections)
+    original_evaluator = loader.finalize()
+    loaded_evaluator = Evaluator.load(path=tmp_path)
+
+    kwargs = dict(
+        score_thresholds=[0.25, 0.5, 0.75, 0.9],
+        iou_thresholds=[0.1, 0.25, 0.5, 0.75],
+    )
+    assert original_evaluator.compute_precision_recall(
+        **kwargs
+    ) == loaded_evaluator.compute_precision_recall(**kwargs)

tests/object_detection/test_examples.py

@@ -283,13 +283,13 @@ def test_examples(detections_for_detailed_counting: Evaluator):
 
 
 def test_examples_using_torch_metrics_example(
-    torchmetrics_detections: Evaluator,
+    torchmetrics_evaluator: Evaluator,
 ):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6
     assert evaluator.info.number_of_groundtruth_annotations == 20
@@ -1079,13 +1079,13 @@ def test_examples_ranked_pair_ordering(
 
 
 def test_examples_using_torch_metrics_example_paginated(
-    torchmetrics_detections: Evaluator,
+    torchmetrics_evaluator: Evaluator,
 ):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6
     assert evaluator.info.number_of_groundtruth_annotations == 20

tests/object_detection/test_pr_curve.py

@@ -2,13 +2,13 @@
 
 
 def test_pr_curve_using_torch_metrics_example(
-    torchmetrics_detections: Evaluator,
+    torchmetrics_evaluator: Evaluator,
 ):
     """
     cf with torch metrics/pycocotools results listed here:
     https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231
     """
-    evaluator = torchmetrics_detections
+    evaluator = torchmetrics_evaluator
     assert evaluator.info.number_of_datums == 4
     assert evaluator.info.number_of_labels == 6
     assert evaluator.info.number_of_groundtruth_annotations == 20