diff --git a/src/pyproject.toml b/src/pyproject.toml index 3df442e5e..0afc2911d 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -33,7 +33,7 @@ docs = [ ] test = ["pytest", "coverage", "pre-commit"] benchmark = ["requests"] -dev = ["valor-lite[nlp, openai, mistral, benchmark, test, docs]"] +dev = ["valor-lite[nlp, openai, mistral, benchmark, test, docs]", "pyarrow-stubs"] [tool.black] line-length = 79 diff --git a/src/valor_lite/cache/compute.py b/src/valor_lite/cache/compute.py index f7d828c32..1a1dbf682 100644 --- a/src/valor_lite/cache/compute.py +++ b/src/valor_lite/cache/compute.py @@ -1,9 +1,10 @@ import heapq import tempfile from pathlib import Path -from typing import Callable +from typing import Callable, Generator import pyarrow as pa +import pyarrow.compute as pc from valor_lite.cache.ephemeral import MemoryCacheReader, MemoryCacheWriter from valor_lite.cache.persistent import FileCacheReader, FileCacheWriter @@ -50,7 +51,9 @@ def create_sort_key( batch_iterators = [] batches = [] for batch_idx, batch_iter in enumerate( - intermediate_source.iterate_fragments(batch_size=batch_size) + intermediate_source.iterate_fragment_batch_iterators( + batch_size=batch_size + ) ): batch_iterators.append(batch_iter) batches.append(next(batch_iterators[batch_idx], None)) @@ -152,3 +155,57 @@ def sort( columns=columns, table_sort_override=table_sort_override, ) + + +def paginate_index( + source: MemoryCacheReader | FileCacheReader, + column_key: str, + modifier: pc.Expression | None = None, + limit: int | None = None, + offset: int = 0, +) -> Generator[pa.Table, None, None]: + """ + Iterate through a paginated cache reader. + + Note this function expects unqiue keys to be fragment-aligned. + """ + total = source.count_rows() + limit = limit if limit else total + + # pagination broader than data scope + if offset == 0 and limit >= total: + for tbl in source.iterate_tables(filter=modifier): + yield tbl + return + elif offset >= total: + return + + curr_idx = 0 + for tbl in source.iterate_tables(filter=modifier): + if tbl.num_rows == 0: + continue + + # sort the unique keys as they may be out of order + unique_values = pc.unique(tbl[column_key]).sort() # type: ignore[reportAttributeAccessIssue] + n_unique = len(unique_values) + prev_idx = curr_idx + curr_idx += n_unique + + # check for page overlap + if curr_idx <= offset: + continue + elif prev_idx >= (offset + limit): + return + + # apply any pagination conditions + condition = pc.scalar(True) + if prev_idx < offset and curr_idx > offset: + condition &= ( + pc.field(column_key) >= unique_values[offset - prev_idx] + ) + if prev_idx < (offset + limit) and curr_idx > (offset + limit): + condition &= ( + pc.field(column_key) < unique_values[offset + limit - prev_idx] + ) + + yield tbl.filter(condition) diff --git a/src/valor_lite/cache/ephemeral.py b/src/valor_lite/cache/ephemeral.py index 385b6b9de..92c4e5d0b 100644 --- a/src/valor_lite/cache/ephemeral.py +++ b/src/valor_lite/cache/ephemeral.py @@ -118,7 +118,7 @@ def iterate_tables_with_arrays( [tbl[col].to_numpy() for col in table_columns] ) - def iterate_fragments( + def iterate_fragment_batch_iterators( self, batch_size: int ) -> Iterator[Iterator[pa.RecordBatch]]: """ diff --git a/src/valor_lite/cache/persistent.py b/src/valor_lite/cache/persistent.py index 709401968..05356a990 100644 --- a/src/valor_lite/cache/persistent.py +++ b/src/valor_lite/cache/persistent.py @@ -154,6 +154,23 @@ def _retrieve(config: dict, key: str): compression=compression, ) + def iterate_ordered_fragments( + self, filter: pc.Expression | None = None + ) -> Iterator[ds.Fragment]: + """ + Iterate through ordered cache fragments. + """ + dataset = ds.dataset( + source=self._path, + schema=self._schema, + format="parquet", + ) + fragments = list(dataset.get_fragments(filter=filter)) + for fragment in sorted( + fragments, key=lambda x: int(Path(x.path).stem) + ): + yield fragment + def iterate_tables( self, columns: list[str] | None = None, @@ -173,12 +190,7 @@ def iterate_tables( ------- Iterator[pa.Table] """ - dataset = ds.dataset( - source=self._path, - schema=self._schema, - format="parquet", - ) - for fragment in dataset.get_fragments(): + for fragment in self.iterate_ordered_fragments(filter=filter): yield fragment.to_table(columns=columns, filter=filter) def iterate_arrays( @@ -243,7 +255,7 @@ def iterate_tables_with_arrays( [tbl[col].to_numpy() for col in table_columns] ) - def iterate_fragments( + def iterate_fragment_batch_iterators( self, batch_size: int ) -> Iterator[Iterator[pa.RecordBatch]]: """ @@ -258,12 +270,7 @@ def iterate_fragments( ------- Iterator[Iterator[pa.RecordBatch]] """ - dataset = ds.dataset( - source=self._path, - schema=self._schema, - format="parquet", - ) - for fragment in dataset.get_fragments(): + for fragment in self.iterate_ordered_fragments(): yield fragment.to_batches(batch_size=batch_size) diff --git a/src/valor_lite/classification/evaluator.py b/src/valor_lite/classification/evaluator.py index b84f97145..e2e1cc7f6 100644 --- a/src/valor_lite/classification/evaluator.py +++ b/src/valor_lite/classification/evaluator.py @@ -528,23 +528,6 @@ def iterate_values(self, datums: pc.Expression | None = None): matches = tbl["match"].to_numpy() yield ids, scores, winners, matches - def iterate_values_with_tables(self, datums: pc.Expression | None = None): - for tbl in self._reader.iterate_tables(filter=datums): - ids = np.column_stack( - [ - tbl[col].to_numpy() - for col in [ - "datum_id", - "gt_label_id", - "pd_label_id", - ] - ] - ) - scores = tbl["pd_score"].to_numpy() - winners = tbl["pd_winner"].to_numpy() - matches = tbl["match"].to_numpy() - yield ids, scores, winners, matches, tbl - def compute_rocauc(self) -> dict[MetricType, list[Metric]]: """ Compute ROCAUC. @@ -723,6 +706,8 @@ def compute_examples( score_thresholds: list[float] = [0.0], hardmax: bool = True, datums: pc.Expression | None = None, + limit: int | None = None, + offset: int = 0, ) -> list[Metric]: """ Compute examples per datum. @@ -737,6 +722,10 @@ def compute_examples( Toggles whether a hardmax is applied to predictions. datums : pyarrow.compute.Expression, optional Option to filter datums by an expression. + limit : int, optional + Option to set a limit to the number of returned datum examples. + offset : int, default=0 + Option to offset where examples are being created in the datum index. Returns ------- @@ -747,16 +736,29 @@ def compute_examples( raise ValueError("At least one score threshold must be passed.") metrics = [] - for ( - ids, - scores, - winners, - _, - tbl, - ) in self.iterate_values_with_tables(datums=datums): - if ids.size == 0: + for tbl in compute.paginate_index( + source=self._reader, + column_key="datum_id", + modifier=datums, + limit=limit, + offset=offset, + ): + if tbl.num_rows == 0: continue + ids = np.column_stack( + [ + tbl[col].to_numpy() + for col in [ + "datum_id", + "gt_label_id", + "pd_label_id", + ] + ] + ) + scores = tbl["pd_score"].to_numpy() + winners = tbl["pd_winner"].to_numpy() + # extract external identifiers index_to_datum_id = create_mapping( tbl, ids, 0, "datum_id", "datum_uid" @@ -829,16 +831,23 @@ def compute_confusion_matrix_with_examples( ) for score_idx, score_thresh in enumerate(score_thresholds) } - for ( - ids, - scores, - winners, - _, - tbl, - ) in self.iterate_values_with_tables(datums=datums): - if ids.size == 0: + for tbl in self._reader.iterate_tables(filter=datums): + if tbl.num_rows == 0: continue + ids = np.column_stack( + [ + tbl[col].to_numpy() + for col in [ + "datum_id", + "gt_label_id", + "pd_label_id", + ] + ] + ) + scores = tbl["pd_score"].to_numpy() + winners = tbl["pd_winner"].to_numpy() + # extract external identifiers index_to_datum_id = create_mapping( tbl, ids, 0, "datum_id", "datum_uid" diff --git a/src/valor_lite/object_detection/evaluator.py b/src/valor_lite/object_detection/evaluator.py index 798a6738c..e9d3d70bb 100644 --- a/src/valor_lite/object_detection/evaluator.py +++ b/src/valor_lite/object_detection/evaluator.py @@ -600,6 +600,8 @@ def compute_examples( iou_thresholds: list[float], score_thresholds: list[float], datums: pc.Expression | None = None, + limit: int | None = None, + offset: int = 0, ) -> list[Metric]: """ Computes examples at various thresholds. @@ -614,6 +616,10 @@ def compute_examples( A list of score thresholds to compute metrics over. datums : pyarrow.compute.Expression, optional Option to filter datums by an expression. + limit : int, optional + Option to set a limit to the number of returned datum examples. + offset : int, default=0 + Option to offset where examples are being created in the datum index. Returns ------- @@ -626,11 +632,6 @@ def compute_examples( raise ValueError("At least one score threshold must be passed.") metrics = [] - tbl_columns = [ - "datum_uid", - "gt_uid", - "pd_uid", - ] numeric_columns = [ "datum_id", "gt_id", @@ -640,14 +641,20 @@ def compute_examples( "iou", "pd_score", ] - for tbl, pairs in self._detailed_reader.iterate_tables_with_arrays( - columns=tbl_columns + numeric_columns, - numeric_columns=numeric_columns, - filter=datums, + for tbl in compute.paginate_index( + source=self._detailed_reader, + column_key="datum_id", + modifier=datums, + limit=limit, + offset=offset, ): - if pairs.size == 0: + if tbl.num_rows == 0: continue + pairs = np.column_stack( + [tbl[col].to_numpy() for col in numeric_columns] + ) + index_to_datum_id = {} index_to_groundtruth_id = {} index_to_prediction_id = {} diff --git a/tests/classification/test_examples.py b/tests/classification/test_examples.py index c4f9cf764..ac2c00aaf 100644 --- a/tests/classification/test_examples.py +++ b/tests/classification/test_examples.py @@ -681,3 +681,65 @@ def test_examples_without_hardmax_animal_example( assert m in expected_metrics for m in expected_metrics: assert m in actual_metrics + + +def test_examples_with_color_example_paginated( + loader: Loader, + classifications_color_example: list[Classification], +): + + loader.add_data(classifications_color_example) + evaluator = loader.finalize() + + actual_metrics = evaluator.compute_examples( + score_thresholds=[0.5], + limit=3, + offset=1, + ) + + actual_metrics = [m.to_dict() for m in actual_metrics] + expected_metrics = [ + { + "type": "Examples", + "value": { + "datum_id": "uid1", + "true_positives": [], + "false_positives": ["blue"], + "false_negatives": ["white"], + }, + "parameters": { + "score_threshold": 0.5, + "hardmax": True, + }, + }, + { + "type": "Examples", + "value": { + "datum_id": "uid2", + "true_positives": [], + "false_positives": [], + "false_negatives": ["red"], + }, + "parameters": { + "score_threshold": 0.5, + "hardmax": True, + }, + }, + { + "type": "Examples", + "value": { + "datum_id": "uid3", + "true_positives": [], + "false_positives": ["white"], + "false_negatives": ["blue"], + }, + "parameters": { + "score_threshold": 0.5, + "hardmax": True, + }, + }, + ] + for m in actual_metrics: + assert m in expected_metrics + for m in expected_metrics: + assert m in actual_metrics diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 9d9b77ff5..853e96b19 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -1,5 +1,7 @@ from pathlib import Path +from typing import Callable +import pyarrow as pa import pytest from valor_lite.cache.ephemeral import MemoryCacheWriter @@ -20,7 +22,9 @@ "in-memory_small_chunks", ], ) -def create_writer(request, tmp_path: Path): +def create_writer( + request, tmp_path: Path +) -> Callable[[pa.Schema], MemoryCacheWriter | FileCacheWriter]: file_type, batch_size, rows_per_file = request.param match file_type: case "memory": @@ -35,3 +39,5 @@ def create_writer(request, tmp_path: Path): batch_size=batch_size, rows_per_file=rows_per_file, ) + case unknown: + raise RuntimeError(unknown) diff --git a/tests/common/test_cache_pagination.py b/tests/common/test_cache_pagination.py new file mode 100644 index 000000000..8448cfe84 --- /dev/null +++ b/tests/common/test_cache_pagination.py @@ -0,0 +1,140 @@ +from typing import Callable + +import pyarrow as pa +import pyarrow.compute as pc +import pytest + +from valor_lite.cache import ( + FileCacheReader, + FileCacheWriter, + MemoryCacheReader, + MemoryCacheWriter, +) +from valor_lite.cache.compute import paginate_index + + +@pytest.fixture +def reader( + create_writer: Callable[[pa.Schema], MemoryCacheWriter | FileCacheWriter] +) -> MemoryCacheReader | FileCacheReader: + schema = pa.schema( + [ + ("a", pa.int64()), + ("b", pa.int64()), + ] + ) + writer = create_writer(schema) + writer.write_columns( + { + "a": list(range(100)), + "b": [i % 10 for i in range(100)], + } + ) + writer.flush() + return writer.to_reader() + + +def test_paginate_index_skipping(reader: MemoryCacheReader | FileCacheReader): + # offset = 0, limit = None, modifier = None + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=None, + limit=None, + offset=0, + ): + actual += tbl["a"].to_pylist() + assert actual == list(range(100)) + + # offset = 0, limit = 100, modifier = None + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=None, + limit=None, + offset=0, + ): + actual += tbl["a"].to_pylist() + assert actual == list(range(100)) + + +def test_paginate_index_offset(reader: MemoryCacheReader | FileCacheReader): + # limit = None, modifier = None + for offset in range(0, 101): + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=None, + limit=None, + offset=offset, + ): + actual += tbl["a"].to_pylist() + assert actual == list(range(offset, 100)) + + # offset = 1_000, limit=None, modifier = None + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=None, + limit=None, + offset=1_000, + ): + actual += tbl["a"].to_pylist() + assert actual == [] + + +def test_paginate_index_limit(reader: MemoryCacheReader | FileCacheReader): + # offset = 0, modifier = None + for limit in range(1, 101): + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=None, + limit=limit, + offset=0, + ): + actual += tbl["a"].to_pylist() + assert actual == list(range(0, limit)) + + +def test_pagination_index_modifer(reader: MemoryCacheReader | FileCacheReader): + # offset = 0, limit = None, modifer = 4 + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=pc.field("b") == 4, + limit=None, + offset=0, + ): + actual += tbl["a"].to_pylist() + assert actual == [4, 14, 24, 34, 44, 54, 64, 74, 84, 94] + + # offset = 5, limit = None, modifer = 4 + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=pc.field("b") == 4, + limit=None, + offset=5, + ): + actual += tbl["a"].to_pylist() + assert actual == [54, 64, 74, 84, 94] + + # offset = 5, limit = 2, modifer = 4 + actual = [] + for tbl in paginate_index( + reader, + column_key="a", + modifier=pc.field("b") == 4, + limit=2, + offset=5, + ): + actual += tbl["a"].to_pylist() + assert actual == [54, 64] diff --git a/tests/object_detection/conftest.py b/tests/object_detection/conftest.py index 00e94c912..c08b95cb0 100644 --- a/tests/object_detection/conftest.py +++ b/tests/object_detection/conftest.py @@ -377,7 +377,7 @@ def basic_detections( @pytest.fixture -def torchmetrics_detections(loader: Loader) -> Evaluator: +def torchmetrics_detections() -> list[Detection[BoundingBox]]: """Creates a model called "test_model" with some predicted detections on the dataset "test_dataset". These predictions are taken from a torchmetrics unit test (see test_metrics.py) @@ -520,7 +520,14 @@ def torchmetrics_detections(loader: Loader) -> Evaluator: ) for idx, (gt, pd) in enumerate(zip(groundtruths, predictions)) ] - loader.add_bounding_boxes(detections) + return detections + + +@pytest.fixture +def torchmetrics_evaluator( + loader: Loader, torchmetrics_detections: list[Detection[BoundingBox]] +) -> Evaluator: + loader.add_bounding_boxes(torchmetrics_detections) return loader.finalize() diff --git a/tests/object_detection/test_average_precision.py b/tests/object_detection/test_average_precision.py index 1938870c8..e39dc6bc8 100644 --- a/tests/object_detection/test_average_precision.py +++ b/tests/object_detection/test_average_precision.py @@ -249,12 +249,12 @@ def test_ap_basic_detections(basic_detections: Evaluator): assert m in actual_metrics -def test_ap_using_torch_metrics_example(torchmetrics_detections: Evaluator): +def test_ap_using_torch_metrics_example(torchmetrics_evaluator: Evaluator): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections + evaluator = torchmetrics_evaluator assert evaluator.info.number_of_datums == 4 assert evaluator.info.number_of_labels == 6 assert evaluator.info.number_of_groundtruth_annotations == 20 diff --git a/tests/object_detection/test_average_recall.py b/tests/object_detection/test_average_recall.py index f37ac076b..b36324050 100644 --- a/tests/object_detection/test_average_recall.py +++ b/tests/object_detection/test_average_recall.py @@ -205,12 +205,12 @@ def test_ar_metrics_second_class(basic_detections_second_class: Evaluator): assert m in actual_metrics -def test_ar_using_torch_metrics_example(torchmetrics_detections: Evaluator): +def test_ar_using_torch_metrics_example(torchmetrics_evaluator: Evaluator): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections + evaluator = torchmetrics_evaluator assert evaluator.info.number_of_datums == 4 assert evaluator.info.number_of_labels == 6 diff --git a/tests/object_detection/test_confusion_matrix.py b/tests/object_detection/test_confusion_matrix.py index 042e711cc..20cce53a1 100644 --- a/tests/object_detection/test_confusion_matrix.py +++ b/tests/object_detection/test_confusion_matrix.py @@ -322,13 +322,13 @@ def test_confusion_matrix(detections_for_detailed_counting: Evaluator): def test_confusion_matrix_using_torch_metrics_example( - torchmetrics_detections: Evaluator, + torchmetrics_evaluator: Evaluator, ): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections + evaluator = torchmetrics_evaluator assert evaluator.info.number_of_datums == 4 assert evaluator.info.number_of_labels == 6 assert evaluator.info.number_of_groundtruth_annotations == 20 diff --git a/tests/object_detection/test_confusion_matrix_with_examples.py b/tests/object_detection/test_confusion_matrix_with_examples.py index a2fe7ecf8..3e4658d93 100644 --- a/tests/object_detection/test_confusion_matrix_with_examples.py +++ b/tests/object_detection/test_confusion_matrix_with_examples.py @@ -838,13 +838,13 @@ def _filter_out_examples_and_zero_counts(cm: dict, hl: dict, mp: dict): def test_confusion_matrix_with_examples_using_torch_metrics_example( - torchmetrics_detections: Evaluator, + torchmetrics_evaluator: Evaluator, ): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections + evaluator = torchmetrics_evaluator assert evaluator.info.number_of_datums == 4 assert evaluator.info.number_of_labels == 6 assert evaluator.info.number_of_groundtruth_annotations == 20 diff --git a/tests/object_detection/test_evaluator.py b/tests/object_detection/test_evaluator.py index f6b3e1041..263d09fcb 100644 --- a/tests/object_detection/test_evaluator.py +++ b/tests/object_detection/test_evaluator.py @@ -4,7 +4,14 @@ import numpy as np import pytest -from valor_lite.object_detection import Evaluator, Metric, MetricType +from valor_lite.object_detection import ( + BoundingBox, + Detection, + Evaluator, + Loader, + Metric, + MetricType, +) def test_evaluator_file_not_found(tmp_path: Path): @@ -21,7 +28,7 @@ def test_evaluator_not_a_directory(tmp_path: Path): Evaluator.load(filepath) -def test_evaluator_valid_thresholds(tmp_path: Path): +def test_evaluator_valid_thresholds(): eval = Evaluator( detailed_reader=None, # type: ignore - testing ranked_reader=None, # type: ignore - testing @@ -41,17 +48,15 @@ def test_evaluator_valid_thresholds(tmp_path: Path): assert "score" in str(e) -def test_info_using_torch_metrics_example(torchmetrics_detections: Evaluator): +def test_info_using_torch_metrics_example(torchmetrics_evaluator: Evaluator): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections - - assert evaluator.info.number_of_datums == 4 - assert evaluator.info.number_of_labels == 6 - assert evaluator.info.number_of_groundtruth_annotations == 20 - assert evaluator.info.number_of_prediction_annotations == 19 + assert torchmetrics_evaluator.info.number_of_datums == 4 + assert torchmetrics_evaluator.info.number_of_labels == 6 + assert torchmetrics_evaluator.info.number_of_groundtruth_annotations == 20 + assert torchmetrics_evaluator.info.number_of_prediction_annotations == 19 def test_no_thresholds(detection_ranked_pair_ordering: Evaluator): @@ -177,3 +182,20 @@ def test_output_types_dont_contain_numpy(basic_detections: Evaluator): for value in values: if isinstance(value, (np.generic, np.ndarray)): raise TypeError(f"Value `{value}` has type `{type(value)}`.") + + +def test_evaluator_loading_using_torch_metrics_example( + tmp_path: Path, torchmetrics_detections: list[Detection[BoundingBox]] +): + loader = Loader.persistent(path=tmp_path) + loader.add_bounding_boxes(torchmetrics_detections) + original_evaluator = loader.finalize() + loaded_evaluator = Evaluator.load(path=tmp_path) + + kwargs = dict( + score_thresholds=[0.25, 0.5, 0.75, 0.9], + iou_thresholds=[0.1, 0.25, 0.5, 0.75], + ) + assert original_evaluator.compute_precision_recall( + **kwargs + ) == loaded_evaluator.compute_precision_recall(**kwargs) diff --git a/tests/object_detection/test_examples.py b/tests/object_detection/test_examples.py index 3d9b6ba34..67e5149a2 100644 --- a/tests/object_detection/test_examples.py +++ b/tests/object_detection/test_examples.py @@ -283,13 +283,13 @@ def test_examples(detections_for_detailed_counting: Evaluator): def test_examples_using_torch_metrics_example( - torchmetrics_detections: Evaluator, + torchmetrics_evaluator: Evaluator, ): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections + evaluator = torchmetrics_evaluator assert evaluator.info.number_of_datums == 4 assert evaluator.info.number_of_labels == 6 assert evaluator.info.number_of_groundtruth_annotations == 20 @@ -1076,3 +1076,314 @@ def test_examples_ranked_pair_ordering( assert m in expected_metrics for m in expected_metrics: assert m in actual_metrics + + +def test_examples_using_torch_metrics_example_paginated( + torchmetrics_evaluator: Evaluator, +): + """ + cf with torch metrics/pycocotools results listed here: + https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 + """ + evaluator = torchmetrics_evaluator + assert evaluator.info.number_of_datums == 4 + assert evaluator.info.number_of_labels == 6 + assert evaluator.info.number_of_groundtruth_annotations == 20 + assert evaluator.info.number_of_prediction_annotations == 19 + + iou_thresholds = [0.5, 0.9] + score_thresholds = [0.05, 0.25, 0.35, 0.55, 0.75, 0.8, 0.85, 0.95] + actual_metrics = evaluator.compute_examples( + iou_thresholds=iou_thresholds, + score_thresholds=score_thresholds, + offset=1, + limit=2, + ) + + actual_metrics = [m.to_dict() for m in actual_metrics] + expected_metrics = [ + # datum 1 + *[ + { + "type": "Examples", + "value": { + "datum_id": "1", + "true_positives": [("uid_1_gt_1", "uid_1_pd_1")], + "false_positives": ["uid_1_pd_0"], + "false_negatives": ["uid_1_gt_0"], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": iou_thresh, + }, + } + for iou_thresh in iou_thresholds + for score_thresh in score_thresholds[:2] + ], + *[ + { + "type": "Examples", + "value": { + "datum_id": "1", + "true_positives": [("uid_1_gt_1", "uid_1_pd_1")], + "false_positives": [], + "false_negatives": ["uid_1_gt_0"], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": iou_thresh, + }, + } + for iou_thresh in iou_thresholds + for score_thresh in score_thresholds[2:4] + ], + *[ + { + "type": "Examples", + "value": { + "datum_id": "1", + "true_positives": [], + "false_positives": [], + "false_negatives": ["uid_1_gt_0", "uid_1_gt_1"], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": iou_thresh, + }, + } + for iou_thresh in iou_thresholds + for score_thresh in score_thresholds[4:] + ], + # datum 2 + *[ + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [ + ("uid_2_gt_0", "uid_2_pd_0"), + ("uid_2_gt_1", "uid_2_pd_1"), + ("uid_2_gt_2", "uid_2_pd_2"), + ("uid_2_gt_3", "uid_2_pd_3"), + ("uid_2_gt_4", "uid_2_pd_4"), + ("uid_2_gt_5", "uid_2_pd_5"), + ("uid_2_gt_6", "uid_2_pd_6"), + ], + "false_positives": [], + "false_negatives": [], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": 0.5, + }, + } + for score_thresh in score_thresholds[:2] + ], + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [ + ("uid_2_gt_0", "uid_2_pd_0"), + ("uid_2_gt_2", "uid_2_pd_2"), + ("uid_2_gt_3", "uid_2_pd_3"), + ("uid_2_gt_5", "uid_2_pd_5"), + ("uid_2_gt_6", "uid_2_pd_6"), + ], + "false_positives": [], + "false_negatives": ["uid_2_gt_1", "uid_2_gt_4"], + }, + "parameters": {"score_threshold": 0.35, "iou_threshold": 0.5}, + }, + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [ + ("uid_2_gt_3", "uid_2_pd_3"), + ("uid_2_gt_5", "uid_2_pd_5"), + ("uid_2_gt_6", "uid_2_pd_6"), + ], + "false_positives": [], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_4", + ], + }, + "parameters": {"score_threshold": 0.55, "iou_threshold": 0.5}, + }, + *[ + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [ + ("uid_2_gt_5", "uid_2_pd_5"), + ("uid_2_gt_6", "uid_2_pd_6"), + ], + "false_positives": [], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_4", + ], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": 0.5, + }, + } + for score_thresh in score_thresholds[4:6] + ], + *[ + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [("uid_2_gt_6", "uid_2_pd_6")], + "false_positives": [], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_4", + "uid_2_gt_5", + ], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": 0.5, + }, + } + for score_thresh in score_thresholds[6:] + ], + *[ + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [("uid_2_gt_4", "uid_2_pd_4")], + "false_positives": [ + "uid_2_pd_0", + "uid_2_pd_1", + "uid_2_pd_2", + "uid_2_pd_3", + "uid_2_pd_5", + "uid_2_pd_6", + ], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_5", + "uid_2_gt_6", + ], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": 0.9, + }, + } + for score_thresh in score_thresholds[:2] + ], + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [], + "false_positives": [ + "uid_2_pd_0", + "uid_2_pd_2", + "uid_2_pd_3", + "uid_2_pd_5", + "uid_2_pd_6", + ], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_4", + "uid_2_gt_5", + "uid_2_gt_6", + ], + }, + "parameters": {"score_threshold": 0.35, "iou_threshold": 0.9}, + }, + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [], + "false_positives": ["uid_2_pd_3", "uid_2_pd_5", "uid_2_pd_6"], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_4", + "uid_2_gt_5", + "uid_2_gt_6", + ], + }, + "parameters": {"score_threshold": 0.55, "iou_threshold": 0.9}, + }, + *[ + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [], + "false_positives": ["uid_2_pd_5", "uid_2_pd_6"], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_4", + "uid_2_gt_5", + "uid_2_gt_6", + ], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": 0.9, + }, + } + for score_thresh in [0.75, 0.8] + ], + *[ + { + "type": "Examples", + "value": { + "datum_id": "2", + "true_positives": [], + "false_positives": ["uid_2_pd_6"], + "false_negatives": [ + "uid_2_gt_0", + "uid_2_gt_1", + "uid_2_gt_2", + "uid_2_gt_3", + "uid_2_gt_4", + "uid_2_gt_5", + "uid_2_gt_6", + ], + }, + "parameters": { + "score_threshold": score_thresh, + "iou_threshold": 0.9, + }, + } + for score_thresh in [0.85, 0.95] + ], + ] + for m in actual_metrics: + assert m in expected_metrics + for m in expected_metrics: + assert m in actual_metrics diff --git a/tests/object_detection/test_pr_curve.py b/tests/object_detection/test_pr_curve.py index 881da7320..077084c16 100644 --- a/tests/object_detection/test_pr_curve.py +++ b/tests/object_detection/test_pr_curve.py @@ -2,13 +2,13 @@ def test_pr_curve_using_torch_metrics_example( - torchmetrics_detections: Evaluator, + torchmetrics_evaluator: Evaluator, ): """ cf with torch metrics/pycocotools results listed here: https://github.com/Lightning-AI/metrics/blob/107dbfd5fb158b7ae6d76281df44bd94c836bfce/tests/unittests/detection/test_map.py#L231 """ - evaluator = torchmetrics_detections + evaluator = torchmetrics_evaluator assert evaluator.info.number_of_datums == 4 assert evaluator.info.number_of_labels == 6 assert evaluator.info.number_of_groundtruth_annotations == 20