add better exception handling for cachce building and filtering

Author: czaloom

src/valor_lite/classification/computation.py

@@ -230,6 +230,26 @@ def compute_precision_recall_rocauc(
     n_labels = label_metadata.shape[0]
     n_scores = score_thresholds.shape[0]
 
+    # allocate results
+    rocauc = np.zeros(n_labels, dtype=np.float64)
+    mean_rocauc = 0.0
+    counts = np.zeros((n_scores, n_labels, 4), dtype=np.int32)
+    recall = np.zeros((n_scores, n_labels), dtype=np.float64)
+    precision = np.zeros_like(recall)
+    accuracy = np.zeros(n_scores, dtype=np.float64)
+    f1_score = np.zeros_like(recall)
+
+    if detailed_pairs.size == 0:
+        return (
+            counts,
+            precision,
+            recall,
+            accuracy,
+            f1_score,
+            rocauc,
+            mean_rocauc,
+        )
+
     pd_labels = detailed_pairs[:, 2].astype(int)
 
     mask_matching_labels = np.isclose(
@@ -249,7 +269,6 @@ def compute_precision_recall_rocauc(
     )
 
     # calculate metrics at various score thresholds
-    counts = np.zeros((n_scores, n_labels, 4), dtype=np.int32)
     for score_idx in range(n_scores):
         mask_score_threshold = (
             detailed_pairs[:, 3] >= score_thresholds[score_idx]
@@ -276,30 +295,26 @@ def compute_precision_recall_rocauc(
         counts[score_idx, :, 2] = np.bincount(fn[:, 1], minlength=n_labels)
         counts[score_idx, :, 3] = np.bincount(tn[:, 1], minlength=n_labels)
 
-    recall = np.zeros((n_scores, n_labels), dtype=np.float64)
     np.divide(
         counts[:, :, 0],
         (counts[:, :, 0] + counts[:, :, 2]),
         where=(counts[:, :, 0] + counts[:, :, 2]) > 1e-9,
         out=recall,
     )
 
-    precision = np.zeros_like(recall)
     np.divide(
         counts[:, :, 0],
         (counts[:, :, 0] + counts[:, :, 1]),
         where=(counts[:, :, 0] + counts[:, :, 1]) > 1e-9,
         out=precision,
     )
 
-    accuracy = np.zeros(n_scores, dtype=np.float64)
     np.divide(
         counts[:, :, 0].sum(axis=1),
         float(n_datums),
         out=accuracy,
     )
 
-    f1_score = np.zeros_like(recall)
     np.divide(
         (2 * precision * recall),
         (precision + recall),

src/valor_lite/classification/manager.py

@@ -1,4 +1,3 @@
-import warnings
 from dataclasses import asdict, dataclass
 
 import numpy as np
@@ -17,6 +16,7 @@
     unpack_confusion_matrix_into_metric_list,
     unpack_precision_recall_rocauc_into_metric_lists,
 )
+from valor_lite.exceptions import EmptyEvaluatorException, EmptyFilterException
 
 """
 Usage
@@ -85,6 +85,18 @@ class Filter:
     valid_label_indices: NDArray[np.int32] | None
     metadata: Metadata
 
+    def __post_init__(self):
+        # validate datum mask
+        if not self.datum_mask.any():
+            raise EmptyFilterException("filter removes all datums")
+
+        # validate label indices
+        if (
+            self.valid_label_indices is not None
+            and self.valid_label_indices.size == 0
+        ):
+            raise EmptyFilterException("filter removes all labels")
+
 
 class Evaluator:
     """
@@ -155,7 +167,6 @@ def create_filter(
         datum_mask = np.ones(n_pairs, dtype=np.bool_)
         if datum_ids is not None:
             if not datum_ids:
-                warnings.warn("no valid filtered pairs")
                 return Filter(
                     datum_mask=np.zeros_like(datum_mask),
                     valid_label_indices=None,
@@ -173,7 +184,6 @@ def create_filter(
         valid_label_indices = None
         if labels is not None:
             if not labels:
-                warnings.warn("no valid filtered pairs")
                 return Filter(
                     datum_mask=datum_mask,
                     valid_label_indices=np.array([], dtype=np.int32),
@@ -224,21 +234,6 @@ def filter(
         NDArray[int32]
             The filtered label metadata.
         """
-        empty_datum_mask = not filter_.datum_mask.any()
-        empty_label_mask = (
-            filter_.valid_label_indices.size == 0
-            if filter_.valid_label_indices is not None
-            else False
-        )
-        if empty_datum_mask or empty_label_mask:
-            if empty_datum_mask:
-                warnings.warn("filter removes all datums")
-            if empty_label_mask:
-                warnings.warn("filter removes all labels")
-            return (
-                np.array([], dtype=np.float64),
-                np.zeros((self.metadata.number_of_labels, 2), dtype=np.int32),
-            )
         return filter_cache(
             detailed_pairs=self._detailed_pairs,
             datum_mask=filter_.datum_mask,
@@ -502,9 +497,7 @@ def finalize(self):
             A ready-to-use evaluator object.
         """
         if self._detailed_pairs.size == 0:
-            self._label_metadata = np.array([], dtype=np.int32)
-            warnings.warn("evaluator is empty")
-            return self
+            raise EmptyEvaluatorException
 
         self._label_metadata = compute_label_metadata(
             ids=self._detailed_pairs[:, :3].astype(np.int32),

src/valor_lite/exceptions.py

@@ -0,0 +1,15 @@
+class EmptyEvaluatorException(Exception):
+    def __init__(self):
+        super().__init__(
+            "evaluator cannot be finalized as it contains no data"
+        )
+
+
+class EmptyFilterException(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)
+
+
+class InternalCacheException(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)

src/valor_lite/object_detection/computation.py

@@ -1,4 +1,3 @@
-import warnings
 from enum import IntFlag, auto
 
 import numpy as np
@@ -280,14 +279,6 @@ def filter_cache(
     )
     detailed_pairs = detailed_pairs[~mask_null_pairs]
 
-    if detailed_pairs.size == 0:
-        warnings.warn("no valid filtered pairs")
-        return (
-            np.array([], dtype=np.float64),
-            np.array([], dtype=np.float64),
-            np.zeros((n_labels, 2), dtype=np.int32),
-        )
-
     # sorts by score, iou with ground truth id as a tie-breaker
     indices = np.lexsort(
         (
@@ -441,15 +432,6 @@ def compute_precion_recall(
     counts = np.zeros((n_ious, n_scores, n_labels, 6), dtype=np.float64)
     pr_curve = np.zeros((n_ious, n_labels, 101, 2))
 
-    if ranked_pairs.size == 0:
-        warnings.warn("no valid ranked pairs")
-        return (
-            (average_precision, mAP),
-            (average_recall, mAR),
-            counts,
-            pr_curve,
-        )
-
     # start computation
     ids = ranked_pairs[:, :5].astype(np.int32)
     gt_ids = ids[:, 1]