Add contains_any and contains_all checksFeat/contains any all checks

libs/giskard-checks/src/giskard/checks/__init__.py

@@ -9,6 +9,8 @@
 from .builtin import (
     AllOf,
     AnyOf,
+    ContainsAll,
+    ContainsAny,
     Equals,
     FnCheck,
     GreaterEquals,
@@ -110,6 +112,8 @@
     "SemanticSimilarity",
     "Toxicity",
     "StringMatching",
+    "ContainsAny",
+    "ContainsAll",
     "RegexMatching",
     # Generators
     "UserSimulator",

libs/giskard-checks/src/giskard/checks/builtin/__init__.py

@@ -25,7 +25,7 @@
 from .composition import AllOf, AnyOf, Not
 from .fn import FnCheck, from_fn
 from .semantic_similarity import SemanticSimilarity
-from .text_matching import RegexMatching, StringMatching
+from .text_matching import ContainsAll, ContainsAny, RegexMatching, StringMatching
 
 __all__ = [
     "AllOf",
@@ -34,6 +34,8 @@
     "from_fn",
     "FnCheck",
     "StringMatching",
+    "ContainsAny",
+    "ContainsAll",
     "RegexMatching",
     "Equals",
     "NotEquals",

libs/giskard-checks/src/giskard/checks/builtin/text_matching.py

@@ -3,6 +3,8 @@
 This module provides checks for text matching:
 - StringMatching: Literal substring matching with normalization
 - RegexMatching: Regular expression pattern matching
+- ContainsAny: Checks whether text contains at least one value from a list
+- ContainsAll: Checks whether text contains every value from a list
 """
 
 from abc import ABC, abstractmethod
@@ -320,6 +322,136 @@ async def run(self, trace: TraceType) -> CheckResult:
         )
 
 
+class ListStringMatching[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
+    TextBasedCheck[InputType, OutputType, TraceType], ABC
+):
+    """Base class for checks that validate text against a list of string values."""
+
+    values: list[str] = Field(
+        description="The list of strings to check against.",
+    )
+    normalization_form: NormalizationForm | None = Field(
+        default="NFKC",
+        description="Unicode normalization form to apply (NFC, NFD, NFKC, NFKD). Defaults to NFKC.",
+    )
+    case_sensitive: bool = Field(
+        default=False,
+        description="If True, matching is case-sensitive. If False, text and values are lowercased before comparison.",
+    )
+
+    def _format_str(self, value: str) -> str:
+        """Format a string for matching by applying normalization and case handling."""
+        value = normalize_string(value, self.normalization_form)
+
+        if not self.case_sensitive:
+            value = value.lower()
+
+        return value
+
+    def _extract_and_validate_text(
+        self, trace: TraceType
+    ) -> tuple[str, dict[str, Any]] | CheckResult:
+        """Extract and validate text from trace or direct value."""
+        text = provided_or_resolve(
+            trace, key=self.text_key, value=provide_not_none(self.text)
+        )
+
+        details: dict[str, Any] = {
+            "text": text,
+            "values": self.values,
+            "normalization_form": self.normalization_form,
+            "case_sensitive": self.case_sensitive,
+        }
+
+        if isinstance(text, NoMatch):
+            return CheckResult.failure(
+                message=f"No value found for text key '{self.text_key}'.",
+                details=details,
+            )
+
+        if not isinstance(text, str):
+            return CheckResult.failure(
+                message=f"Value for text is not a string, expected string but got {type(text).__name__}.",
+                details=details,
+            )
+
+        return text, details
+
+
+@Check.register("contains_any")
+class ContainsAny[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
+    ListStringMatching[InputType, OutputType, TraceType]
+):
+    """Check that validates whether text contains at least one value from a list."""
+
+    @override
+    async def run(self, trace: TraceType) -> CheckResult:
+        """Execute the contains-any check."""
+        extracted = self._extract_and_validate_text(trace)
+        if isinstance(extracted, CheckResult):
+            return extracted
+
+        text, details = extracted
+        formatted_text = self._format_str(text)
+        formatted_values = [self._format_str(value) for value in self.values]
+        matched_values = [
+            value
+            for value, formatted_value in zip(self.values, formatted_values, strict=True)
+            if formatted_value in formatted_text
+        ]
+
+        details["matched_values"] = matched_values
+
+        if matched_values:
+            return CheckResult.success(
+                message=f"The answer contains at least one expected value: {matched_values!r}.",
+                details=details,
+            )
+
+        return CheckResult.failure(
+            message="The answer does not contain any of the expected values.",
+            details=details,
+        )
+
+
+@Check.register("contains_all")
+class ContainsAll[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
+    ListStringMatching[InputType, OutputType, TraceType]
+):
+    """Check that validates whether text contains every value from a list."""
+
+    @override
+    async def run(self, trace: TraceType) -> CheckResult:
+        """Execute the contains-all check."""
+        extracted = self._extract_and_validate_text(trace)
+        if isinstance(extracted, CheckResult):
+            return extracted
+
+        text, details = extracted
+        formatted_text = self._format_str(text)
+        formatted_values = [self._format_str(value) for value in self.values]
+        missing_values = [
+            value
+            for value, formatted_value in zip(self.values, formatted_values, strict=True)
+            if formatted_value not in formatted_text
+        ]
+        matched_values = [value for value in self.values if value not in missing_values]
+
+        details["matched_values"] = matched_values
+        details["missing_values"] = missing_values
+
+        if not missing_values:
+            return CheckResult.success(
+                message="The answer contains all expected values.",
+                details=details,
+            )
+
+        return CheckResult.failure(
+            message=f"The answer is missing expected values: {missing_values!r}.",
+            details=details,
+        )
+
+
 @Check.register("regex_matching")
 class RegexMatching[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
     TextBasedCheck[InputType, OutputType, TraceType]

libs/giskard-checks/tests/builtin/test_string_matching.py

@@ -1,7 +1,14 @@
 """Tests for the StringMatching check."""
 
 import pytest
-from giskard.checks import CheckStatus, Interaction, StringMatching, Trace
+from giskard.checks import (
+    CheckStatus,
+    ContainsAll,
+    ContainsAny,
+    Interaction,
+    StringMatching,
+    Trace,
+)
 from giskard.checks.core.extraction import NoMatch
 
 
@@ -413,3 +420,131 @@ async def test_unicode_e_acute_no_normalization_fails() -> None:
     result = await check.run(Trace())
     # Without normalization, they should not match
     assert result.status == CheckStatus.FAIL
+
+
+async def test_contains_any_passes_when_any_value_is_found() -> None:
+    """Test that ContainsAny passes when at least one value is found."""
+    check = ContainsAny(
+        text="ML is a subset of AI.",
+        values=["machine learning", "ML", "artificial intelligence"],
+    )
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.PASS
+    assert result.details["matched_values"] == ["ML"]
+
+
+async def test_contains_any_fails_when_no_value_is_found() -> None:
+    """Test that ContainsAny fails when none of the values are found."""
+    check = ContainsAny(text="The answer is about databases.", values=["ML", "AI"])
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.FAIL
+    assert result.details["matched_values"] == []
+
+
+async def test_contains_all_passes_when_all_values_are_found() -> None:
+    """Test that ContainsAll passes when every value is found."""
+    check = ContainsAll(
+        text="The response includes a definition and an example.",
+        values=["definition", "example"],
+    )
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.PASS
+    assert result.details["matched_values"] == ["definition", "example"]
+    assert result.details["missing_values"] == []
+
+
+async def test_contains_all_fails_when_any_value_is_missing() -> None:
+    """Test that ContainsAll fails when at least one value is missing."""
+    check = ContainsAll(
+        text="The response includes a definition.",
+        values=["definition", "example"],
+    )
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.FAIL
+    assert result.details["matched_values"] == ["definition"]
+    assert result.details["missing_values"] == ["example"]
+
+
+async def test_contains_checks_are_case_insensitive_by_default() -> None:
+    """Test that list matching checks are case-insensitive by default."""
+    check = ContainsAny(text="Machine Learning is useful.", values=["machine learning"])
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.PASS
+
+
+async def test_contains_checks_support_case_sensitive_matching() -> None:
+    """Test case-sensitive matching behavior for list matching checks."""
+    check = ContainsAny(
+        text="Machine Learning is useful.",
+        values=["machine learning"],
+        case_sensitive=True,
+    )
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.FAIL
+
+
+async def test_contains_checks_extract_text_from_trace() -> None:
+    """Test extracting text from trace for list matching checks."""
+    check = ContainsAll(
+        text_key="trace.last.outputs.response",
+        values=["Paris", "France"],
+    )
+    interaction = Interaction(
+        inputs={"query": "Where is Paris?"},
+        outputs={"response": "Paris is the capital of France."},
+    )
+    result = await check.run(Trace(interactions=[interaction]))
+
+    assert result.status == CheckStatus.PASS
+    assert result.details["text"] == "Paris is the capital of France."
+
+
+async def test_contains_checks_support_unicode_normalization() -> None:
+    """Test Unicode normalization for list matching checks."""
+    check = ContainsAny(
+        text="Hello Ａ World",
+        values=["A"],
+        normalization_form="NFKC",
+    )
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.PASS
+
+
+async def test_contains_any_with_empty_values_fails() -> None:
+    """Test ContainsAny behavior with an empty values list."""
+    check = ContainsAny(text="Some text", values=[])
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.FAIL
+    assert result.details["matched_values"] == []
+
+
+async def test_contains_all_with_empty_values_passes() -> None:
+    """Test ContainsAll behavior with an empty values list."""
+    check = ContainsAll(text="Some text", values=[])
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.PASS
+    assert result.details["matched_values"] == []
+    assert result.details["missing_values"] == []
+
+
+async def test_contains_checks_report_missing_text_key() -> None:
+    """Test missing text extraction behavior for list matching checks."""
+    check = ContainsAny(
+        text_key="trace.last.outputs.nonexistent",
+        values=["test"],
+    )
+    result = await check.run(Trace())
+
+    assert result.status == CheckStatus.FAIL
+    assert result.message is not None
+    assert "No value found for text key 'trace.last.outputs.nonexistent'" in result.message
+    assert isinstance(result.details["text"], NoMatch)