diff --git a/libs/giskard-checks/src/giskard/checks/__init__.py b/libs/giskard-checks/src/giskard/checks/__init__.py index d476790777..faa12a146f 100644 --- a/libs/giskard-checks/src/giskard/checks/__init__.py +++ b/libs/giskard-checks/src/giskard/checks/__init__.py @@ -9,6 +9,8 @@ from .builtin import ( AllOf, AnyOf, + ContainsAll, + ContainsAny, Equals, FnCheck, GreaterEquals, @@ -133,6 +135,8 @@ "Toxicity", "StringMatching", "RegexMatching", + "ContainsAny", + "ContainsAll", # Exceptions "InputGenerationException", # LLM-based generators diff --git a/libs/giskard-checks/src/giskard/checks/builtin/__init__.py b/libs/giskard-checks/src/giskard/checks/builtin/__init__.py index be95f496aa..56261923e1 100644 --- a/libs/giskard-checks/src/giskard/checks/builtin/__init__.py +++ b/libs/giskard-checks/src/giskard/checks/builtin/__init__.py @@ -30,7 +30,7 @@ from .nlp_metrics import Readability from .rego_policy import RegoPolicy from .semantic_similarity import SemanticSimilarity -from .text_matching import RegexMatching, StringMatching +from .text_matching import ContainsAll, ContainsAny, RegexMatching, StringMatching __all__ = [ "AllOf", @@ -43,6 +43,8 @@ "RegoPolicy", "StringMatching", "RegexMatching", + "ContainsAny", + "ContainsAll", "Equals", "NotEquals", "LessThan", diff --git a/libs/giskard-checks/src/giskard/checks/builtin/text_matching.py b/libs/giskard-checks/src/giskard/checks/builtin/text_matching.py index 931d8212bb..f18998a59b 100644 --- a/libs/giskard-checks/src/giskard/checks/builtin/text_matching.py +++ b/libs/giskard-checks/src/giskard/checks/builtin/text_matching.py @@ -3,6 +3,8 @@ This module provides checks for text matching: - StringMatching: Literal substring matching with normalization - RegexMatching: Regular expression pattern matching +- ContainsAny: Passes if text contains at least one value from a list +- ContainsAll: Passes if text contains every value from a list """ from abc import ABC, abstractmethod @@ -19,6 +21,71 @@ from ..utils.normalization import NormalizationForm, normalize_string +def _format_str( + value: str, normalization_form: NormalizationForm | None, case_sensitive: bool +) -> str: + """Format a string for matching by applying normalization and case handling. + + 1. Applies Unicode normalization if specified. + 2. Converts to lowercase if case-insensitive matching is enabled. + + Parameters + ---------- + value : str + The string to format. + normalization_form : NormalizationForm | None + Unicode normalization form to apply, or None to skip normalization. + case_sensitive : bool + If False, the string is lowercased. + + Returns + ------- + str + The formatted string ready for comparison. + """ + value = normalize_string(value, normalization_form) + + if not case_sensitive: + value = value.lower() + + return value + + +def _partition_by_containment( + values: list[str], + formatted_text: str, + normalization_form: NormalizationForm | None, + case_sensitive: bool, +) -> tuple[list[str], list[str]]: + """Partition values into matched/missing based on containment in formatted_text. + + Each value is formatted once and classified in a single pass, avoiding the + O(N^2) cost of testing list membership per value. + + Parameters + ---------- + values : list[str] + The values to check for containment. + formatted_text : str + The already-formatted text to search within. + normalization_form : NormalizationForm | None + Unicode normalization form to apply to each value. + case_sensitive : bool + If False, values are lowercased before comparison. + + Returns + ------- + tuple[list[str], list[str]] + The (matched, missing) values, in their original form and order. + """ + matched: list[str] = [] + missing: list[str] = [] + for v in values: + formatted_v = _format_str(v, normalization_form, case_sensitive) + (matched if formatted_v in formatted_text else missing).append(v) + return matched, missing + + class TextBasedCheck[InputType, OutputType, TraceType: Trace]( # pyright: ignore[reportMissingTypeArgument] Check[InputType, OutputType, TraceType], ABC ): @@ -240,31 +307,6 @@ def validate_keyword_or_keyword_key(self) -> Self: return self - def _format_str(self, value: str) -> str: - """Format a string for matching by applying normalization and case handling. - - This method: - 1. Applies Unicode normalization if specified - 2. Converts to lowercase if case-insensitive matching is enabled - 3. Normalizes whitespace (collapses multiple spaces to single space, trims) - - Parameters - ---------- - value : str - The string to format. - - Returns - ------- - str - The formatted string ready for comparison. - """ - value = normalize_string(value, self.normalization_form) - - if not self.case_sensitive: - value = value.lower() - - return value - @override async def run(self, trace: TraceType) -> CheckResult: """Execute the string matching check. @@ -302,8 +344,10 @@ async def run(self, trace: TraceType) -> CheckResult: details["case_sensitive"] = self.case_sensitive # Format both strings for comparison - formatted_text = self._format_str(text) - formatted_keyword = self._format_str(keyword) + formatted_text = _format_str(text, self.normalization_form, self.case_sensitive) + formatted_keyword = _format_str( + keyword, self.normalization_form, self.case_sensitive + ) # Check if keyword appears in text if formatted_keyword in formatted_text: @@ -472,3 +516,273 @@ async def run(self, trace: TraceType) -> CheckResult: message=f"Text does not match the regex pattern '{pattern}'.", details=details, ) + + +class _ContainsBase[InputType, OutputType, TraceType: Trace]( # pyright: ignore[reportMissingTypeArgument] + TextBasedCheck[InputType, OutputType, TraceType], ABC +): + """Base class for checks that validate text against a list of values. + + Attributes + ---------- + values : list[str] | MISSING + The list of values to search for within the text. Either this or + ``values_key`` must be provided. + values_key : JSONPathStr | MISSING + JSONPath expression to extract the values list from the trace. + Either this or ``values`` must be provided. + normalization_form : NormalizationForm | None + Unicode normalization form to apply before matching. Defaults to "NFKC". + case_sensitive : bool + If True, matching is case-sensitive. Defaults to True. + """ + + values: list[str] | MISSING = Field( + default=MISSING, + description="The list of values to search for within the text. Either this or values_key must be provided.", + ) + values_key: JSONPathStr | MISSING = Field( + default=MISSING, + description="JSONPath expression to extract the values list from the trace (e.g., 'trace.last.inputs.expected_topics'). Either this or values must be provided.", + ) + normalization_form: NormalizationForm | None = Field( + default="NFKC", + description="Unicode normalization form to apply (NFC, NFD, NFKC, NFKD). Defaults to NFKC.", + ) + case_sensitive: bool = Field( + default=True, + description="If True, matching is case-sensitive. If False, both strings are lowercased before comparison.", + ) + + @model_validator(mode="after") + def validate_values_or_values_key(self) -> Self: + """Validate that exactly one of values or values_key is provided. + + Returns + ------- + Self + The validated instance. + + Raises + ------ + ValueError + If neither or both values and values_key are provided. + """ + if (self.values is MISSING) == (self.values_key is MISSING): + raise ValueError( + "Exactly one of 'values' or 'values_key' must be provided, not both or neither." + ) + return self + + def _extract_text_and_values( + self, trace: TraceType + ) -> tuple[str, list[str], dict[str, Any]] | tuple[None, None, CheckResult]: + """Extract and validate text and the values list from trace or direct values. + + Parameters + ---------- + trace : TraceType + The trace to extract values from. + + Returns + ------- + tuple[str, list[str], dict] | tuple[None, None, CheckResult] + Either (text, values, details) on success, or (None, None, error_result) on failure. + """ + text = provided_or_resolve(trace, key=self.text_key, value=self.text) + values = provided_or_resolve(trace, key=self.values_key, value=self.values) + + details: dict[str, Any] = {"text": text, "values": values} + + if isinstance(values, NoMatch): + return ( + None, + None, + CheckResult.failure( + message=f"No value found for values key '{self.values_key}'.", + details=details, + ), + ) + + if not isinstance(values, list) or not all(isinstance(v, str) for v in values): + return ( + None, + None, + CheckResult.failure( + message=f"Value for values is not a list of strings, expected list[str] but got {type(values).__name__}.", + details=details, + ), + ) + + if not values: + return ( + None, + None, + CheckResult.failure( + message="The values list is empty.", + details=details, + ), + ) + + if isinstance(text, NoMatch): + return ( + None, + None, + CheckResult.failure( + message=f"No value found for text key '{self.text_key}'.", + details=details, + ), + ) + + if not isinstance(text, str): + return ( + None, + None, + CheckResult.failure( + message=f"Value for text is not a string, expected string but got {type(text).__name__}.", + details=details, + ), + ) + + return text, values, details + + @abstractmethod + async def run(self, trace: TraceType) -> CheckResult: + """Execute the check. Must be implemented by subclasses.""" + ... + + +@Check.register("contains_any") +class ContainsAny[InputType, OutputType, TraceType: Trace]( # pyright: ignore[reportMissingTypeArgument] + _ContainsBase[InputType, OutputType, TraceType] +): + """Check that validates if text contains at least one value from a list. + + Examples + -------- + Direct text and values:: + + check = ContainsAny( + text="The answer covers pricing and support.", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + + Extract text from trace:: + + check = ContainsAny( + values=["Paris", "London"], + text_key="trace.last.outputs.response", + ) + """ + + @override + async def run(self, trace: TraceType) -> CheckResult: + """Execute the contains-any check. + + Parameters + ---------- + trace : TraceType + The trace containing interaction history. Used to extract + text/values if not provided directly. + + Returns + ------- + CheckResult + Success if at least one value is found in text, failure otherwise. + Includes details about the text, values, matched, and missing values. + """ + result = self._extract_text_and_values(trace) + if result[0] is None: + return result[2] # type: ignore[return-value] + + text, values, details = result[0], result[1], result[2] + details["normalization_form"] = self.normalization_form + details["case_sensitive"] = self.case_sensitive + + formatted_text = _format_str(text, self.normalization_form, self.case_sensitive) + matched, missing = _partition_by_containment( + values, formatted_text, self.normalization_form, self.case_sensitive + ) + + details["matched"] = matched + details["missing"] = missing + + if matched: + return CheckResult.success( + message=f"The answer contains at least one of the values: {matched}.", + details=details, + ) + + return CheckResult.failure( + message=f"The answer does not contain any of the values: {values}.", + details=details, + ) + + +@Check.register("contains_all") +class ContainsAll[InputType, OutputType, TraceType: Trace]( # pyright: ignore[reportMissingTypeArgument] + _ContainsBase[InputType, OutputType, TraceType] +): + """Check that validates if text contains every value from a list. + + Examples + -------- + Direct text and values:: + + check = ContainsAll( + text="The answer covers pricing, refunds and shipping.", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + + Extract text from trace:: + + check = ContainsAll( + values=["Paris", "France"], + text_key="trace.last.outputs.response", + ) + """ + + @override + async def run(self, trace: TraceType) -> CheckResult: + """Execute the contains-all check. + + Parameters + ---------- + trace : TraceType + The trace containing interaction history. Used to extract + text/values if not provided directly. + + Returns + ------- + CheckResult + Success if every value is found in text, failure otherwise. + Includes details about the text, values, matched, and missing values. + """ + result = self._extract_text_and_values(trace) + if result[0] is None: + return result[2] # type: ignore[return-value] + + text, values, details = result[0], result[1], result[2] + details["normalization_form"] = self.normalization_form + details["case_sensitive"] = self.case_sensitive + + formatted_text = _format_str(text, self.normalization_form, self.case_sensitive) + matched, missing = _partition_by_containment( + values, formatted_text, self.normalization_form, self.case_sensitive + ) + + details["matched"] = matched + details["missing"] = missing + + if not missing: + return CheckResult.success( + message=f"The answer contains all of the values: {values}.", + details=details, + ) + + return CheckResult.failure( + message=f"The answer is missing the following values: {missing}.", + details=details, + ) diff --git a/libs/giskard-checks/tests/builtin/test_contains_matching.py b/libs/giskard-checks/tests/builtin/test_contains_matching.py new file mode 100644 index 0000000000..3a82ce2687 --- /dev/null +++ b/libs/giskard-checks/tests/builtin/test_contains_matching.py @@ -0,0 +1,184 @@ +"""Tests for the ContainsAny and ContainsAll checks.""" + +import pytest +from giskard.checks import CheckStatus, ContainsAll, ContainsAny, Interaction, Trace +from giskard.checks.core.extraction import NoMatch + + +async def test_contains_any_passes_with_one_match() -> None: + """ContainsAny passes when at least one value is found.""" + check = ContainsAny( + text="The answer covers pricing and support.", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.PASS + assert result.details["matched"] == ["pricing"] + assert result.details["missing"] == ["refunds", "shipping"] + + +async def test_contains_any_passes_with_all_matches() -> None: + """ContainsAny passes when every value is found.""" + check = ContainsAny( + text="pricing, refunds and shipping", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.PASS + assert result.details["matched"] == ["pricing", "refunds", "shipping"] + + +async def test_contains_any_fails_with_no_matches() -> None: + """ContainsAny fails when none of the values are found.""" + check = ContainsAny( + text="The answer is unrelated.", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.FAIL + assert result.message is not None + assert "does not contain any" in result.message + assert result.details["matched"] == [] + assert result.details["missing"] == ["pricing", "refunds", "shipping"] + + +async def test_contains_all_passes_when_all_present() -> None: + """ContainsAll passes when every value is found.""" + check = ContainsAll( + text="The answer covers pricing, refunds and shipping.", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.PASS + assert result.message is not None + assert "contains all" in result.message + assert result.details["missing"] == [] + + +async def test_contains_all_fails_when_partial_match() -> None: + """ContainsAll fails when only some values are found.""" + check = ContainsAll( + text="The answer covers pricing only.", + values=["pricing", "refunds", "shipping"], + case_sensitive=False, + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.FAIL + assert result.message is not None + assert "missing" in result.message + assert result.details["matched"] == ["pricing"] + assert result.details["missing"] == ["refunds", "shipping"] + + +async def test_case_sensitivity() -> None: + """Case sensitivity is configurable for both checks.""" + check_sensitive = ContainsAny( + text="Hello World", values=["world"], case_sensitive=True + ) + result = await check_sensitive.run(Trace()) + assert result.status == CheckStatus.FAIL + + check_insensitive = ContainsAny( + text="Hello World", values=["world"], case_sensitive=False + ) + result = await check_insensitive.run(Trace()) + assert result.status == CheckStatus.PASS + + +async def test_text_and_values_from_trace() -> None: + """Both text and values can be extracted from the trace via JSONPath.""" + check = ContainsAll( + text_key="trace.last.outputs.response", + values_key="trace.last.inputs.expected_topics", + ) + interaction = Interaction( + inputs={"expected_topics": ["Paris", "Eiffel"]}, + outputs={"response": "The Eiffel Tower is in Paris."}, + ) + result = await check.run(Trace(interactions=[interaction])) + assert result.status == CheckStatus.PASS + + +async def test_unicode_normalization() -> None: + """Unicode normalization is applied before matching.""" + check = ContainsAny( + text="Hello Ａ World", + values=["A"], + normalization_form="NFKC", + case_sensitive=False, + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.PASS + + +async def test_empty_values_list_fails() -> None: + """An empty values list is a validation failure, not a pass or crash.""" + check = ContainsAny(text="Hello", values=[]) + result = await check.run(Trace()) + assert result.status == CheckStatus.FAIL + assert result.message is not None + assert "empty" in result.message + + +async def test_missing_values_in_trace() -> None: + """Missing values in the trace produce a failure result, not an exception.""" + check = ContainsAny( + text="Some text", + values_key="trace.last.inputs.nonexistent", + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.FAIL + assert result.message is not None + assert ( + "No value found for values key 'trace.last.inputs.nonexistent'." + in result.message + ) + assert isinstance(result.details["values"], NoMatch) + + +async def test_missing_text_in_trace() -> None: + """Missing text in the trace produces a failure result, not an exception.""" + check = ContainsAny( + text_key="trace.last.outputs.nonexistent", + values=["test"], + ) + result = await check.run(Trace()) + assert result.status == CheckStatus.FAIL + assert result.message is not None + assert ( + "No value found for text key 'trace.last.outputs.nonexistent'" in result.message + ) + assert isinstance(result.details["text"], NoMatch) + + +async def test_non_list_values_fails() -> None: + """A non-list value for `values` is a validation failure, not a crash.""" + check = ContainsAny( + text_key="trace.last.outputs.response", + values_key="trace.last.outputs.response", + ) + interaction = Interaction(inputs={}, outputs={"response": "not a list"}) + result = await check.run(Trace(interactions=[interaction])) + assert result.status == CheckStatus.FAIL + assert result.message is not None + assert "not a list of strings" in result.message + + +async def test_cannot_provide_both_values_and_values_key() -> None: + """Providing both values and values_key raises at construction time.""" + with pytest.raises(ValueError, match="Exactly one"): + ContainsAny( + text="Hello World", + values=["hello"], + values_key="trace.last.inputs.key", + ) + + +async def test_cannot_provide_neither_values_nor_values_key() -> None: + """Providing neither values nor values_key raises at construction time.""" + with pytest.raises(ValueError, match="Exactly one"): + ContainsAny(text="Hello World")