test(examples): keep README snippets and examples correct via CI

.github/workflows/ci.yml

@@ -30,6 +30,24 @@ jobs:
       - run: make install install-tools
       - run: make check
 
+  test-examples:
+    name: test-examples
+    needs: lint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read # checkout repository
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+      - run: make install
+      - run: make test-examples
+
   test-unit:
     name: test-unit / ${{ matrix.package }} / ${{ matrix.python-version }}
     needs: lint

CONTRIBUTING.md

@@ -68,4 +68,10 @@ make test      # pytest for packages under libs/
 
 Run `make help` for other targets (for example scoped tests with `PACKAGE=giskard-checks`).
 
+See also [`.cursor/rules/documentation.mdc`](.cursor/rules/documentation.mdc) for module-level and inline comment guidance.
+
+### Public examples
+
+Keep `examples/` and in-repo README Python fences runnable and importable. When you change a public API shown in documentation, update the matching example and run `make test-examples`.
+
 **This guide was heavily inspired by the awesome [Hugging Face guide to contributing](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md).**

Makefile

@@ -68,6 +68,10 @@ else
 	$(foreach lib,$(LIBS),uv run pytest libs/$(lib) -m "not functional" &&) true
 endif
 
+test-examples: ## Run canonical examples and README snippet lint
+	uv run pytest examples -q
+	uv run python examples/lint_readme_snippets.py
+
 test-no-providers: ## Run tests that verify behavior when provider SDKs are missing
 	uv run pytest libs/giskard-llm -m "no_providers"
 

README.md

@@ -72,13 +72,15 @@ from giskard.checks import Scenario, Groundedness
 
 client = OpenAI()
 
+
 def get_answer(inputs: str) -> str:
     response = client.chat.completions.create(
         model="gpt-5-mini",
         messages=[{"role": "user", "content": inputs}],
     )
     return response.choices[0].message.content
 
+
 scenario = (
     Scenario("test_dynamic_output")
     .interact(
@@ -121,13 +123,15 @@ Use Giskard Scan to:
 import asyncio
 from giskard.scan import vulnerability_scan
 
+
 async def main():
     await vulnerability_scan(
         target=my_agent,
         description="A customer support chatbot for an e-commerce platform.",
         languages=["en"],
     )
 
+
 asyncio.run(main())
 ```
 
@@ -147,11 +151,13 @@ Wrap your model and run the scan:
 import giskard
 import pandas as pd
 
+
 # Replace my_llm_chain with your actual LLM chain or model inference logic
 def model_predict(df: pd.DataFrame):
     """The function takes a DataFrame and must return a list of outputs (one per row)."""
     return [my_llm_chain.run({"query": question}) for question in df["question"]]
 
+
 giskard_model = giskard.Model(
     model=model_predict,
     model_type="text_generation",
@@ -183,7 +189,7 @@ knowledge_base = KnowledgeBase.from_pandas(df, columns=["column_1", "column_2"])
 testset = generate_testset(
     knowledge_base,
     num_questions=60,
-    language='en',
+    language="en",
     agent_description="A customer support chatbot for company X",
 )
 ```

examples/checks_static/test_checks_static.py

@@ -0,0 +1,37 @@
+"""Offline checks happy path — mirrors the canonical example contract."""
+
+import asyncio
+
+from giskard.checks import CheckResult, Equals, Scenario, from_fn
+
+
+def echo(inputs: str) -> str:
+    return inputs
+
+
+@from_fn
+async def not_empty(trace) -> CheckResult:
+    outputs = trace.last.outputs
+    if str(outputs).strip():
+        return CheckResult.success(message="non-empty")
+    return CheckResult.failure(message="empty")
+
+
+async def main() -> None:
+    result = await (
+        Scenario("echo")
+        .interact(inputs="hello", outputs=echo)
+        .check(Equals(key="trace.last.outputs", expected_value="hello"))
+        .check(not_empty)
+        .run()
+    )
+    assert result.passed
+    result.print_report()
+
+
+async def test_checks_static_happy_path() -> None:
+    await main()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/lint_readme_snippets.py

@@ -0,0 +1,67 @@
+"""Fail when README fences use positional CheckResult.success/failure strings."""
+
+from __future__ import annotations
+
+import ast
+import re
+import sys
+from pathlib import Path
+
+_FENCE_RE = re.compile(r"```(?:python|py)\n(.*?)```", re.DOTALL)
+_FORBIDDEN_CALLS = frozenset({"CheckResult.success", "CheckResult.failure"})
+
+
+def _check_source(source: str, location: str) -> list[str]:
+    errors: list[str] = []
+    try:
+        tree = ast.parse(source)
+    except SyntaxError as exc:
+        errors.append(f"{location}: invalid Python fence: {exc.msg}")
+        return errors
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func = node.func
+        if not isinstance(func, ast.Attribute):
+            continue
+        if not isinstance(func.value, ast.Name) or func.value.id != "CheckResult":
+            continue
+        if func.attr not in ("success", "failure"):
+            continue
+        if node.args:
+            first = node.args[0]
+            if isinstance(first, ast.Constant) and isinstance(first.value, str):
+                errors.append(
+                    f"{location}:{node.lineno}: positional CheckResult.{func.attr}(str) "
+                    "is forbidden; use message="
+                )
+    return errors
+
+
+def lint_markdown(path: Path) -> list[str]:
+    text = path.read_text(encoding="utf-8")
+    errors: list[str] = []
+    for match in _FENCE_RE.finditer(text):
+        source = match.group(1)
+        errors.extend(_check_source(source, str(path)))
+    return errors
+
+
+def main() -> int:
+    root = Path(__file__).resolve().parents[1]
+    targets = [
+        root / "README.md",
+        *sorted((root / "libs").glob("*/README.md")),
+    ]
+    errors: list[str] = []
+    for path in targets:
+        if path.is_file():
+            errors.extend(lint_markdown(path))
+    for error in errors:
+        print(error, file=sys.stderr)
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/scan_stub/test_scan_stub.py

@@ -0,0 +1,67 @@
+"""Offline scan stub — exercises generate_suite with no network generators."""
+
+from typing import Any
+
+import pytest
+from giskard.checks import Equals, Scenario, SuiteResult
+from giskard.scan.catalog import generate_suite
+
+
+async def echo(inputs: str) -> str:
+    return inputs
+
+
+async def test_generate_suite_empty_generators_offline() -> None:
+    suite = await generate_suite(
+        description="Demo support agent",
+        languages=["en"],
+        generators=[],
+        max_scenarios=5,
+    )
+    assert suite.scenarios == []
+
+
+async def test_run_static_scenario_as_scan_stub(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    scenario = (
+        Scenario("stub")
+        .interact(inputs="ping", outputs=echo)
+        .check(Equals(key="trace.last.outputs", expected_value="ping"))
+    )
+
+    class _FakeSuite:
+        def __init__(self) -> None:
+            self.scenarios = [scenario]
+
+        async def run(
+            self,
+            target: object,
+            parallel: bool = True,
+            max_concurrency: int | None = None,
+            return_exception: bool = False,
+        ) -> SuiteResult:
+            _ = parallel, max_concurrency, return_exception
+            scenario_result = await scenario.run(target=target)  # pyright: ignore[reportArgumentType]
+            return SuiteResult(
+                results=[scenario_result],
+                duration_ms=scenario_result.duration_ms,
+            )
+
+    async def fake_generate_suite(**kwargs: Any) -> _FakeSuite:
+        _ = kwargs
+        return _FakeSuite()
+
+    import giskard.scan.vulnerability as vulnerability_module
+
+    monkeypatch.setattr(vulnerability_module, "generate_suite", fake_generate_suite)
+
+    from giskard.scan import vulnerability_scan
+
+    result = await vulnerability_scan(
+        target=echo,
+        description="Demo agent",
+        languages=["en"],
+        max_scenarios=1,
+    )
+    assert result.pass_rate == 1.0