feat(scan): add PostHog telemetry for scan entry points

.github/workflows/integration-tests.yml

@@ -186,3 +186,26 @@ jobs:
           GISKARD_CHECKS_DEFAULT_MODEL: "google/gemini-3.5-flash"
           GISKARD_CHECKS_DEFAULT_EMBEDDING_MODEL: "google/gemini-embedding-001"
         run: make test-functional PACKAGE=giskard-checks PROVIDER=$PROVIDER
+
+  test-scan-garak:
+    needs: authorize
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read # checkout repository
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+    name: scan / garak / ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - run: make install-garak-test
+      - run: make test-garak

Makefile

@@ -39,9 +39,9 @@ endif
 
 test-unit: ## Run unit tests only (excludes functional), optional PACKAGE=<name>
 ifdef PACKAGE
-	uv run pytest libs/$(PACKAGE) -m "not functional"
+	uv run --directory libs/$(PACKAGE) pytest tests src -m "not functional"
 else
-	$(foreach lib,$(LIBS),uv run pytest libs/$(lib) -m "not functional" &&) true
+	$(foreach lib,$(LIBS),uv run --directory libs/$(lib) pytest tests src -m "not functional" &&) true
 endif
 
 test-functional: ## Run functional tests only (requires API keys), optional PACKAGE=<name> PROVIDER=<name>
@@ -61,6 +61,15 @@ install-no-providers: ## Install giskard-llm without provider SDKs (for no_provi
 install-minimal: ## Install with test group only (no provider SDKs, all packages)
 	uv sync --only-group test
 
+install-garak-test: ## Install garak optional extra for scan integration tests
+	uv sync --group garak-test
+
+install-lidar-test: ## Install lidar private dependency for scan integration tests
+	uv sync --group lidar-test
+
+test-lidar: install-lidar-test ## Run lidar integration tests
+	uv run pytest libs/giskard-scan/tests/integrations/lidar -v
+
 test-unit-minimal: ## Run unit tests on minimal deps (no provider SDKs), optional PACKAGE=<name>
 ifdef PACKAGE
 	uv run pytest libs/$(PACKAGE) -m "not functional"
@@ -71,6 +80,9 @@ endif
 test-no-providers: ## Run tests that verify behavior when provider SDKs are missing
 	uv run pytest libs/giskard-llm -m "no_providers"
 
+test-garak: ## Run garak integration tests (requires: make install-garak-test)
+	uv run pytest libs/giskard-scan/tests/integrations/garak
+
 test-package-conflict: ## Test package conflict with giskard legacy package installed
 	@echo "Testing package conflict..."
 	@echo "Creating virtual environment..."

THIRD_PARTY_NOTICES.md

@@ -229,11 +229,11 @@ Find a list of packages below
 - License: Apache-2.0
 - Compatible: True
 
-### fsspec-2026.6.0
+### fsspec-2025.3.0
 
 - HomePage:
 - Author:
-- License: BSD-3-Clause
+- License: BSD License
 - Compatible: True
 
 ### google-auth-2.55.0
@@ -397,7 +397,7 @@ Find a list of packages below
 - License: Apache License 2.0
 - Compatible: True
 
-### numpy-2.5.0
+### numpy-2.4.6
 
 - HomePage:
 - Author: Travis E. Oliphant et al.
@@ -551,7 +551,7 @@ Find a list of packages below
 - License: MIT License
 - Compatible: True
 
-### tokenizers-0.23.1
+### tokenizers-0.22.2
 
 - HomePage:
 - Author: Nicolas Patry
@@ -586,11 +586,11 @@ Find a list of packages below
 - License: MIT
 - Compatible: True
 
-### websockets-16.0
+### websockets-15.0.1
 
 - HomePage:
 - Author: Aymeric Augustin
-- License: BSD-3-Clause
+- License: BSD License
 - Compatible: True
 
 ### yarl-1.24.2

libs/giskard-checks/src/giskard/checks/__init__.py

@@ -34,6 +34,7 @@
     GroupedSuiteResult,
     GroupStats,
     InputGenerationException,
+    InputGenerator,
     Interact,
     Interaction,
     InteractionSpec,
@@ -103,6 +104,7 @@
     "Interact",
     "Interaction",
     "InteractionSpec",
+    "InputGenerator",
     "WithGeneratorMixin",
     "WithEmbeddingMixin",
     # Builtin and LLM-based checks

libs/giskard-checks/src/giskard/checks/core/__init__.py

@@ -1,6 +1,7 @@
 from .check import Check
 from .exceptions import InputGenerationException
 from .extraction import resolve
+from .input_generator import InputGenerator
 from .interaction import Interact, Interaction, InteractionSpec, Trace
 from .result import (
     CheckResult,
@@ -24,6 +25,7 @@
     "InteractionSpec",
     "Interact",
     "Interaction",
+    "InputGenerator",
     "Check",
     "CheckResult",
     "CheckStatus",

libs/giskard-checks/src/giskard/checks/core/interaction/trace.py

@@ -1,10 +1,11 @@
-from typing import Any, Self
+from typing import Any, Self, cast
 
 from pydantic import BaseModel, Field, computed_field
 from rich.console import Console, ConsoleOptions, RenderResult
 from rich.rule import Rule
 
 from ..protocols import InteractionGenerator
+from ..types import Target
 from .interaction import Interaction
 
 
@@ -135,3 +136,17 @@ def __rich_console__(
         for idx, interaction in enumerate(self.interactions):
             yield Rule(f"Interaction {idx + 1}", style="bold")
             yield from interaction.__rich_console__(console, options)
+
+    @classmethod
+    def for_target[In, Out, Tr: Trace](  # pyright: ignore[reportMissingTypeArgument]
+        cls, target: Target[In, Out, Tr]
+    ) -> Tr:
+        # Local import: utils.inference imports Trace, so a module-level import
+        # here would create a circular import at load time.
+        from ...utils.inference import _infer_trace_type
+
+        target_type = _infer_trace_type(target)
+        if target_type is None:
+            return cast(Tr, cls())
+
+        return target_type()

libs/giskard-checks/src/giskard/checks/core/result.py

@@ -155,6 +155,7 @@ def success(
         *,
         message: str | None = None,
         details: dict[str, Any] | None = None,
+        metrics: list[Metric] | None = None,
     ) -> "CheckResult":
         """Construct a successful result.
 
@@ -165,6 +166,7 @@ def success(
             status=CheckStatus.PASS,
             message=message,
             details={} if details is None else details,
+            metrics=metrics or [],
         )
 
     @classmethod
@@ -173,12 +175,14 @@ def failure(
         *,
         message: str | None = None,
         details: dict[str, Any] | None = None,
+        metrics: list[Metric] | None = None,
     ) -> "CheckResult":
         """Construct a failure result."""
         return cls(
             status=CheckStatus.FAIL,
             message=message,
             details={} if details is None else details,
+            metrics=metrics or [],
         )
 
     @classmethod

libs/giskard-checks/src/giskard/checks/utils/inference.py

@@ -5,6 +5,7 @@
 from pydantic import PydanticUserError, TypeAdapter
 
 from ..core.interaction.trace import Trace
+from ..core.types import Target
 
 
 def _get_param_hints(target: object) -> dict[str, Any]:
@@ -46,7 +47,9 @@ def _infer_input_type(outputs: object) -> type | None:
     return first_param_type
 
 
-def _infer_trace_type(target: object) -> type[Trace] | None:  # pyright: ignore[reportMissingTypeArgument]
+def _infer_trace_type[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
+    target: Target[InputType, OutputType, TraceType],
+) -> type[TraceType] | None:
     """Return second parameter's type if it is a Trace subclass, otherwise None."""
     param_hints = _get_param_hints(target)
     if len(param_hints) < 2:

libs/giskard-scan/README.md

@@ -1,3 +1,55 @@
 # giskard-scan
 
 Agent vulnerability scanner — red teaming, prompt injection, adversarial scenario generation.
+
+## Third-party scanners (experimental)
+
+`third_party_scan` runs an external security scanner against a Giskard target and
+returns a `SuiteResult`. Only [garak](https://github.com/NVIDIA/garak) is supported
+today, and it ships as an optional extra:
+
+```bash
+pip install giskard-scan[garak]
+```
+
+```python
+import asyncio
+
+from giskard.scan import third_party_scan
+
+
+def target(inputs: str) -> str:
+    # Your agent / model call. Structured (BaseModel) inputs also work.
+    return call_my_agent(inputs)
+
+
+result = asyncio.run(
+    third_party_scan(
+        target,
+        tool="garak",
+        description="A helpful assistant",  # required; lidar builds its target profile from this, garak ignores it
+        probes=["probes.goodside.ThreatenJSON"],  # omit to run all active probes
+        target_mode="multiturn",  # "singleturn" skips garak's iterative probes
+    )
+)
+
+print(result)
+```
+
+Probes run in parallel; the target is invoked concurrently, so it must be safe to
+call from multiple threads (per-conversation state is tracked in the `Trace`, not on
+the target).
+
+### API keys and LLM-judge detectors
+
+Some garak detectors need an LLM or a third-party API to score a probe:
+
+- **LLM-judge detectors** (garak's `judge.*`, e.g. refusal detection) normally require
+  their own OpenAI key. Instead, they are automatically backed by Giskard's default
+  generator (`giskard.checks.get_default_generator()`), so they run with the same
+  credentials as the rest of Giskard — no separate OpenAI key needed.
+- **Detectors that need a third-party API key** you have not set (for example
+  `perspective.*`, which needs `PERSPECTIVE_API_KEY`) are **skipped** rather than
+  silently dropping the whole probe. Each skipped detector surfaces as a skip result
+  (`CheckResult.skip`) in the returned `SuiteResult`, with the missing key named in the
+  message, so the rest of the probe's detectors still run.

libs/giskard-scan/pyproject.toml

@@ -13,6 +13,10 @@ dependencies = [
     "huggingface-hub>=1.11.0,<2",
 ]
 
+[project.optional-dependencies]
+garak = ["garak>=0.15,<1"]
+
+
 [build-system]
 requires = ["hatchling>=1.25.0"]
 build-backend = "hatchling.build"
@@ -28,12 +32,13 @@ include = [
 ]
 
 [tool.pytest.ini_options]
-testpaths = ["tests", "src"]
-addopts = "--doctest-modules"
+testpaths = ["tests"]
+addopts = "--doctest-modules --ignore=src/giskard/scan/integrations/garak --ignore=src/giskard/scan/integrations/lidar"
 asyncio_mode = "auto"
 pythonpath = ["src"]
 markers = [
     "integration: integration tests that are skipped unless --run-integration is provided",
+    "functional: functional tests that exercise a real external tool (skipped when it is not installed)",
 ]
 
 [tool.ruff.lint]

libs/giskard-scan/src/giskard/scan/__init__.py

@@ -20,6 +20,7 @@
     SycophancyScenarioGenerator,
 )
 from .generators.prompt_injection import PromptInjectionScenarioGenerator
+from .integrations import third_party_scan
 from .quality import quality_scan, quality_suite_generator_registry
 from .registry import SuiteGeneratorRegistry
 from .types import ScanOptions
@@ -51,6 +52,7 @@
     "SuiteGeneratorRegistry",
     "quality_suite_generator_registry",
     "quality_scan",
+    "third_party_scan",
     "vulnerability_suite_generator_registry",
     "vulnerability_scan",
 ]

libs/giskard-scan/src/giskard/scan/_telemetry_props.py

@@ -0,0 +1,57 @@
+"""Aggregate, non-identifying properties for PostHog (no names, messages, or content)."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from .generators.base import ScenarioGenerator, TargetMode
+
+
+def generator_type_counts(generators: list[ScenarioGenerator]) -> dict[str, int]:
+    return dict(Counter(type(generator).__name__ for generator in generators))
+
+
+def suite_scan_shape_properties(
+    *,
+    scan_kind: str,
+    language_count: int,
+    target_mode: TargetMode,
+    generator_count: int,
+    scenario_count: int,
+    generator_types: dict[str, int],
+    parallel: bool,
+    max_concurrency: int | None,
+    **extra: Any,
+) -> dict[str, Any]:
+    return {
+        "integration": "giskard-scan",
+        "scan_kind": scan_kind,
+        "language_count": language_count,
+        "target_mode": target_mode,
+        "generator_count": generator_count,
+        "scenario_count": scenario_count,
+        "generator_types": generator_types,
+        "parallel": parallel,
+        "max_concurrency": max_concurrency,
+        **extra,
+    }
+
+
+def third_party_scan_shape_properties(
+    *,
+    tool: str,
+    language_count: int | None,
+    target_mode: TargetMode,
+    has_probe_filter: bool,
+    has_tag_filter: bool,
+) -> dict[str, Any]:
+    return {
+        "integration": "giskard-scan",
+        "scan_kind": f"third_party_{tool}",
+        "tool": tool,
+        "language_count": language_count,
+        "target_mode": target_mode,
+        "has_probe_filter": has_probe_filter,
+        "has_tag_filter": has_tag_filter,
+    }

libs/giskard-scan/src/giskard/scan/integrations/__init__.py

@@ -0,0 +1,18 @@
+"""Third-party scanner integrations for giskard.scan (experimental)."""
+
+from typing import Any, Protocol
+
+from giskard.checks import SuiteResult, Target, Trace
+
+from ._entry_point import third_party_scan
+
+
+class ScanAdapter(Protocol):
+    async def run[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
+        self,
+        target: Target[InputType, OutputType, TraceType],
+        **kwargs: Any,
+    ) -> SuiteResult: ...
+
+
+__all__ = ["ScanAdapter", "third_party_scan"]