feat(scan): add PostHog telemetry for scan entry points

libs/giskard-scan/src/giskard/scan/_telemetry_props.py

@@ -0,0 +1,90 @@
+"""Shape properties for PostHog scan telemetry (no user content or secrets)."""
+
+from __future__ import annotations
+
+import time
+from collections import Counter
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any
+
+from giskard.checks import SuiteResult
+from giskard.core import telemetry_capture, telemetry_run_context, telemetry_tag
+
+from .generators.base import ScenarioGenerator, TargetMode
+
+
+def generator_type_counts(generators: list[ScenarioGenerator]) -> dict[str, int]:
+    return dict(Counter(type(generator).__name__ for generator in generators))
+
+
+def scan_shape_properties(
+    *,
+    scan_type: str,
+    languages: list[str],
+    target_mode: TargetMode,
+    max_scenarios: int | None = None,
+    seed: int | None = None,
+    group_by: str | None = None,
+    parallel: bool | None = None,
+    max_concurrency: int | None = None,
+    generator_count: int | None = None,
+    generator_types: dict[str, int] | None = None,
+    scenario_count: int | None = None,
+    knowledge_base_document_count: int | None = None,
+    commercial_use: bool | None = None,
+    third_party_probes: list[str] | None = None,
+    third_party_tags: list[str] | None = None,
+) -> dict[str, Any]:
+    return {
+        "integration": "giskard-scan",
+        "scan_type": scan_type,
+        "languages": languages,
+        "target_mode": target_mode,
+        "max_scenarios": max_scenarios,
+        "seed": seed,
+        "group_by": group_by,
+        "parallel": parallel,
+        "max_concurrency": max_concurrency,
+        "generator_count": generator_count,
+        "generator_types": generator_types,
+        "scenario_count": scenario_count,
+        "knowledge_base_document_count": knowledge_base_document_count,
+        "commercial_use": commercial_use,
+        "third_party_probes": third_party_probes,
+        "third_party_tags": third_party_tags,
+    }
+
+
+def suite_result_properties(result: SuiteResult) -> dict[str, Any]:
+    return {
+        "passed_count": result.passed_count,
+        "failed_count": result.failed_count,
+        "errored_count": result.errored_count,
+        "skipped_count": result.skipped_count,
+        "scenario_count": len(result.results),
+    }
+
+
+@contextmanager
+def scan_telemetry_scope(
+    scan_type: str, shape_props: dict[str, Any]
+) -> Iterator[dict[str, Any]]:
+    """Open a telemetry scope for a scan run and capture started/finished events."""
+    with telemetry_run_context():
+        telemetry_tag("giskard_component", "scan")
+        telemetry_tag("giskard_operation", "scan_run")
+        telemetry_tag("scan_type", scan_type)
+        telemetry_capture("scan_run_started", properties=shape_props)
+        start_time = time.perf_counter()
+        finished_props: dict[str, Any] = {}
+        try:
+            yield finished_props
+        finally:
+            finished_props.setdefault(
+                "duration_ms", int((time.perf_counter() - start_time) * 1000)
+            )
+            telemetry_capture(
+                "scan_run_finished",
+                properties={**shape_props, **finished_props},
+            )

libs/giskard-scan/src/giskard/scan/integrations/_entry_point.py

@@ -4,6 +4,13 @@
 
 from giskard.checks import SuiteResult, Target, Trace
 
+from .._telemetry_props import (
+    scan_shape_properties,
+    scan_telemetry_scope,
+    suite_result_properties,
+)
+from ..generators.base import TargetMode
+
 
 async def third_party_scan[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
     target: Target[InputType, OutputType, TraceType],
@@ -40,17 +47,32 @@ async def third_party_scan[InputType, OutputType, TraceType: Trace](  # pyright:
     Returns:
         The completed suite result.
     """
-    if tool == "garak":
-        from .garak import GarakScanAdapter
-
-        # Garak has no TargetInfo concept: drop the context args so its
-        # run() signature is unaffected.
-        return await GarakScanAdapter().run(target, **kwargs)
-    elif tool == "lidar":
-        from .lidar import LidarScanAdapter
-
-        return await LidarScanAdapter().run(
-            target, description=description, languages=languages, **kwargs
-        )
-    else:
-        raise ValueError(f"Unknown tool {tool!r}. Available: ['garak', 'lidar']")
+    target_mode: TargetMode = kwargs.get("target_mode", "multiturn")
+    probes = kwargs.get("probes")
+    tags = kwargs.get("tags")
+    shape_props = scan_shape_properties(
+        scan_type=tool,
+        languages=languages or [],
+        target_mode=target_mode,
+        third_party_probes=probes,
+        third_party_tags=tags,
+    )
+
+    with scan_telemetry_scope(tool, shape_props) as finished:
+        if tool == "garak":
+            from .garak import GarakScanAdapter
+
+            # Garak has no TargetInfo concept: drop the context args so its
+            # run() signature is unaffected.
+            result = await GarakScanAdapter().run(target, **kwargs)
+        elif tool == "lidar":
+            from .lidar import LidarScanAdapter
+
+            result = await LidarScanAdapter().run(
+                target, description=description, languages=languages, **kwargs
+            )
+        else:
+            raise ValueError(f"Unknown tool {tool!r}. Available: ['garak', 'lidar']")
+        finished.update(suite_result_properties(result))
+
+    return result

libs/giskard-scan/src/giskard/scan/quality.py

@@ -5,6 +5,12 @@
 
 from giskard.checks import SuiteResult, Target, Trace
 
+from ._telemetry_props import (
+    generator_type_counts,
+    scan_shape_properties,
+    scan_telemetry_scope,
+    suite_result_properties,
+)
 from .catalog import generate_suite
 from .generators.base import TargetMode
 from .generators.knowledge_base import (
@@ -78,27 +84,51 @@ async def quality_scan[InputType, OutputType, TraceType: Trace](  # pyright: ign
         _warn_if_missing_knowledge_base(knowledge_base)
     )
 
+    generators = quality_suite_generator_registry.generators()
     suite = await generate_suite(
         description=description,
         languages=languages,
-        generators=quality_suite_generator_registry.generators(),
+        generators=generators,
         max_scenarios=max_scenarios,
         seed=seed,
         target_mode=target_mode,
         knowledge_base=knowledge_base,
     )
 
-    result: SuiteResult = await suite.run(
-        target,
+    shape_props = scan_shape_properties(
+        scan_type="quality",
+        languages=languages,
+        target_mode=target_mode,
+        max_scenarios=max_scenarios,
+        seed=seed,
+        group_by=group_by,
         parallel=parallel,
         max_concurrency=max_concurrency,
+        generator_count=len(generators),
+        generator_types=generator_type_counts(generators),
+        scenario_count=len(suite.scenarios),
+        knowledge_base_document_count=(
+            len(knowledge_base.documents) if knowledge_base is not None else 0
+        ),
     )
-    try:
-        recommendation = await generate_quality_recommendation(result)
-    except Exception:
-        logger.exception("Quality recommendation generation failed")
-        recommendation = ""
-    quality_result = result.model_copy(update={"recommendation": recommendation})
+
+    with scan_telemetry_scope("quality", shape_props) as finished:
+        result: SuiteResult = await suite.run(
+            target,
+            parallel=parallel,
+            max_concurrency=max_concurrency,
+        )
+
+        try:
+            recommendation = await generate_quality_recommendation(result)
+        except Exception:
+            logger.exception("Quality recommendation generation failed")
+            recommendation = ""
+        quality_result = result.model_copy(update={"recommendation": recommendation})
+
+        finished.update(suite_result_properties(quality_result))
+        finished["has_recommendation"] = bool(recommendation)
+
     quality_result.print_report(group_by=group_by)
     return quality_result
 

libs/giskard-scan/src/giskard/scan/vulnerability.py

@@ -2,6 +2,12 @@
 
 from giskard.checks import SuiteResult, Target, Trace
 
+from ._telemetry_props import (
+    generator_type_counts,
+    scan_shape_properties,
+    scan_telemetry_scope,
+    suite_result_properties,
+)
 from .catalog import generate_suite
 from .generators.adversarial import AdversarialScenarioGenerator
 from .generators.base import TargetMode
@@ -70,21 +76,40 @@ async def vulnerability_scan[InputType, OutputType, TraceType: Trace](  # pyrigh
     """
     opts: ResolvedScanOptions = {**_DEFAULT_SCAN_OPTIONS, **options}
 
+    generators = vulnerability_suite_generator_registry.generators(
+        commercial_use=opts["commercial_use"]
+    )
     suite = await generate_suite(
         description=description,
         languages=languages,
-        generators=vulnerability_suite_generator_registry.generators(
-            commercial_use=opts["commercial_use"]
-        ),
+        generators=generators,
         max_scenarios=opts["max_scenarios"],
         seed=opts["seed"],
         target_mode=target_mode,
     )
 
-    result = await suite.run(
-        target,
+    shape_props = scan_shape_properties(
+        scan_type="vulnerability",
+        languages=languages,
+        target_mode=target_mode,
+        max_scenarios=opts["max_scenarios"],
+        seed=opts["seed"],
+        group_by=opts["group_by"],
         parallel=opts["parallel"],
         max_concurrency=opts["max_concurrency"],
+        generator_count=len(generators),
+        generator_types=generator_type_counts(generators),
+        scenario_count=len(suite.scenarios),
+        commercial_use=opts["commercial_use"],
     )
+
+    with scan_telemetry_scope("vulnerability", shape_props) as finished:
+        result = await suite.run(
+            target,
+            parallel=opts["parallel"],
+            max_concurrency=opts["max_concurrency"],
+        )
+        finished.update(suite_result_properties(result))
+
     result.print_report(group_by=opts["group_by"])
     return result

libs/giskard-scan/tests/conftest.py

@@ -1,8 +1,15 @@
 import pytest
+from giskard.core import disable_telemetry
 from giskard.scan.quality import quality_suite_generator_registry
 from giskard.scan.vulnerability import vulnerability_suite_generator_registry
 
 
+@pytest.fixture(autouse=True)
+def _disable_telemetry():
+    """Disable telemetry for tests."""
+    disable_telemetry()
+
+
 @pytest.fixture
 def isolated_vulnerability_registry():
     """Snapshot and restore the vulnerability generator registry."""

libs/giskard-scan/tests/test_quality.py

@@ -227,6 +227,8 @@ async def test_quality_scan_warns_and_skips_empty_raw_knowledge_base(
     captured_knowledge_bases: list[object] = []
 
     class _FakeSuite:
+        scenarios: list[Scenario[Any, Any, Trace[Any, Any]]] = []
+
         async def run(
             self,
             target: object,
@@ -278,6 +280,8 @@ async def test_quality_scan_configures_knowledge_base_generator(
     printed_reports: list[str | None] = []
 
     class _FakeSuite:
+        scenarios: list[Scenario[Any, Any, Trace[Any, Any]]] = []
+
         async def run(
             self,
             target: object,