Refine posthog telemetry for execute sandbox script (#3471)

Author: wwwillchen

src/__tests__/posthogTelemetry.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import {
   createExceptionFromTelemetry,
   getExceptionTelemetryContext,
+  shouldBypassNonProTelemetrySampling,
 } from "@/lib/posthogTelemetry";
 
 describe("createExceptionFromTelemetry", () => {
@@ -26,6 +27,74 @@ describe("createExceptionFromTelemetry", () => {
   });
 });
 
+describe("shouldBypassNonProTelemetrySampling", () => {
+  it("always sends sandbox.script.* events for non-Pro sampling", () => {
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "sandbox.script.completed",
+        properties: { chatId: 1, appId: 2 },
+      }),
+    ).toBe(true);
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "sandbox.script.truncated",
+        properties: { chatId: 1 },
+      }),
+    ).toBe(true);
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "sandbox.script.failed",
+        properties: { error: "Unexpected token" },
+      }),
+    ).toBe(true);
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "sandbox.script.timeout",
+        properties: { error: "Script timed out" },
+      }),
+    ).toBe(true);
+  });
+
+  it("does not bypass unrelated sandbox telemetry", () => {
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "sandbox.tool.unused_with_attachment",
+        properties: { chatId: 1 },
+      }),
+    ).toBe(false);
+  });
+
+  it("still bypasses sampling for error-shaped events", () => {
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "$exception",
+        properties: { exception_message: "boom" },
+      }),
+    ).toBe(true);
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "extra-files:error",
+        properties: {},
+      }),
+    ).toBe(true);
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "app:crash_detected",
+        properties: { error: true },
+      }),
+    ).toBe(true);
+  });
+
+  it("allows routine events to be sampled", () => {
+    expect(
+      shouldBypassNonProTelemetrySampling({
+        event: "chat:submit",
+        properties: { chatMode: "build" },
+      }),
+    ).toBe(false);
+  });
+});
+
 describe("getExceptionTelemetryContext", () => {
   it("removes exception payload fields before passing custom context to PostHog", () => {
     expect(

src/lib/posthogTelemetry.ts

@@ -1,5 +1,33 @@
 type TelemetryProperties = Record<string, unknown> | undefined;
 
+/** PostHog event shape used by renderer `before_send` sampling. */
+export type PostHogTelemetryEvent = {
+  event?: string;
+  properties?: TelemetryProperties;
+};
+
+/**
+ * Non-Pro telemetry sends only ~10% of events. These events are always sent.
+ * Keep `sandbox.script.*` here so script instrumentation is never sampled out.
+ */
+export function shouldBypassNonProTelemetrySampling(
+  event: PostHogTelemetryEvent | null | undefined,
+): boolean {
+  const eventName = event?.event;
+  const properties = event?.properties;
+
+  if (eventName?.startsWith("sandbox.script.")) {
+    return true;
+  }
+
+  return (
+    eventName === "$exception" ||
+    eventName?.toLowerCase().includes("error") === true ||
+    !!properties?.$exception_type ||
+    !!properties?.error
+  );
+}
+
 export function createExceptionFromTelemetry(properties: TelemetryProperties) {
   const exception = new Error(
     typeof properties?.exception_message === "string"

src/pro/main/ipc/handlers/local_agent/tools/execute_sandbox_script.ts

@@ -168,11 +168,6 @@ Paths are app-relative (including \`.dyad/media/<stored-name>\`), or attachment
       args.description?.trim() || "Run a read-only script",
 
     execute: async (args: ExecuteSandboxScriptArgs, ctx: AgentContext) => {
-      sendTelemetryEvent("sandbox.script.run", {
-        chatId: ctx.chatId,
-        appId: ctx.appId,
-      });
-
       try {
         const result = await runSandboxScript({
           appPath: ctx.appPath,

src/renderer.tsx

@@ -32,6 +32,7 @@ import { queryKeys } from "./lib/queryKeys";
 import {
   createExceptionFromTelemetry,
   getExceptionTelemetryContext,
+  shouldBypassNonProTelemetrySampling,
 } from "./lib/posthogTelemetry";
 
 // @ts-ignore
@@ -97,15 +98,13 @@ const posthogClient = posthog.init(
         event.properties["$ip"] = null;
       }
 
-      // For non-Pro users, only send 10% of events (but always send errors)
+      // For non-Pro users, only send 10% of events (but always send errors and
+      // sandbox.script.* instrumentation — see shouldBypassNonProTelemetrySampling).
       if (!isDyadProUser()) {
-        const isErrorEvent =
-          event?.event === "$exception" ||
-          event?.event?.toLowerCase().includes("error") ||
-          event?.properties?.$exception_type ||
-          event?.properties?.error;
-
-        if (!isErrorEvent && Math.random() > 0.1) {
+        if (
+          !shouldBypassNonProTelemetrySampling(event) &&
+          Math.random() > 0.1
+        ) {
           console.debug("Non-Pro user: sampling out event", event?.event);
           return null;
         }