diff --git a/apps/daemon/src/run-retry-policy.ts b/apps/daemon/src/run-retry-policy.ts index eb9940b75b7..4d6b577c629 100644 --- a/apps/daemon/src/run-retry-policy.ts +++ b/apps/daemon/src/run-retry-policy.ts @@ -103,16 +103,44 @@ function normalizeMaxAttempts(maxAttempts: number | undefined): number { return Math.floor(maxAttempts); } -function isTransientRetryCategory( +function transientSuppressedReason( category: TrackingRunFailureCategory | undefined, detail: TrackingRunFailureDetail | undefined, stage: TrackingRunFailureStage | undefined, -): boolean { - if (category === 'rate_limit') return detail !== 'hard_quota'; - if (category === 'upstream_unavailable') return true; - if (category === 'empty_output') return stage === undefined || stage === 'first_token_wait'; - if (category === 'timeout') return stage === 'first_token_wait'; - return false; +): TrackingRunRetrySuppressedReason | null { + if (category === undefined) return 'missing_failure_signal'; + if (category === 'rate_limit') { + return detail === 'rate_limit_429' ? null : 'non_retryable_category'; + } + if (category === 'upstream_unavailable') { + return detail === 'stream_disconnected' || + detail === 'upstream_5xx' || + detail === 'provider_high_demand' || + detail === 'provider_routing_error' || + detail === 'network_error' + ? null + : 'non_retryable_category'; + } + if (category === 'empty_output') { + return stage === undefined || stage === 'first_token_wait' + ? null + : 'unsafe_failure_stage'; + } + if (category === 'timeout') { + return stage === 'first_token_wait' + ? null + : 'unsafe_failure_stage'; + } + if (category === 'process_exit') { + return detail === 'agent_protocol_error' || + detail === 'qoder_stop_sequence' || + detail === 'session_resume_expired' || + detail === 'stream_error' || + detail === 'fatal_rpc_error' + ? null + : 'non_retryable_category'; + } + return 'non_retryable_category'; } export function decideSafeRunRetry( @@ -140,27 +168,16 @@ export function decideSafeRunRetry( if (sideEffects.cancelRequested) return suppress('cancel_requested'); const failure = input.failure; + if (!failure) return suppress('missing_failure_signal'); if (failure?.failure_detail === 'hard_quota') return suppress('hard_quota'); - if ( - failure?.failure_category !== undefined && - !isTransientRetryCategory( - failure.failure_category, - failure.failure_detail, - failure.failure_stage, - ) - ) { - return suppress('unsupported_category'); - } + const transientReason = transientSuppressedReason( + failure.failure_category, + failure.failure_detail, + failure.failure_stage, + ); + if (transientReason === 'non_retryable_category') return suppress(transientReason); if (!failure?.retryable) return suppress('not_retryable'); - if ( - !isTransientRetryCategory( - failure.failure_category, - failure.failure_detail, - failure.failure_stage, - ) - ) { - return suppress('unsupported_category'); - } + if (transientReason) return suppress(transientReason); if (attemptCount >= retryMaxAttempts) return suppress('attempt_limit_reached'); if (sideEffects.userVisibleOutputSeen) return suppress('user_visible_output_seen'); if (sideEffects.toolCallSeen) return suppress('tool_call_seen'); diff --git a/apps/daemon/tests/run-retry-policy.test.ts b/apps/daemon/tests/run-retry-policy.test.ts index 3d26408aada..0048f0da9d2 100644 --- a/apps/daemon/tests/run-retry-policy.test.ts +++ b/apps/daemon/tests/run-retry-policy.test.ts @@ -89,6 +89,76 @@ describe('decideSafeRunRetry', () => { ).toBe(true); }); + it('allows named transient process-exit details before side effects', () => { + for (const failure_detail of [ + 'agent_protocol_error', + 'qoder_stop_sequence', + 'session_resume_expired', + 'stream_error', + 'fatal_rpc_error', + ] as const) { + expect( + decide({ + failure: { + failure_category: 'process_exit', + failure_detail, + failure_stage: 'child_close', + retryable: true, + }, + }), + ).toMatchObject({ + shouldRetry: true, + retryReason: 'transient_failure', + }); + } + }); + + it('keeps upstream client errors out of the transient retry allowlist', () => { + expect( + decide({ + failure: { + failure_category: 'upstream_unavailable', + failure_detail: 'upstream_client_error', + failure_stage: 'first_token_wait', + retryable: true, + }, + }), + ).toMatchObject({ + shouldRetry: false, + retrySuppressedReason: 'non_retryable_category', + }); + }); + + it('uses unsafe_failure_stage for transient categories after unsafe output phases', () => { + expect( + decide({ + failure: { + failure_category: 'empty_output', + failure_detail: 'empty_output', + failure_stage: 'artifact_write', + retryable: true, + }, + }), + ).toMatchObject({ + shouldRetry: false, + retrySuppressedReason: 'unsafe_failure_stage', + }); + + expect( + decide({ + failure: { + failure_category: 'timeout', + failure_detail: 'inactivity_timeout', + failure_stage: 'child_close', + retryable: true, + }, + }), + ).toMatchObject({ + shouldRetry: false, + retrySuppressedReason: 'unsafe_failure_stage', + }); + }); + it('does not retry successful or cancelled terminal results', () => { expect(decide({ result: 'success' })).toMatchObject({ shouldRetry: false, @@ -116,6 +186,18 @@ describe('decideSafeRunRetry', () => { }); }); + it('suppresses missing classifier signals separately from non-retryable failures', () => { + expect( + decideSafeRunRetry({ + result: 'failed', + attemptCount: 0, + }), + ).toMatchObject({ + shouldRetry: false, + retrySuppressedReason: 'missing_failure_signal', + }); + }); + it('suppresses non-transient categories even when the classifier marks them retryable', () => { for (const failure_category of [ 'auth', @@ -135,7 +217,7 @@ describe('decideSafeRunRetry', () => { }), ).toMatchObject({ shouldRetry: false, - retrySuppressedReason: 'unsupported_category', + retrySuppressedReason: 'non_retryable_category', }); } }); diff --git a/packages/contracts/src/analytics/events.ts b/packages/contracts/src/analytics/events.ts index 8dd3340a1cc..fb5471c79b0 100644 --- a/packages/contracts/src/analytics/events.ts +++ b/packages/contracts/src/analytics/events.ts @@ -399,6 +399,9 @@ export type TrackingRunRetrySuppressedReason = | 'not_failed' | 'not_retryable' | 'unsupported_category' + | 'non_retryable_category' + | 'unsafe_failure_stage' + | 'missing_failure_signal' | 'hard_quota' | 'attempt_limit_reached' | 'cancel_requested'