Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 43 additions & 26 deletions apps/daemon/src/run-retry-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,44 @@ function normalizeMaxAttempts(maxAttempts: number | undefined): number {
return Math.floor(maxAttempts);
}

function isTransientRetryCategory(
function transientSuppressedReason(
category: TrackingRunFailureCategory | undefined,
detail: TrackingRunFailureDetail | undefined,
stage: TrackingRunFailureStage | undefined,
): boolean {
if (category === 'rate_limit') return detail !== 'hard_quota';
if (category === 'upstream_unavailable') return true;
if (category === 'empty_output') return stage === undefined || stage === 'first_token_wait';
if (category === 'timeout') return stage === 'first_token_wait';
return false;
): TrackingRunRetrySuppressedReason | null {
if (category === undefined) return 'missing_failure_signal';
if (category === 'rate_limit') {
return detail === 'rate_limit_429' ? null : 'non_retryable_category';
}
if (category === 'upstream_unavailable') {
return detail === 'stream_disconnected' ||
detail === 'upstream_5xx' ||
detail === 'provider_high_demand' ||
detail === 'provider_routing_error' ||
detail === 'network_error'
? null
: 'non_retryable_category';
}
if (category === 'empty_output') {
return stage === undefined || stage === 'first_token_wait'
? null
: 'unsafe_failure_stage';
}
if (category === 'timeout') {
return stage === 'first_token_wait'
? null
: 'unsafe_failure_stage';
}
if (category === 'process_exit') {
return detail === 'agent_protocol_error' ||
detail === 'qoder_stop_sequence' ||
detail === 'session_resume_expired' ||
detail === 'stream_error' ||
detail === 'fatal_rpc_error'
? null
: 'non_retryable_category';
}
return 'non_retryable_category';
}

export function decideSafeRunRetry(
Expand Down Expand Up @@ -140,27 +168,16 @@ export function decideSafeRunRetry(
if (sideEffects.cancelRequested) return suppress('cancel_requested');

const failure = input.failure;
if (!failure) return suppress('missing_failure_signal');
if (failure?.failure_detail === 'hard_quota') return suppress('hard_quota');
if (
failure?.failure_category !== undefined &&
!isTransientRetryCategory(
failure.failure_category,
failure.failure_detail,
failure.failure_stage,
)
) {
return suppress('unsupported_category');
}
const transientReason = transientSuppressedReason(
failure.failure_category,
failure.failure_detail,
failure.failure_stage,
);
if (transientReason === 'non_retryable_category') return suppress(transientReason);
if (!failure?.retryable) return suppress('not_retryable');
if (
!isTransientRetryCategory(
failure.failure_category,
failure.failure_detail,
failure.failure_stage,
)
) {
return suppress('unsupported_category');
}
if (transientReason) return suppress(transientReason);
if (attemptCount >= retryMaxAttempts) return suppress('attempt_limit_reached');
if (sideEffects.userVisibleOutputSeen) return suppress('user_visible_output_seen');
if (sideEffects.toolCallSeen) return suppress('tool_call_seen');
Expand Down
84 changes: 83 additions & 1 deletion apps/daemon/tests/run-retry-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,76 @@ describe('decideSafeRunRetry', () => {
).toBe(true);
});

it('allows named transient process-exit details before side effects', () => {
for (const failure_detail of [
'agent_protocol_error',
'qoder_stop_sequence',
'session_resume_expired',
'stream_error',
'fatal_rpc_error',
] as const) {
expect(
decide({
failure: {
failure_category: 'process_exit',
failure_detail,
failure_stage: 'child_close',
retryable: true,
},
}),
).toMatchObject({
shouldRetry: true,
retryReason: 'transient_failure',
});
}
});

it('keeps upstream client errors out of the transient retry allowlist', () => {
expect(
decide({
failure: {
failure_category: 'upstream_unavailable',
failure_detail: 'upstream_client_error',
failure_stage: 'first_token_wait',
retryable: true,
},
}),
).toMatchObject({
shouldRetry: false,
retrySuppressedReason: 'non_retryable_category',
});
});

it('uses unsafe_failure_stage for transient categories after unsafe output phases', () => {
expect(
decide({
failure: {
failure_category: 'empty_output',
failure_detail: 'empty_output',
failure_stage: 'artifact_write',
retryable: true,
},
}),
).toMatchObject({
shouldRetry: false,
retrySuppressedReason: 'unsafe_failure_stage',
});

expect(
decide({
failure: {
failure_category: 'timeout',
failure_detail: 'inactivity_timeout',
failure_stage: 'child_close',
retryable: true,
},
}),
).toMatchObject({
shouldRetry: false,
retrySuppressedReason: 'unsafe_failure_stage',
});
});

it('does not retry successful or cancelled terminal results', () => {
expect(decide({ result: 'success' })).toMatchObject({
shouldRetry: false,
Expand Down Expand Up @@ -116,6 +186,18 @@ describe('decideSafeRunRetry', () => {
});
});

it('suppresses missing classifier signals separately from non-retryable failures', () => {
expect(
decideSafeRunRetry({
result: 'failed',
attemptCount: 0,
}),
).toMatchObject({
shouldRetry: false,
retrySuppressedReason: 'missing_failure_signal',
});
});

it('suppresses non-transient categories even when the classifier marks them retryable', () => {
for (const failure_category of [
'auth',
Expand All @@ -135,7 +217,7 @@ describe('decideSafeRunRetry', () => {
}),
).toMatchObject({
shouldRetry: false,
retrySuppressedReason: 'unsupported_category',
retrySuppressedReason: 'non_retryable_category',
});
}
});
Expand Down
3 changes: 3 additions & 0 deletions packages/contracts/src/analytics/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,9 @@ export type TrackingRunRetrySuppressedReason =
| 'not_failed'
| 'not_retryable'
| 'unsupported_category'
| 'non_retryable_category'
| 'unsafe_failure_stage'
| 'missing_failure_signal'
| 'hard_quota'
| 'attempt_limit_reached'
| 'cancel_requested'
Expand Down
Loading