Skip to content

Commit 5e3cc96

Browse files
PerishCodeSiri-Ray
authored and
xxiaoxiong
committed
fix(retry): tighten transient retry policy (nexu-io#5025)
Co-authored-by: Siri-Ray <2667192167@qq.com>
1 parent 62114e1 commit 5e3cc96

3 files changed

Lines changed: 129 additions & 27 deletions

File tree

‎apps/daemon/src/run-retry-policy.ts‎

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -103,16 +103,44 @@ function normalizeMaxAttempts(maxAttempts: number | undefined): number {
103103
return Math.floor(maxAttempts);
104104
}
105105

106-
function isTransientRetryCategory(
106+
function transientSuppressedReason(
107107
category: TrackingRunFailureCategory | undefined,
108108
detail: TrackingRunFailureDetail | undefined,
109109
stage: TrackingRunFailureStage | undefined,
110-
): boolean {
111-
if (category === 'rate_limit') return detail !== 'hard_quota';
112-
if (category === 'upstream_unavailable') return true;
113-
if (category === 'empty_output') return stage === undefined || stage === 'first_token_wait';
114-
if (category === 'timeout') return stage === 'first_token_wait';
115-
return false;
110+
): TrackingRunRetrySuppressedReason | null {
111+
if (category === undefined) return 'missing_failure_signal';
112+
if (category === 'rate_limit') {
113+
return detail === 'rate_limit_429' ? null : 'non_retryable_category';
114+
}
115+
if (category === 'upstream_unavailable') {
116+
return detail === 'stream_disconnected' ||
117+
detail === 'upstream_5xx' ||
118+
detail === 'provider_high_demand' ||
119+
detail === 'provider_routing_error' ||
120+
detail === 'network_error'
121+
? null
122+
: 'non_retryable_category';
123+
}
124+
if (category === 'empty_output') {
125+
return stage === undefined || stage === 'first_token_wait'
126+
? null
127+
: 'unsafe_failure_stage';
128+
}
129+
if (category === 'timeout') {
130+
return stage === 'first_token_wait'
131+
? null
132+
: 'unsafe_failure_stage';
133+
}
134+
if (category === 'process_exit') {
135+
return detail === 'agent_protocol_error' ||
136+
detail === 'qoder_stop_sequence' ||
137+
detail === 'session_resume_expired' ||
138+
detail === 'stream_error' ||
139+
detail === 'fatal_rpc_error'
140+
? null
141+
: 'non_retryable_category';
142+
}
143+
return 'non_retryable_category';
116144
}
117145

118146
export function decideSafeRunRetry(
@@ -140,27 +168,16 @@ export function decideSafeRunRetry(
140168
if (sideEffects.cancelRequested) return suppress('cancel_requested');
141169

142170
const failure = input.failure;
171+
if (!failure) return suppress('missing_failure_signal');
143172
if (failure?.failure_detail === 'hard_quota') return suppress('hard_quota');
144-
if (
145-
failure?.failure_category !== undefined &&
146-
!isTransientRetryCategory(
147-
failure.failure_category,
148-
failure.failure_detail,
149-
failure.failure_stage,
150-
)
151-
) {
152-
return suppress('unsupported_category');
153-
}
173+
const transientReason = transientSuppressedReason(
174+
failure.failure_category,
175+
failure.failure_detail,
176+
failure.failure_stage,
177+
);
178+
if (transientReason === 'non_retryable_category') return suppress(transientReason);
154179
if (!failure?.retryable) return suppress('not_retryable');
155-
if (
156-
!isTransientRetryCategory(
157-
failure.failure_category,
158-
failure.failure_detail,
159-
failure.failure_stage,
160-
)
161-
) {
162-
return suppress('unsupported_category');
163-
}
180+
if (transientReason) return suppress(transientReason);
164181
if (attemptCount >= retryMaxAttempts) return suppress('attempt_limit_reached');
165182
if (sideEffects.userVisibleOutputSeen) return suppress('user_visible_output_seen');
166183
if (sideEffects.toolCallSeen) return suppress('tool_call_seen');

‎apps/daemon/tests/run-retry-policy.test.ts‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,76 @@ describe('decideSafeRunRetry', () => {
8989
).toBe(true);
9090
});
9191

92+
it('allows named transient process-exit details before side effects', () => {
93+
for (const failure_detail of [
94+
'agent_protocol_error',
95+
'qoder_stop_sequence',
96+
'session_resume_expired',
97+
'stream_error',
98+
'fatal_rpc_error',
99+
] as const) {
100+
expect(
101+
decide({
102+
failure: {
103+
failure_category: 'process_exit',
104+
failure_detail,
105+
failure_stage: 'child_close',
106+
retryable: true,
107+
},
108+
}),
109+
).toMatchObject({
110+
shouldRetry: true,
111+
retryReason: 'transient_failure',
112+
});
113+
}
114+
});
115+
116+
it('keeps upstream client errors out of the transient retry allowlist', () => {
117+
expect(
118+
decide({
119+
failure: {
120+
failure_category: 'upstream_unavailable',
121+
failure_detail: 'upstream_client_error',
122+
failure_stage: 'first_token_wait',
123+
retryable: true,
124+
},
125+
}),
126+
).toMatchObject({
127+
shouldRetry: false,
128+
retrySuppressedReason: 'non_retryable_category',
129+
});
130+
});
131+
132+
it('uses unsafe_failure_stage for transient categories after unsafe output phases', () => {
133+
expect(
134+
decide({
135+
failure: {
136+
failure_category: 'empty_output',
137+
failure_detail: 'empty_output',
138+
failure_stage: 'artifact_write',
139+
retryable: true,
140+
},
141+
}),
142+
).toMatchObject({
143+
shouldRetry: false,
144+
retrySuppressedReason: 'unsafe_failure_stage',
145+
});
146+
147+
expect(
148+
decide({
149+
failure: {
150+
failure_category: 'timeout',
151+
failure_detail: 'inactivity_timeout',
152+
failure_stage: 'child_close',
153+
retryable: true,
154+
},
155+
}),
156+
).toMatchObject({
157+
shouldRetry: false,
158+
retrySuppressedReason: 'unsafe_failure_stage',
159+
});
160+
});
161+
92162
it('does not retry successful or cancelled terminal results', () => {
93163
expect(decide({ result: 'success' })).toMatchObject({
94164
shouldRetry: false,
@@ -116,6 +186,18 @@ describe('decideSafeRunRetry', () => {
116186
});
117187
});
118188

189+
it('suppresses missing classifier signals separately from non-retryable failures', () => {
190+
expect(
191+
decideSafeRunRetry({
192+
result: 'failed',
193+
attemptCount: 0,
194+
}),
195+
).toMatchObject({
196+
shouldRetry: false,
197+
retrySuppressedReason: 'missing_failure_signal',
198+
});
199+
});
200+
119201
it('suppresses non-transient categories even when the classifier marks them retryable', () => {
120202
for (const failure_category of [
121203
'auth',
@@ -135,7 +217,7 @@ describe('decideSafeRunRetry', () => {
135217
}),
136218
).toMatchObject({
137219
shouldRetry: false,
138-
retrySuppressedReason: 'unsupported_category',
220+
retrySuppressedReason: 'non_retryable_category',
139221
});
140222
}
141223
});

‎packages/contracts/src/analytics/events.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,9 @@ export type TrackingRunRetrySuppressedReason =
399399
| 'not_failed'
400400
| 'not_retryable'
401401
| 'unsupported_category'
402+
| 'non_retryable_category'
403+
| 'unsafe_failure_stage'
404+
| 'missing_failure_signal'
402405
| 'hard_quota'
403406
| 'attempt_limit_reached'
404407
| 'cancel_requested'

0 commit comments

Comments
 (0)