Skip to content

Commit cffd4f0

Browse files
committed
fix: address Scroll pre-commit checks
1 parent 77123b4 commit cffd4f0

13 files changed

Lines changed: 193 additions & 120 deletions

README.md

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -45,16 +45,12 @@ serving, rate limits, and stochastic generation can affect reruns.
4545

4646
## Evaluation and reproduction
4747

48-
Evaluation is conducted with
49-
[AgentZero](https://github.com/agentscope-ai/AgentZero), our open-source
50-
Harbor-based evaluation framework, so benchmark-specific adapters, task
51-
definitions, verifiers, and generated runs remain separate from the Scroll
52-
implementation.
53-
54-
AgentZero pins this repository as a Git submodule and records the exact QwenPaw
55-
commit used by an evaluation. It provides the Harbor workflows for
56-
LongMemEval, BEAM, RULER, and LOCA, including environment construction,
57-
parallel trials, traces, and scoring.
48+
Implementations for reproducing the reported results are available in
49+
[AgentZero](https://github.com/agentscope-ai/AgentZero). AgentZero uses Harbor
50+
and pins this repository as a Git submodule, recording the exact QwenPaw commit
51+
used by each evaluation. It provides reproduction workflows for LongMemEval,
52+
BEAM, RULER, and LOCA, including environment construction, parallel trials,
53+
traces, and scoring.
5854

5955
```bash
6056
git clone --recurse-submodules https://github.com/agentscope-ai/AgentZero.git
@@ -66,42 +62,48 @@ source .venv/bin/activate
6662
uv pip install "harbor==0.18.0" "ijson>=3.3.0"
6763
```
6864

69-
Build the pinned Scroll/QwenPaw wheel, then generate a small LongMemEval task
70-
set:
65+
Build the pinned Scroll/QwenPaw wheel:
7166

7267
```bash
7368
uv build --project qwenpaw --wheel --out-dir dist
7469
export QWENPAW_WHEEL="$(find "$PWD/dist" -name 'qwenpaw-*.whl' -print -quit)"
7570
export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}"
76-
77-
python scripts/download_longmemeval_data.py --dataset oracle
78-
python benchmarks/longmemeval/generate.py \
79-
benchmarks/longmemeval/data/longmemeval_oracle.json \
80-
--split smoke \
81-
--output local-tasks/longmemeval \
82-
--limit 3
8371
```
8472

85-
Supply your own evaluated-model and judge API keys, base URLs, and model
86-
identifiers, then launch Harbor:
73+
Choose a benchmark (`longmemeval`, `beam`, `ruler`, or `loca`) and prepare its
74+
task packages using the corresponding AgentZero instructions. Then select its
75+
task path and adapter:
76+
77+
| Benchmark | Task path | Adapter |
78+
| --- | --- | --- |
79+
| LongMemEval | `local-tasks/longmemeval/{split}` | `adapters.qwenpaw.longmemeval:QwenPawLongMemEvalAgent` |
80+
| BEAM | `local-tasks/beam/{task}` | `adapters.qwenpaw.beam:QwenPawBeamAgent` |
81+
| RULER | `local-tasks/ruler` | `adapters.qwenpaw.ruler:QwenPawRulerAgent` |
82+
| LOCA | `local-tasks/loca/{task}` | `adapters.qwenpaw.loca:QwenPawLOCAAgent` |
83+
84+
Supply your own model API key, base URL, and model identifier, replace the
85+
placeholders below, and launch Harbor:
8786

8887
```bash
88+
export BENCHMARK=YOUR_BENCHMARK
89+
export TASK_PATH=YOUR_TASK_PATH
90+
export ADAPTER=YOUR_ADAPTER
91+
8992
harbor run \
90-
--job-name longmemeval-scroll-smoke \
91-
-p local-tasks/longmemeval/smoke \
92-
-a adapters.qwenpaw.longmemeval:QwenPawLongMemEvalAgent \
93+
--job-name "scroll-${BENCHMARK}" \
94+
-p "$TASK_PATH" \
95+
-a "$ADAPTER" \
9396
-m YOUR_PROVIDER_ID/YOUR_MODEL_ID \
9497
--ae QWENPAW_WHEEL="$QWENPAW_WHEEL" \
9598
--ae QWENPAW_MODEL_API_KEY="$QWENPAW_MODEL_API_KEY" \
9699
--ae QWENPAW_MODEL_BASE_URL="$QWENPAW_MODEL_BASE_URL" \
97-
--ve LONGMEMEVAL_JUDGE_API_KEY="$LONGMEMEVAL_JUDGE_API_KEY" \
98-
--ve LONGMEMEVAL_JUDGE_BASE_URL="$LONGMEMEVAL_JUDGE_BASE_URL" \
99-
--ve LONGMEMEVAL_JUDGE_MODEL="$LONGMEMEVAL_JUDGE_MODEL" \
100-
-n 3
100+
-n 8
101101
```
102102

103-
See the AgentZero README for Oracle validation, full S/M runs, other
104-
benchmarks, concurrency guidance, and result inspection.
103+
For judged benchmarks, also pass the benchmark-specific judge API key, base
104+
URL, and model identifier documented in AgentZero. See the AgentZero README
105+
for task generation, Oracle validation, concurrency guidance, and result
106+
inspection.
105107

106108
## License
107109

src/qwenpaw/agents/context/scroll/repl.py

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import sys
2121
import uuid
2222
from pathlib import Path
23-
from types import SimpleNamespace
2423
from typing import Any, Optional
2524

2625
from agentscope.message import TextBlock, ToolResultState
@@ -207,25 +206,31 @@ async def _run_kernel(source: str) -> tuple[str, bool] | None:
207206
except KernelUnavailableError:
208207
return None
209208

210-
async def _blocked_dispatch(
211-
exposed_path: str,
212-
_args: dict[str, Any],
213-
*,
214-
kernel_task_id: str,
215-
) -> Any:
216-
raise ToolForwardingError(
217-
f"failed|paw.tools.{exposed_path} is not available inside "
218-
"recall_history_python cells; query via ms directly, or use "
219-
"repl_exec for governed tool calls",
220-
)
221-
222209
# Shim bridge: carrying the handle's current specs makes the manager's
223210
# spec refresh a no-op, and recall cells stay pure Python + ms.
224-
bridge = SimpleNamespace(
225-
specs=handle.specs,
226-
is_read_only=lambda _path: False,
227-
dispatch=_blocked_dispatch,
228-
)
211+
class _BlockedBridge:
212+
def __init__(self) -> None:
213+
self.specs = handle.specs
214+
215+
@staticmethod
216+
def is_read_only(_exposed_path: str) -> bool:
217+
return False
218+
219+
@staticmethod
220+
async def dispatch(
221+
exposed_path: str,
222+
_args: dict[str, Any],
223+
*,
224+
kernel_task_id: str,
225+
) -> Any:
226+
del kernel_task_id
227+
raise ToolForwardingError(
228+
f"failed|paw.tools.{exposed_path} is not available inside "
229+
"recall_history_python cells; query via ms directly, or "
230+
"use repl_exec for governed tool calls",
231+
)
232+
233+
bridge = _BlockedBridge()
229234
try:
230235
result = await manager.execute(
231236
handle,

src/qwenpaw/repl/errors.py

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,14 @@
7373
"auth_missing": (
7474
"Credentials are missing. Ask the user to authenticate; do not retry."
7575
),
76-
"rate_limited": "Wait or reduce call frequency, then retry a limited number of times.",
76+
"rate_limited": (
77+
"Wait or reduce call frequency, then retry a limited number of times."
78+
),
7779
"timeout": "The call timed out. Reduce the workload or retry once.",
78-
"interrupted": "The cell was interrupted. Variables are retained; retry if appropriate.",
80+
"interrupted": (
81+
"The cell was interrupted. Variables are retained; retry if "
82+
"appropriate."
83+
),
7984
"kernel_restarted": (
8085
"The kernel restarted and variables were lost. Re-run the required "
8186
"setup cells or restore persisted variables."
@@ -114,10 +119,10 @@ def make_error(
114119
}
115120

116121

117-
def classify_exception(exc: BaseException) -> dict[str, Any]:
122+
def classify_exception( # pylint: disable=too-many-return-statements
123+
exc: BaseException,
124+
) -> dict[str, Any]:
118125
"""Map a Python exception raised inside a cell to a structured error."""
119-
import ast # noqa: F401 (kept local to avoid import-time cost)
120-
121126
from .proxy_runtime import PawToolError
122127

123128
if isinstance(exc, PawToolError):

src/qwenpaw/repl/exec_server.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,10 @@ def _execute_tree(
287287
if prefix.body:
288288
exec(compile(prefix, "<cell>", "exec"), namespace) # noqa: S102
289289
expression = ast.Expression(tree.body[-1].value)
290-
value = eval(compile(expression, "<cell>", "eval"), namespace) # noqa: S307
290+
value = eval(
291+
compile(expression, "<cell>", "eval"),
292+
namespace,
293+
) # noqa: S307
291294
if value is not None:
292295
rendered = render_last_expression(value, display)
293296
if rendered is not None:
@@ -402,6 +405,7 @@ def __init__(self, exec_id: str) -> None:
402405
self.interrupt_injected = False
403406

404407

408+
# pylint: disable-next=too-many-branches,too-many-statements
405409
def _serve_message_loop(
406410
inbox: queue.Queue,
407411
channel: StdioKernelChannel,

src/qwenpaw/repl/governance_bridge.py

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,15 @@
1414
import json
1515
import mimetypes
1616
import uuid
17-
from collections.abc import Iterator, Mapping
17+
from collections.abc import Iterator, Mapping, Sequence
1818
from dataclasses import dataclass
1919
from pathlib import Path
2020
from typing import Any
2121

2222
from .proxy_runtime import sanitize_name
2323

2424
EXCLUDED_TOOLS = frozenset(
25-
{"repl_exec", "execute_python", "execute_python_code"}
25+
{"repl_exec", "execute_python", "execute_python_code"},
2626
)
2727

2828

@@ -35,9 +35,9 @@ class CodeProvenance:
3535
provenance: str = "code"
3636

3737

38-
_CURRENT_PROVENANCE: contextvars.ContextVar[CodeProvenance | None] = (
39-
contextvars.ContextVar("qwenpaw_repl_provenance", default=None)
40-
)
38+
_CURRENT_PROVENANCE: contextvars.ContextVar[
39+
CodeProvenance | None
40+
] = contextvars.ContextVar("qwenpaw_repl_provenance", default=None)
4141

4242

4343
def get_code_provenance() -> CodeProvenance | None:
@@ -94,7 +94,7 @@ def _mcp_path(tool: Any, fallback_name: str) -> str | None:
9494
if capability is not None and getattr(capability, "protocol", "") == "mcp":
9595
server = sanitize_name(str(getattr(capability, "driver_name", "mcp")))
9696
original = sanitize_name(
97-
str(getattr(capability, "name", fallback_name))
97+
str(getattr(capability, "name", fallback_name)),
9898
)
9999
return f"mcp.{server}.{original}"
100100
if bool(getattr(tool, "is_mcp", False)):
@@ -182,7 +182,7 @@ def __init__(
182182
agent_state: Any,
183183
workspace: Path,
184184
workspace_id: str,
185-
specs: list[Mapping[str, Any]],
185+
specs: Sequence[Mapping[str, Any]],
186186
) -> None:
187187
self.toolkit = toolkit
188188
self.agent_state = agent_state
@@ -263,24 +263,27 @@ def _resolve_tool(self, exposed_path: str) -> str:
263263
return name
264264

265265
def _save_media(
266-
self, block: Any, call_id: str, position: int
266+
self,
267+
block: Any,
268+
call_id: str,
269+
position: int,
267270
) -> dict[str, str]:
268271
source = _block_attr(block, "source")
269272
media_type = str(
270-
_block_attr(source, "media_type", "application/octet-stream")
273+
_block_attr(source, "media_type", "application/octet-stream"),
271274
)
272275
data = _block_attr(source, "data")
273276
if not isinstance(data, str):
274277
raise ToolForwardingError(
275-
"binary tool result did not contain base64 data"
278+
"binary tool result did not contain base64 data",
276279
)
277280
relative = Path("out") / (
278281
f"tool_{call_id}_{position}{_media_extension(media_type)}"
279282
)
280283
destination = (self.workspace / relative).resolve()
281284
if self.workspace not in destination.parents:
282285
raise ToolForwardingError(
283-
"media result path escaped the workspace"
286+
"media result path escaped the workspace",
284287
)
285288
destination.parent.mkdir(parents=True, exist_ok=True)
286289
destination.write_bytes(base64.b64decode(data, validate=True))

src/qwenpaw/repl/kernel_manager.py

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,12 @@
1818
import uuid
1919
from dataclasses import dataclass, field
2020
from pathlib import Path
21-
from typing import Any
21+
from typing import Any, Protocol
2222

2323
from ..sandbox import MountSpec, SandboxConfig, SandboxMode
2424
from .backend import resolve_backend_kind
2525
from .errors import classify_tool_error, make_error
26-
from .governance_bridge import GovernanceBridge, ToolForwardingError
26+
from .governance_bridge import ToolForwardingError
2727
from .output_policy import DEFAULT_STDOUT_LIMIT
2828
from .persistence import latest_snapshot_dir
2929
from .protocol import (
@@ -45,6 +45,24 @@
4545
RESTORE_ACK_TIMEOUT = 15.0
4646

4747

48+
class KernelToolBridge(Protocol):
49+
"""Tool-forwarding interface required while servicing a kernel cell."""
50+
51+
specs: list[dict[str, Any]]
52+
53+
def is_read_only(self, exposed_path: str) -> bool:
54+
"""Return whether a tool may be forwarded concurrently."""
55+
56+
async def dispatch(
57+
self,
58+
exposed_path: str,
59+
args: dict[str, Any],
60+
*,
61+
kernel_task_id: str,
62+
) -> Any:
63+
"""Forward a governed tool call."""
64+
65+
4866
class KernelUnavailableError(RuntimeError):
4967
"""The REPL cannot safely start on this platform or configuration."""
5068

@@ -387,7 +405,7 @@ def _bwrap_supports_clearenv(executable: str) -> bool:
387405
return _CLEARENV_SUPPORTED
388406

389407

390-
def _bubblewrap_command(
408+
def _bubblewrap_command( # pylint: disable=too-many-branches
391409
config: SandboxConfig,
392410
argv: list[str],
393411
) -> list[str]:
@@ -468,13 +486,12 @@ def _seatbelt_command(
468486
from ..sandbox.macos_sandbox import MacOSSandbox
469487

470488
sandbox = MacOSSandbox(config)
471-
profile = (
472-
sandbox._compile_seatbelt_profile()
473-
) # pylint: disable=protected-access
489+
# pylint: disable-next=protected-access
490+
profile = sandbox._compile_seatbelt_profile()
474491
executable = shutil.which("sandbox-exec")
475492
if executable is None:
476493
raise KernelUnavailableError(
477-
"sandbox-exec disappeared after capability probe"
494+
"sandbox-exec disappeared after capability probe",
478495
)
479496
return [executable, "-p", profile, *argv]
480497

@@ -771,7 +788,9 @@ async def _maybe_restore_snapshot(
771788
)
772789

773790
async def _send(
774-
self, handle: KernelHandle, message: dict[str, Any]
791+
self,
792+
handle: KernelHandle,
793+
message: dict[str, Any],
775794
) -> None:
776795
if (
777796
handle.process.stdin is None
@@ -828,11 +847,11 @@ async def _update_specs(
828847
handle.specs = stable_specs
829848
handle.specs_hash = new_hash
830849

831-
async def execute(
850+
async def execute( # pylint: disable=too-many-branches,too-many-statements
832851
self,
833852
handle: KernelHandle,
834853
code: str,
835-
bridge: GovernanceBridge,
854+
bridge: KernelToolBridge,
836855
*,
837856
timeout: float = DEFAULT_EXEC_TIMEOUT,
838857
display: str = "summary",
@@ -1043,7 +1062,7 @@ async def respond_interrupted(
10431062
exec_id=exec_id,
10441063
ok=result.ok,
10451064
error_kind=(
1046-
result.error.get("kind")
1065+
str(result.error.get("kind") or "")
10471066
if result.error
10481067
else ""
10491068
),
@@ -1154,7 +1173,7 @@ async def _drain_background_cell(
11541173
if handle.process.returncode is not None:
11551174
raise KernelCrashedError(
11561175
self._crash_message(handle),
1157-
)
1176+
) from None
11581177
continue
11591178
message_type = message["type"]
11601179
if message_type == "log":

0 commit comments

Comments
 (0)