From e0d62442ced3b1631b031c79bb7945816f5138b9 Mon Sep 17 00:00:00 2001 From: Doggie B <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:45:43 -0400 Subject: [PATCH 1/6] feat(runner): add BacktestRunner w/ `silverback test` command --- setup.py | 1 + silverback/__init__.py | 1 + silverback/_cli.py | 18 +++++ silverback/pytest.py | 142 ++++++++++++++++++++++++++++++++++++++ silverback/runner.py | 141 +++++++++++++++++++++++++++++++++++-- tests/backtest_merge.yaml | 7 ++ 6 files changed, 303 insertions(+), 7 deletions(-) create mode 100644 silverback/pytest.py create mode 100644 tests/backtest_merge.yaml diff --git a/setup.py b/setup.py index eb5ad174..2385e3ba 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ ], entry_points={ "console_scripts": ["silverback=silverback._cli:cli"], + "pytest11": ["silverback_test=silverback.pytest"], }, python_requires=">=3.10,<4", extras_require=extras_require, diff --git a/silverback/__init__.py b/silverback/__init__.py index 1f55c662..75a3e070 100644 --- a/silverback/__init__.py +++ b/silverback/__init__.py @@ -22,6 +22,7 @@ def __getattr__(name: str): __all__ = [ "StateSnapshot", + "BacktestRunner", "CircuitBreaker", "SilverbackBot", "SilverbackException", diff --git a/silverback/_cli.py b/silverback/_cli.py index aca2aeb6..21eeb5af 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -1,11 +1,13 @@ import asyncio import os +import sys from collections import defaultdict from datetime import datetime, timedelta, timezone from pathlib import Path from typing import TYPE_CHECKING, Optional import click +import pytest import yaml # type: ignore[import-untyped] from ape import Contract, convert from ape.cli import ( @@ -15,6 +17,7 @@ account_option, ape_cli_context, network_option, + verbosity_option, ) from ape.exceptions import Abort, ApeException, ConversionError from ape.logging import LogLevel @@ -182,6 +185,21 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, debug, b ) +@cli.command( + section="Local Commands", + add_help_option=False, # NOTE: This allows pass-through to pytest's help + short_help="Run bot backtests (`tests/**/backtest_*.yaml`)", + context_settings=dict(ignore_unknown_options=True), +) +@ape_cli_context() +@verbosity_option() +@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED) +def test(cli_ctx, pytest_args): + if return_code := pytest.main([*pytest_args], ["silverback.pytest"]): + # only exit with non-zero status to make testing easier + sys.exit(return_code) + + @cli.command(section="Cloud Commands (https://silverback.apeworx.io)") @auth_required def login(auth: "FiefAuth"): diff --git a/silverback/pytest.py b/silverback/pytest.py new file mode 100644 index 00000000..9b5f3c21 --- /dev/null +++ b/silverback/pytest.py @@ -0,0 +1,142 @@ +import asyncio +import os +from functools import cached_property +from pathlib import Path + +import pytest +import yaml # type: ignore[import] +from ape import networks + +from silverback._importer import import_from_string +from silverback.exceptions import SilverbackException +from silverback.recorder import TaskResult +from silverback.runner import BacktestRunner + + +class AssertionViolation(SilverbackException): + pass + + +def pytest_collect_file(parent, file_path): + if file_path.suffix == ".yaml" and file_path.name.startswith("backtest"): + return BacktestFile.from_parent(parent, path=file_path) + + +class BacktestFile(pytest.File): + def collect(self): + raw = yaml.safe_load(self.path.open()) + if not (network_triple := raw.get("network")): + raise ValueError(f"{self.path} is missing key 'network'.") + + start_block = raw.get("start_block", 0) + stop_block = raw.get("stop_block", -1) + assertion_checks = raw.get("assertions", {}) + + # NOTE: Use default assumption rest of CLI uses + raw_bot_paths = raw.get("bots", "bot:bot") + if raw_bot_paths == "*": + raw_bot_paths = list(f.stem for f in (Path.cwd() / "bots").glob("*.py")) + + if isinstance(raw_bot_paths, list): + for bot_path in raw_bot_paths: + if ":" in bot_path: + bot_path, bot_name = bot_path.split(":") + else: + bot_name = "bot" + + yield BacktestItem.from_parent( + self, + name=f"{self.path.stem}[{bot_path}]", + file_path=self.path, + bot_path=bot_path, + bot_name=bot_name, + network_triple=network_triple, + start_block=start_block, + stop_block=stop_block, + assertion_checks=assertion_checks, + ) + + else: + if ":" in raw_bot_paths: + bot_path, bot_name = raw_bot_paths.split(":") + else: + bot_path = raw_bot_paths + bot_name = "bot" + + yield BacktestItem.from_parent( + self, + name=self.path.stem, + file_path=self.path, + bot_path=bot_path, + bot_name=bot_name, + network_triple=network_triple, + start_block=start_block, + stop_block=stop_block, + assertion_checks=assertion_checks, + ) + + +class BacktestItem(pytest.Item): + def __init__( + self, + *, + file_path, + bot_path, + bot_name, + network_triple, + start_block, + stop_block, + assertion_checks, + **kwargs, + ): + super().__init__(**kwargs) + self.file_path = file_path + self.bot_path = bot_path + self.bot_name = bot_name + self.network_triple = network_triple + self.start_block = start_block + self.stop_block = stop_block + self.assertion_checks = assertion_checks + + self.assertion_failures = 0 + self.overruns = 0 + self.total_tasks = 0 + self.task_failures = 0 + + def check_assertions(self, result: TaskResult): + self.total_tasks += 1 + + if result.execution_time > float(self.assertion_checks.get("execution_time", "inf")): + self.overruns += 1 + + if result.error: + self.task_failures += 1 + + @cached_property + def runner(self) -> BacktestRunner: + # NOTE: Set parameters for loading settings properly + os.environ["SILVERBACK_BOT_NAME"] = self.bot_path + with networks.parse_network_choice(self.network_triple): + # NOTE: Loading bot requires a network connection + bot = import_from_string(f"{self.bot_path}:{self.bot_name}") + + return BacktestRunner( + bot, + start_block=self.start_block, + stop_block=self.stop_block, + network_triple=self.network_triple, + check_assertions=self.check_assertions, + ) + + def runtest(self): + asyncio.run(self.runner.run()) + + # Test is over, check test status (if it completed) + self.raise_run_status() + + def raise_run_status(self): + if self.overruns > 0 or self.assertion_failures > 0: + raise AssertionViolation() + + def reportinfo(self): + return self.path, None, self.name diff --git a/silverback/runner.py b/silverback/runner.py index b90b84d8..a5e94e3b 100644 --- a/silverback/runner.py +++ b/silverback/runner.py @@ -1,11 +1,10 @@ import asyncio import signal import sys -from abc import ABC, abstractmethod -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable import quattro -from ape import chain +from ape.contracts import ContractEvent from ape.logging import logger from ape.utils import ManagerAccessMixin from ape_ethereum.ecosystem import keccak @@ -30,8 +29,11 @@ from .types import TaskType from .utils import async_wrap_iter, run_taskiq_task_group_wait_results, run_taskiq_task_wait_result +if TYPE_CHECKING: + pass -class BaseRunner(ABC): + +class BaseRunner: def __init__( self, # TODO: Make fully stateless by replacing `bot` with `broker` and `identifier` @@ -99,17 +101,17 @@ async def _checkpoint( else: await self.datastore.save(result.return_value) - @abstractmethod async def _block_task(self, task_data: TaskData) -> asyncio.Task | None: """ Handle a block_handler task """ + raise NotImplementedError - @abstractmethod async def _event_task(self, task_data: TaskData) -> asyncio.Task | None: """ Handle an event handler task for the given contract event """ + raise NotImplementedError async def run(self, *runtime_tasks: asyncio.Task | Callable[[], asyncio.Task]): """ @@ -381,7 +383,7 @@ async def _block_task(self, task_data: TaskData) -> asyncio.Task: async def block_handler(): async for block in async_wrap_iter( - chain.blocks.poll_blocks( + self.chain_manager.blocks.poll_blocks( # NOTE: No start block because we should begin polling from head new_block_timeout=new_block_timeout, ) @@ -425,3 +427,128 @@ async def log_handler(): await self._checkpoint(last_block_processed=event.block_number) return asyncio.create_task(log_handler()) + + +class BacktestRunner(BaseRunner, ManagerAccessMixin): + def __init__( + self, + bot: SilverbackBot, + start_block: int, + stop_block: int, + network_triple: str, + check_assertions: Callable[[TaskResult], None], + ): + super().__init__(bot) + self.start_block = start_block + self.stop_block = stop_block + self.network_triple = network_triple + self.check_assertions = check_assertions + + async def _handle_task(self, task: AsyncTaskiqTask): + self.check_assertions(TaskResult.from_taskiq(await task.wait_result())) + + def _event_from_taskdata(self, task_data: TaskData) -> ContractEvent: + if not (event_signature := task_data.labels.get("event_signature")): + raise StartupFailure("No Event Signature provided.") + + event_abi = EventABI.from_signature(event_signature) + + if not (contract_address := task_data.labels.get("contract_address")): + raise StartupFailure("Contract instance required.") + + events = self.chain_manager.contracts.instance_at(contract_address)._events_.get( + event_abi.name, [] + ) + + assert len(events) == 1, "Could not find event" + return events[0] + + async def run(self, *runtime_tasks: asyncio.Task | Callable[[], asyncio.Task]): + # Initialize broker (run worker startup events) + await self.bot.broker.startup() + + startup_state = StateSnapshot( + # TODO: Migrate these to parameters (remove explicitly from state) + last_block_seen=self.start_block - 1, + last_block_processed=self.start_block - 1, + ) # Use snapshot starting at previous block + + # Send startup state to bot + await self.run_system_task(TaskType.SYSTEM_LOAD_SNAPSHOT, startup_state) + + # Execute Silverback startup tasks before we enter into runtime + if startup_tasks_taskdata := await self.run_system_task( + TaskType.SYSTEM_USER_TASKDATA, TaskType.STARTUP + ): + # NOTE: Must start at block before start of test case period + with self.network_manager.fork(block_number=self.start_block - 1) as provider: + # NOTE: Do not automatically mine new transactions, tester will mine manually + provider.auto_mine = False + + startup_task_results = await run_taskiq_task_group_wait_results( + (map(self._create_task_kicker, startup_tasks_taskdata)), startup_state + ) + + assert not any(result.is_err for result in startup_task_results), "\n".join( + str(result.error) for result in startup_task_results if result.is_err + ) + + # Get all the user runtime tasks + new_block_tasks_taskdata = await self.run_system_task( + TaskType.SYSTEM_USER_TASKDATA, TaskType.NEW_BLOCK + ) + block_handlers = ( + self._create_task_kicker(taskdata) for taskdata in new_block_tasks_taskdata + ) + + event_log_tasks_taskdata = await self.run_system_task( + TaskType.SYSTEM_USER_TASKDATA, TaskType.EVENT_LOG + ) + + event_handlers = [ + (self._event_from_taskdata(taskdata), self._create_task_kicker(taskdata)) + for taskdata in event_log_tasks_taskdata + ] + + assert len(new_block_tasks_taskdata) + len(event_log_tasks_taskdata) > 0, "No Runtime Tasks" + + # Main test loop + for block_number in range(self.start_block, self.stop_block): + with self.network_manager.fork(block_number=block_number) as provider: + + # Run all blocks handlers first with next block + block = provider.get_block(block_id=block_number) + results = await quattro.gather(*(handler.kiq(block) for handler in block_handlers)) + await quattro.gather(*(map(self._handle_task, results))) + + # Then trigger all event log handlers for logs in receipts (confirmation order) + for txn in block.transactions: + receipt = provider.get_receipt(txn.hash) + + if tasks := list( + handler.kiq(log) + for event, handler in event_handlers + for log in event.from_receipt(receipt) + ): + results = await quattro.gather(*tasks) + await quattro.gather(*(map(self._handle_task, results))) + + # NOTE: Finished executing range up to right before `stop_block` + + # Execute all shutdown task(s) before shutting down the broker and bot + shutdown_tasks_taskdata = await self.run_system_task( + TaskType.SYSTEM_USER_TASKDATA, TaskType.SHUTDOWN + ) + + if shutdown_tasks_taskdata: + with self.network_manager.fork(block_number=self.stop_block) as provider: + shutdown_task_results = await run_taskiq_task_group_wait_results( + map(self._create_task_kicker, shutdown_tasks_taskdata) + ) + + assert not any(result.is_err for result in shutdown_task_results), "\n".join( + str(result.error) for result in shutdown_task_results if result.is_err + ) + + # NOTE: Will trigger worker shutdown function(s) + await self.bot.broker.shutdown() diff --git a/tests/backtest_merge.yaml b/tests/backtest_merge.yaml new file mode 100644 index 00000000..19ca1420 --- /dev/null +++ b/tests/backtest_merge.yaml @@ -0,0 +1,7 @@ +bots: "example" +network: "ethereum:mainnet" +# 10 blocks = 120 seconds (2 mins) +start_block: 15_338_009 +stop_block: 15_338_018 +# Ignores extra data +something_else: blah From e12818adc3fce14ed0c2faa3aca884f9c87e5a48 Mon Sep 17 00:00:00 2001 From: Doggie B <3859395+fubuloubu@users.noreply.github.com> Date: Tue, 11 Mar 2025 22:56:20 -0400 Subject: [PATCH 2/6] test: add backtests to CI --- .github/workflows/test.yaml | 5 ++++- tests/{ => backtest}/backtest_merge.yaml | 0 2 files changed, 4 insertions(+), 1 deletion(-) rename tests/{ => backtest}/backtest_merge.yaml (100%) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 03a7d10b..aa228747 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -79,7 +79,10 @@ jobs: pip install .[test] - name: Run Tests - run: pytest -m "not fuzzing" -n 0 -s --cov + run: pytest -p "no:silverback_test" -m "not fuzzing" -n 0 -s --cov + + - name: Run Backtests + run: silverback test example tests/backtest -n 0 -s --cov # NOTE: uncomment this block after you've marked tests with @pytest.mark.fuzzing # fuzzing: diff --git a/tests/backtest_merge.yaml b/tests/backtest/backtest_merge.yaml similarity index 100% rename from tests/backtest_merge.yaml rename to tests/backtest/backtest_merge.yaml From 317b0e9393020bee52f4158e31eba0cda0ff7582 Mon Sep 17 00:00:00 2001 From: Doggie B <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 12 Mar 2025 03:27:13 -0400 Subject: [PATCH 3/6] fix: wrong name for pytest plugin --- silverback/_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 21eeb5af..8b4870b9 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -195,7 +195,7 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, debug, b @verbosity_option() @click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED) def test(cli_ctx, pytest_args): - if return_code := pytest.main([*pytest_args], ["silverback.pytest"]): + if return_code := pytest.main([*pytest_args], ["silverback_test"]): # only exit with non-zero status to make testing easier sys.exit(return_code) From 5bdcf591407cdcd892a57e7983bbaf76e3ff358e Mon Sep 17 00:00:00 2001 From: Doggie B <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 12 Mar 2025 03:38:00 -0400 Subject: [PATCH 4/6] fix: make sure python path has bots folder on it --- silverback/pytest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/silverback/pytest.py b/silverback/pytest.py index 9b5f3c21..22013b25 100644 --- a/silverback/pytest.py +++ b/silverback/pytest.py @@ -1,5 +1,6 @@ import asyncio import os +import sys from functools import cached_property from pathlib import Path @@ -114,6 +115,10 @@ def check_assertions(self, result: TaskResult): @cached_property def runner(self) -> BacktestRunner: + # NOTE: Make sure it can find the path + if (bot_folder := (Path.cwd() / "bots")).exists(): + sys.path.insert(0, str(bot_folder)) + # NOTE: Set parameters for loading settings properly os.environ["SILVERBACK_BOT_NAME"] = self.bot_path with networks.parse_network_choice(self.network_triple): From a913ac987722001cfc2cf1d9bf25e75df37c1ea6 Mon Sep 17 00:00:00 2001 From: Doggie B <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 12 Mar 2025 03:39:21 -0400 Subject: [PATCH 5/6] fix(CI): no need to add bot name anymore --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index aa228747..cea0a3b2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -82,7 +82,7 @@ jobs: run: pytest -p "no:silverback_test" -m "not fuzzing" -n 0 -s --cov - name: Run Backtests - run: silverback test example tests/backtest -n 0 -s --cov + run: silverback test tests/backtest -n 0 -s --cov # NOTE: uncomment this block after you've marked tests with @pytest.mark.fuzzing # fuzzing: From e20002e03b24be4c0cf43a7e11ee41a91cb8af52 Mon Sep 17 00:00:00 2001 From: Doggie B <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 12 Mar 2025 03:52:22 -0400 Subject: [PATCH 6/6] docs(test): add test cli autodoc and userguide --- docs/commands/run.rst | 4 ++++ docs/userguides/development.md | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/commands/run.rst b/docs/commands/run.rst index 4d11b86c..e49ccd65 100644 --- a/docs/commands/run.rst +++ b/docs/commands/run.rst @@ -6,6 +6,10 @@ CLI commands for local development of running Silverback bots and task workers. :prog: silverback run :nested: none +.. click:: silverback._cli:test + :prog: silverback test + :nested: none + .. click:: silverback._cli:worker :prog: silverback worker :nested: none diff --git a/docs/userguides/development.md b/docs/userguides/development.md index ef8e049f..5d57bd0f 100644 --- a/docs/userguides/development.md +++ b/docs/userguides/development.md @@ -306,7 +306,31 @@ The client will send tasks to the 2 worker subprocesses, and all task queue and ## Testing your Bot -TODO: Add backtesting mode w/ `silverback test` +You can test your bot across various ranges of history using the [`silverback test`](../commands/test) command to perform "backtesting". +Backtesting is a super useful technique to measure the performance of your bot by checking it against different periods of history to see how it would react. +Silverback has provisional support for special backtest cases that must have a name like `backtest_*.yaml` under your `tests/` folder in your project. + +Here is an example test case: + +```yaml +# `bots` key is optional, defaults to `bot:bot` if omitted +bots: "" +# can also use array of bot names, or "*" for all bots under `bots/` + +# required, recommended to be a public network +network: "ethereum:mainnet" + +# recommended to pick a small range of blocks for your case +start_block: 15_338_009 +stop_block: 15_338_018 + +# NOTE: Ignores extra data +something_else: blah +``` + +In a complex project (where your bot(s) are not the only component), it is recommended to place your backtests under a separate folder such as `tests/backtest/`. +This will allow you to run backtests separately from your other tests as they take a long time to execute. +Further, you can use any pytest arguments to help select your tests, for example using `-k ` in a [test matrix](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#using-a-matrix-strategy), if you have multiple bots to test. ## Deploying your Bot