feat(checks): parallel-safe eval checkpoint and resume

.gitignore

@@ -6,6 +6,7 @@ local_report.md
 .run
 .DS_Store
 .claude/worktrees/
+.giskard/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -72,13 +72,15 @@ from giskard.checks import Scenario, Groundedness
 
 client = OpenAI()
 
+
 def get_answer(inputs: str) -> str:
     response = client.chat.completions.create(
         model="gpt-5-mini",
         messages=[{"role": "user", "content": inputs}],
     )
     return response.choices[0].message.content
 
+
 scenario = (
     Scenario("test_dynamic_output")
     .interact(
@@ -121,13 +123,15 @@ Use Giskard Scan to:
 import asyncio
 from giskard.scan import vulnerability_scan
 
+
 async def main():
     await vulnerability_scan(
         target=my_agent,
         description="A customer support chatbot for an e-commerce platform.",
         languages=["en"],
     )
 
+
 asyncio.run(main())
 ```
 
@@ -147,11 +151,13 @@ Wrap your model and run the scan:
 import giskard
 import pandas as pd
 
+
 # Replace my_llm_chain with your actual LLM chain or model inference logic
 def model_predict(df: pd.DataFrame):
     """The function takes a DataFrame and must return a list of outputs (one per row)."""
     return [my_llm_chain.run({"query": question}) for question in df["question"]]
 
+
 giskard_model = giskard.Model(
     model=model_predict,
     model_type="text_generation",
@@ -183,7 +189,7 @@ knowledge_base = KnowledgeBase.from_pandas(df, columns=["column_1", "column_2"])
 testset = generate_testset(
     knowledge_base,
     num_questions=60,
-    language='en',
+    language="en",
     agent_description="A customer support chatbot for company X",
 )
 ```

libs/giskard-agents/README.md

@@ -63,8 +63,7 @@ Or add multiple messages to the workflow:
 ```python
 # The chat message role is "user" by default.
 chat = await (
-    generator
-    .chat("You are a helpful assistant.", role="system")
+    generator.chat("You are a helpful assistant.", role="system")
     .chat("Hello, how are you?")
     .chat("I'm fine, thank you!", role="assistant")
     .chat("What's your name?")
@@ -92,7 +91,9 @@ generator = agents.Generator(
 Or use the convenience method:
 
 ```python
-generator = agents.Generator(model="openai/gpt-4o-mini").with_retries(5, base_delay=2.0, max_delay=30.0)
+generator = agents.Generator(model="openai/gpt-4o-mini").with_retries(
+    5, base_delay=2.0, max_delay=30.0
+)
 ```
 
 ### Rate limiting
@@ -109,7 +110,9 @@ generator = agents.Generator(
 Or use the convenience method:
 
 ```python
-generator = generator.with_rate_limiter(MinIntervalRateLimiter.from_rpm(60, max_concurrent=5))
+generator = generator.with_rate_limiter(
+    MinIntervalRateLimiter.from_rpm(60, max_concurrent=5)
+)
 ```
 
 ## Custom middleware
@@ -124,6 +127,7 @@ from giskard.agents.generators import GenerationParams
 from giskard.agents.generators.middleware import CompletionMiddleware, NextFn
 from giskard.llm.types import ChatMessage, CompletionResponse
 
+
 @CompletionMiddleware.register("logging")
 class LoggingMiddleware(CompletionMiddleware):
     async def call(
@@ -138,6 +142,7 @@ class LoggingMiddleware(CompletionMiddleware):
         logging.info(f"Got response: {response.choices[0].finish_reason}")
         return response
 
+
 generator = agents.Generator(
     model="openai/gpt-4o-mini",
     middlewares=[LoggingMiddleware()],
@@ -154,15 +159,13 @@ each completion call:
 ```python
 from pydantic import BaseModel
 
+
 class SimpleOutput(BaseModel):
     mood: str
     greeting: str
 
-chat = await (
-    generator.chat("Hello!")
-    .with_output(SimpleOutput)
-    .run()
-)
+
+chat = await generator.chat("Hello!").with_output(SimpleOutput).run()
 
 assert isinstance(chat.output, SimpleOutput)
 assert chat.output.mood == "happy"
@@ -179,9 +182,7 @@ Here's an example:
 ```python
 # This will run a chat with the message "Hello Test Bot, how are you?"
 chat = await (
-    generator.chat(
-        "Hello {{ name_of_the_bot }}, how are you?", as_template=True
-    )
+    generator.chat("Hello {{ name_of_the_bot }}, how are you?", as_template=True)
     .with_inputs(name_of_the_bot="Test Bot")
     .run()
 )
@@ -246,7 +247,9 @@ You can then load the template as usual:
 ```python
 chat = await (
     generator.template("evaluators.scientific_theory")
-    .with_inputs(theory="Normandy is actually the center of the universe because its perfect balance of rain, cheese, and cider creates a quantum field that bends space-time, making it the most harmonious place on Earth.")
+    .with_inputs(
+        theory="Normandy is actually the center of the universe because its perfect balance of rain, cheese, and cider creates a quantum field that bends space-time, making it the most harmonious place on Earth."
+    )
     .run()
 )
 
@@ -259,10 +262,9 @@ assert score == 5
 You can run multiple chats with different inputs by passing a list of inputs to the `run_batch` method.
 
 ```python
-chats = await (
-    generator.chat("What's the weather in {{ city }}?", as_template=True)
-    .run_batch([{"city": "Paris"}, {"city": "London"}])
-)
+chats = await generator.chat(
+    "What's the weather in {{ city }}?", as_template=True
+).run_batch([{"city": "Paris"}, {"city": "London"}])
 assert len(chats) == 2
 ```
 
@@ -277,6 +279,7 @@ This can be combined with all functionalities described earlier. Here's an examp
 ```python
 from giskard import agents
 
+
 @agents.tool
 def get_weather(city: str) -> str:
     """Get the weather in a city.
@@ -291,6 +294,7 @@ def get_weather(city: str) -> str:
 
     return f"It's sunny in {city}."
 
+
 # Run parallel chats with tools
 chats = await (
     generator.chat("Hello, what's the weather in {{ city }}?", as_template=True)
@@ -367,10 +371,16 @@ Note: when running a single chat (`workflow.run(...)`), error policy `SKIP` beha
 from giskard.agents import ErrorPolicy
 
 # This may return fewer than 3 chats if some fail.
-chats = await generator.chat("Hello!", role="user").on_error(ErrorPolicy.SKIP).run_many(n=3)
+chats = (
+    await generator.chat("Hello!", role="user").on_error(ErrorPolicy.SKIP).run_many(n=3)
+)
 
 # This will return 3 chats, some may be in failed state.
-chats = await generator.chat("Hello!", role="user").on_error(ErrorPolicy.RETURN).run_many(n=3)
+chats = (
+    await generator.chat("Hello!", role="user")
+    .on_error(ErrorPolicy.RETURN)
+    .run_many(n=3)
+)
 
 for chat in chats:
     if chat.failed:
@@ -388,15 +398,17 @@ You can change this behavior by passing the `catch=None` on the tool decorator.
 def get_weather(city: str) -> str:
     raise ValueError("City not found")
 
+
 result = await get_weather.run(arguments={"city": "Paris"})
-print(result) # "ERROR: City not found"
+print(result)  # "ERROR: City not found"
 
 
 # Opt out of the catch
 @agents.tool(catch=None)
 def get_weather(city: str) -> str:
     raise ValueError("City not found")
 
+
 # This will raise an exception
 result = await get_weather.run(arguments={"city": "Paris"})
 ```