Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ jobs:
contents: read # checkout repository
timeout-minutes: 10
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: "3.12"
Expand All @@ -45,10 +45,10 @@ jobs:
env:
PACKAGE: ${{ matrix.package }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
Expand All @@ -69,10 +69,10 @@ jobs:
env:
PACKAGE: ${{ matrix.package }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: "3.12"
Expand All @@ -87,10 +87,10 @@ jobs:
contents: read # checkout repository
timeout-minutes: 10
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: "3.12"
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ jobs:
pytest_mark: "functional and (google or litellm)"
name: agents / ${{ matrix.backend.name }} / ${{ matrix.python-version }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
persist-credentials: false
Expand All @@ -91,7 +91,7 @@ jobs:
# external contributors, and the immutable head.sha pin keeps the
# reviewed commit identical to the executed one.
allow-unsafe-pr-checkout: true
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
Expand Down Expand Up @@ -139,14 +139,14 @@ jobs:
PROVIDER: ${{ matrix.provider }}
EXTRAS: ${{ matrix.extras }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
persist-credentials: false
# Deliberate opt-in — see test-agents-functional for the full
# rationale and compensating controls.
allow-unsafe-pr-checkout: true
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
Expand Down Expand Up @@ -180,14 +180,14 @@ jobs:
env:
PROVIDER: ${{ matrix.provider }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
persist-credentials: false
# Deliberate opt-in — see test-agents-functional for the full
# rationale and compensating controls.
allow-unsafe-pr-checkout: true
- uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,14 @@ jobs:
private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}

- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
fetch-tags: true
persist-credentials: false

- name: Install uv
uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: false
python-version: "3.12"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/zizmor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ jobs:
contents: read # checkout repository
actions: read # needed for private repo analysis
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7
- uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0
with:
persona: pedantic
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,15 @@ from giskard.checks import Scenario, Groundedness

client = OpenAI()


def get_answer(inputs: str) -> str:
response = client.chat.completions.create(
model="gpt-5-mini",
messages=[{"role": "user", "content": inputs}],
)
return response.choices[0].message.content


scenario = (
Scenario("test_dynamic_output")
.interact(
Expand Down Expand Up @@ -121,13 +123,15 @@ Use Giskard Scan to:
import asyncio
from giskard.scan import vulnerability_scan


async def main():
await vulnerability_scan(
target=my_agent,
description="A customer support chatbot for an e-commerce platform.",
languages=["en"],
)


asyncio.run(main())
```

Expand All @@ -147,11 +151,13 @@ Wrap your model and run the scan:
import giskard
import pandas as pd


# Replace my_llm_chain with your actual LLM chain or model inference logic
def model_predict(df: pd.DataFrame):
"""The function takes a DataFrame and must return a list of outputs (one per row)."""
return [my_llm_chain.run({"query": question}) for question in df["question"]]


giskard_model = giskard.Model(
model=model_predict,
model_type="text_generation",
Expand Down Expand Up @@ -183,7 +189,7 @@ knowledge_base = KnowledgeBase.from_pandas(df, columns=["column_1", "column_2"])
testset = generate_testset(
knowledge_base,
num_questions=60,
language='en',
language="en",
agent_description="A customer support chatbot for company X",
)
```
Expand Down
52 changes: 32 additions & 20 deletions libs/giskard-agents/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ Or add multiple messages to the workflow:
```python
# The chat message role is "user" by default.
chat = await (
generator
.chat("You are a helpful assistant.", role="system")
generator.chat("You are a helpful assistant.", role="system")
.chat("Hello, how are you?")
.chat("I'm fine, thank you!", role="assistant")
.chat("What's your name?")
Expand Down Expand Up @@ -92,7 +91,9 @@ generator = agents.Generator(
Or use the convenience method:

```python
generator = agents.Generator(model="openai/gpt-4o-mini").with_retries(5, base_delay=2.0, max_delay=30.0)
generator = agents.Generator(model="openai/gpt-4o-mini").with_retries(
5, base_delay=2.0, max_delay=30.0
)
```

### Rate limiting
Expand All @@ -109,7 +110,9 @@ generator = agents.Generator(
Or use the convenience method:

```python
generator = generator.with_rate_limiter(MinIntervalRateLimiter.from_rpm(60, max_concurrent=5))
generator = generator.with_rate_limiter(
MinIntervalRateLimiter.from_rpm(60, max_concurrent=5)
)
```

## Custom middleware
Expand All @@ -124,6 +127,7 @@ from giskard.agents.generators import GenerationParams
from giskard.agents.generators.middleware import CompletionMiddleware, NextFn
from giskard.llm.types import ChatMessage, CompletionResponse


@CompletionMiddleware.register("logging")
class LoggingMiddleware(CompletionMiddleware):
async def call(
Expand All @@ -138,6 +142,7 @@ class LoggingMiddleware(CompletionMiddleware):
logging.info(f"Got response: {response.choices[0].finish_reason}")
return response


generator = agents.Generator(
model="openai/gpt-4o-mini",
middlewares=[LoggingMiddleware()],
Expand All @@ -154,15 +159,13 @@ each completion call:
```python
from pydantic import BaseModel


class SimpleOutput(BaseModel):
mood: str
greeting: str

chat = await (
generator.chat("Hello!")
.with_output(SimpleOutput)
.run()
)

chat = await generator.chat("Hello!").with_output(SimpleOutput).run()

assert isinstance(chat.output, SimpleOutput)
assert chat.output.mood == "happy"
Expand All @@ -179,9 +182,7 @@ Here's an example:
```python
# This will run a chat with the message "Hello Test Bot, how are you?"
chat = await (
generator.chat(
"Hello {{ name_of_the_bot }}, how are you?", as_template=True
)
generator.chat("Hello {{ name_of_the_bot }}, how are you?", as_template=True)
.with_inputs(name_of_the_bot="Test Bot")
.run()
)
Expand Down Expand Up @@ -246,7 +247,9 @@ You can then load the template as usual:
```python
chat = await (
generator.template("evaluators.scientific_theory")
.with_inputs(theory="Normandy is actually the center of the universe because its perfect balance of rain, cheese, and cider creates a quantum field that bends space-time, making it the most harmonious place on Earth.")
.with_inputs(
theory="Normandy is actually the center of the universe because its perfect balance of rain, cheese, and cider creates a quantum field that bends space-time, making it the most harmonious place on Earth."
)
.run()
)

Expand All @@ -259,10 +262,9 @@ assert score == 5
You can run multiple chats with different inputs by passing a list of inputs to the `run_batch` method.

```python
chats = await (
generator.chat("What's the weather in {{ city }}?", as_template=True)
.run_batch([{"city": "Paris"}, {"city": "London"}])
)
chats = await generator.chat(
"What's the weather in {{ city }}?", as_template=True
).run_batch([{"city": "Paris"}, {"city": "London"}])
assert len(chats) == 2
```

Expand All @@ -277,6 +279,7 @@ This can be combined with all functionalities described earlier. Here's an examp
```python
from giskard import agents


@agents.tool
def get_weather(city: str) -> str:
"""Get the weather in a city.
Expand All @@ -291,6 +294,7 @@ def get_weather(city: str) -> str:

return f"It's sunny in {city}."


# Run parallel chats with tools
chats = await (
generator.chat("Hello, what's the weather in {{ city }}?", as_template=True)
Expand Down Expand Up @@ -367,10 +371,16 @@ Note: when running a single chat (`workflow.run(...)`), error policy `SKIP` beha
from giskard.agents import ErrorPolicy

# This may return fewer than 3 chats if some fail.
chats = await generator.chat("Hello!", role="user").on_error(ErrorPolicy.SKIP).run_many(n=3)
chats = (
await generator.chat("Hello!", role="user").on_error(ErrorPolicy.SKIP).run_many(n=3)
)

# This will return 3 chats, some may be in failed state.
chats = await generator.chat("Hello!", role="user").on_error(ErrorPolicy.RETURN).run_many(n=3)
chats = (
await generator.chat("Hello!", role="user")
.on_error(ErrorPolicy.RETURN)
.run_many(n=3)
)

for chat in chats:
if chat.failed:
Expand All @@ -388,15 +398,17 @@ You can change this behavior by passing the `catch=None` on the tool decorator.
def get_weather(city: str) -> str:
raise ValueError("City not found")


result = await get_weather.run(arguments={"city": "Paris"})
print(result) # "ERROR: City not found"
print(result) # "ERROR: City not found"


# Opt out of the catch
@agents.tool(catch=None)
def get_weather(city: str) -> str:
raise ValueError("City not found")


# This will raise an exception
result = await get_weather.run(arguments={"city": "Paris"})
```
Expand Down
Loading