Skip to content

Commit 13a0e6c

Browse files
alirezarezvanialirezarezvani
authored
Merge pull request #335 from alirezarezvani/dev
Dev
2 parents 713e2de + d196685 commit 13a0e6c

20 files changed

Lines changed: 1789 additions & 0 deletions

‎.github/workflows/skill-eval.yml‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
---
2+
name: Skill Quality Eval (promptfoo)
3+
4+
'on':
5+
pull_request:
6+
types: [opened, synchronize, reopened]
7+
paths:
8+
- '**/SKILL.md'
9+
workflow_dispatch:
10+
inputs:
11+
skill:
12+
description: 'Specific skill eval config to run (e.g. copywriting)'
13+
required: false
14+
15+
concurrency:
16+
group: skill-eval-${{ github.event.pull_request.number || github.run_id }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
detect-changes:
21+
name: Detect changed skills
22+
runs-on: ubuntu-latest
23+
outputs:
24+
skills: ${{ steps.find-evals.outputs.skills }}
25+
has_evals: ${{ steps.find-evals.outputs.has_evals }}
26+
steps:
27+
- name: Checkout
28+
uses: actions/checkout@v4
29+
with:
30+
fetch-depth: 0
31+
32+
- name: Find eval configs for changed skills
33+
id: find-evals
34+
run: |
35+
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.skill }}" ]]; then
36+
SKILL="${{ github.event.inputs.skill }}"
37+
if [[ -f "eval/skills/${SKILL}.yaml" ]]; then
38+
echo "skills=[\"${SKILL}\"]" >> "$GITHUB_OUTPUT"
39+
echo "has_evals=true" >> "$GITHUB_OUTPUT"
40+
else
41+
echo "No eval config found for: ${SKILL}"
42+
echo "has_evals=false" >> "$GITHUB_OUTPUT"
43+
fi
44+
exit 0
45+
fi
46+
47+
# Get changed SKILL.md files in this PR
48+
CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- '**/SKILL.md' | grep -v '.gemini/' | grep -v '.codex/' | grep -v 'sample')
49+
50+
if [[ -z "$CHANGED" ]]; then
51+
echo "No SKILL.md files changed."
52+
echo "has_evals=false" >> "$GITHUB_OUTPUT"
53+
exit 0
54+
fi
55+
56+
echo "Changed SKILL.md files:"
57+
echo "$CHANGED"
58+
59+
# Map changed skills to eval configs
60+
EVALS="[]"
61+
for skill_path in $CHANGED; do
62+
# Extract skill name from path (e.g. marketing-skill/copywriting/SKILL.md -> copywriting)
63+
skill_name=$(basename $(dirname "$skill_path"))
64+
eval_config="eval/skills/${skill_name}.yaml"
65+
66+
if [[ -f "$eval_config" ]]; then
67+
EVALS=$(echo "$EVALS" | python3 -c "
68+
import json, sys
69+
arr = json.load(sys.stdin)
70+
name = '$skill_name'
71+
if name not in arr:
72+
arr.append(name)
73+
print(json.dumps(arr))
74+
")
75+
echo " ✅ $skill_name → $eval_config"
76+
else
77+
echo " ⏭️ $skill_name → no eval config (skipping)"
78+
fi
79+
done
80+
81+
echo "skills=$EVALS" >> "$GITHUB_OUTPUT"
82+
if [[ "$EVALS" == "[]" ]]; then
83+
echo "has_evals=false" >> "$GITHUB_OUTPUT"
84+
else
85+
echo "has_evals=true" >> "$GITHUB_OUTPUT"
86+
fi
87+
88+
eval:
89+
name: "Eval: ${{ matrix.skill }}"
90+
needs: detect-changes
91+
if: needs.detect-changes.outputs.has_evals == 'true'
92+
runs-on: ubuntu-latest
93+
permissions:
94+
contents: read
95+
pull-requests: write
96+
timeout-minutes: 15
97+
strategy:
98+
fail-fast: false
99+
matrix:
100+
skill: ${{ fromJson(needs.detect-changes.outputs.skills) }}
101+
steps:
102+
- name: Checkout
103+
uses: actions/checkout@v4
104+
105+
- name: Set up Node.js
106+
uses: actions/setup-node@v4
107+
with:
108+
node-version: 20
109+
110+
- name: Run promptfoo eval
111+
id: eval
112+
continue-on-error: true
113+
env:
114+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
115+
run: |
116+
npx promptfoo@latest eval \
117+
-c "eval/skills/${{ matrix.skill }}.yaml" \
118+
--no-cache \
119+
--output "/tmp/${{ matrix.skill }}-results.json" \
120+
--output-format json \
121+
2>&1 | tee /tmp/eval-output.log
122+
123+
echo "exit_code=$?" >> "$GITHUB_OUTPUT"
124+
125+
- name: Parse results
126+
id: parse
127+
if: always()
128+
run: |
129+
RESULTS_FILE="/tmp/${{ matrix.skill }}-results.json"
130+
if [[ ! -f "$RESULTS_FILE" ]]; then
131+
echo "summary=⚠️ No results file generated" >> "$GITHUB_OUTPUT"
132+
exit 0
133+
fi
134+
135+
python3 << 'PYEOF'
136+
import json, os
137+
138+
with open(os.environ.get("RESULTS_FILE", f"/tmp/${{ matrix.skill }}-results.json")) as f:
139+
data = json.load(f)
140+
141+
results = data.get("results", data.get("evalResults", []))
142+
total = len(results)
143+
passed = 0
144+
failed = 0
145+
details = []
146+
147+
for r in results:
148+
test_pass = r.get("success", False)
149+
if test_pass:
150+
passed += 1
151+
else:
152+
failed += 1
153+
154+
prompt_vars = r.get("vars", {})
155+
task = prompt_vars.get("task", "unknown")[:80]
156+
157+
assertions = r.get("gradingResult", {}).get("componentResults", [])
158+
for a in assertions:
159+
status = "✅" if a.get("pass", False) else "❌"
160+
reason = a.get("reason", a.get("assertion", {}).get("value", ""))[:100]
161+
details.append(f" {status} {reason}")
162+
163+
rate = (passed / total * 100) if total > 0 else 0
164+
icon = "✅" if rate >= 80 else "⚠️" if rate >= 50 else "❌"
165+
166+
summary = f"{icon} **${{ matrix.skill }}**: {passed}/{total} tests passed ({rate:.0f}%)"
167+
168+
# Write to file for comment step
169+
with open("/tmp/eval-summary.md", "w") as f:
170+
f.write(f"### {summary}\n\n")
171+
if details:
172+
f.write("<details><summary>Assertion details</summary>\n\n")
173+
f.write("\n".join(details))
174+
f.write("\n\n</details>\n")
175+
176+
# Output for workflow
177+
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
178+
f.write(f"summary={summary}\n")
179+
f.write(f"pass_rate={rate:.0f}\n")
180+
PYEOF
181+
182+
env:
183+
RESULTS_FILE: "/tmp/${{ matrix.skill }}-results.json"
184+
185+
- name: Comment on PR
186+
if: github.event_name == 'pull_request' && always()
187+
uses: actions/github-script@v7
188+
with:
189+
script: |
190+
const fs = require('fs');
191+
let body = '### 🧪 Skill Eval: `${{ matrix.skill }}`\n\n';
192+
193+
try {
194+
const summary = fs.readFileSync('/tmp/eval-summary.md', 'utf8');
195+
body += summary;
196+
} catch {
197+
body += '⚠️ Eval did not produce results. Check the workflow logs.\n';
198+
}
199+
200+
body += '\n\n---\n*Powered by [promptfoo](https://promptfoo.dev) · [eval config](eval/skills/${{ matrix.skill }}.yaml)*';
201+
202+
// Find existing comment to update
203+
const { data: comments } = await github.rest.issues.listComments({
204+
owner: context.repo.owner,
205+
repo: context.repo.repo,
206+
issue_number: context.issue.number,
207+
});
208+
209+
const marker = `Skill Eval: \`${{ matrix.skill }}\``;
210+
const existing = comments.find(c => c.body.includes(marker));
211+
212+
if (existing) {
213+
await github.rest.issues.updateComment({
214+
owner: context.repo.owner,
215+
repo: context.repo.repo,
216+
comment_id: existing.id,
217+
body,
218+
});
219+
} else {
220+
await github.rest.issues.createComment({
221+
owner: context.repo.owner,
222+
repo: context.repo.repo,
223+
issue_number: context.issue.number,
224+
body,
225+
});
226+
}
227+
228+
- name: Upload results
229+
if: always()
230+
uses: actions/upload-artifact@v4
231+
with:
232+
name: eval-results-${{ matrix.skill }}
233+
path: /tmp/${{ matrix.skill }}-results.json
234+
retention-days: 30
235+
if-no-files-found: ignore

‎agents/personas/README.md‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Persona-Based Agents
2+
3+
Pre-configured agent personas with curated skill loadouts, workflows, and distinct personalities.
4+
5+
## What's a Persona?
6+
7+
A **persona** is an agent definition that goes beyond "use these skills." Each persona includes:
8+
9+
- **🧠 Identity & Memory** — who this agent is, how they think, what they've learned
10+
- **🎯 Core Mission** — what they optimize for, in priority order
11+
- **🚨 Critical Rules** — hard constraints they never violate
12+
- **📋 Capabilities** — domain expertise organized by area
13+
- **🔄 Workflows** — step-by-step processes for common tasks
14+
- **💭 Communication Style** — how they talk, with concrete examples
15+
- **🎯 Success Metrics** — measurable outcomes that define "good"
16+
- **🚀 Advanced Capabilities** — deeper expertise loaded on demand
17+
- **🔄 Learning & Memory** — what they retain and patterns they recognize
18+
19+
## How to Use
20+
21+
### Claude Code
22+
```bash
23+
cp agents/personas/startup-cto.md ~/.claude/agents/
24+
# Then: "Activate startup-cto mode"
25+
```
26+
27+
### Cursor
28+
```bash
29+
./scripts/convert.sh --tool cursor
30+
# Personas convert to .cursor/rules/*.mdc
31+
```
32+
33+
### Any Supported Tool
34+
```bash
35+
./scripts/install.sh --tool <your-tool>
36+
```
37+
38+
## Available Personas
39+
40+
| Persona | Emoji | Domain | Best For |
41+
|---------|-------|--------|----------|
42+
| [Startup CTO](startup-cto.md) | 🏗️ | Engineering + Strategy | Technical co-founders, architecture decisions, team building |
43+
| [Growth Marketer](growth-marketer.md) | 🚀 | Marketing + Growth | Bootstrapped founders, content-led growth, launches |
44+
| [Solo Founder](solo-founder.md) | 🦄 | Cross-domain | One-person startups, side projects, MVP building |
45+
46+
## Personas vs Task Agents
47+
48+
| | Task Agents (`agents/`) | Personas (`agents/personas/`) |
49+
|---|---|---|
50+
| **Focus** | Task execution | Role embodiment |
51+
| **Scope** | Single domain | Cross-domain curated set |
52+
| **Voice** | Neutral/professional | Personality-driven with backstory |
53+
| **Workflows** | Single-step | Multi-step with decision points |
54+
| **Use case** | "Do this task" | "Think like this person" |
55+
56+
Both coexist. Use task agents for focused work, personas for ongoing collaboration.
57+
58+
## Creating Your Own
59+
60+
See [TEMPLATE.md](TEMPLATE.md) for the format specification. Key elements:
61+
62+
```yaml
63+
---
64+
name: Agent Name
65+
description: What this agent does and when to activate it.
66+
color: blue # Agent color theme
67+
emoji: 🎯 # Single emoji identifier
68+
vibe: One sentence personality capture.
69+
tools: Read, Write, Bash, Grep, Glob
70+
---
71+
```
72+
73+
Follow the section structure (Identity → Mission → Rules → Capabilities → Workflows → Communication → Metrics → Advanced → Learning) for consistency with existing personas.

0 commit comments

Comments
 (0)