87 lines
2.8 KiB
YAML
87 lines
2.8 KiB
YAML
tracing:
|
|
enabled: true
|
|
otlp:
|
|
http:
|
|
enabled: true
|
|
port: 4318
|
|
|
|
prompts:
|
|
- "{{prompt}}"
|
|
|
|
providers:
|
|
- id: "file:///home/vptyp/git/promptfoo-eval-pi/pi_provider.py"
|
|
label: "pi-agent"
|
|
config:
|
|
timeout: 6000000
|
|
pythonExecutable: "/home/vptyp/git/promptfoo-eval-pi/uv_python.sh"
|
|
isolation:
|
|
enabled: false
|
|
strategy: "git-worktree"
|
|
clean: false
|
|
|
|
defaultTest:
|
|
options:
|
|
# Universal Agent-as-a-Judge provider (supports agy, claude, codex, openclaw, etc.)
|
|
provider:
|
|
id: "file:///home/vptyp/git/promptfoo-eval-pi/eval_judge.py"
|
|
config:
|
|
command: "agy -p"
|
|
|
|
tests:
|
|
- description: "[PROMPTS] research-init: reads sources, writes plan, never builds"
|
|
vars:
|
|
prompt: "/research-init Add a build flag for enabling of infobars visibility through tabs api to a Chromium component. Write to IMPLEMENTATION_PLAN_VISIBILITY_BUILD_FLAG.md"
|
|
assert:
|
|
- type: javascript
|
|
value: "context.metadata.exitCode === 0"
|
|
- type: skill-used
|
|
value: "chromium-docs"
|
|
- type: python
|
|
value: |
|
|
tool_calls = (context.get("metadata") or {}).get("toolCalls") or []
|
|
edited_cc = [
|
|
(tc.get("args") or {}).get("path", "?")
|
|
for tc in tool_calls
|
|
if tc.get("tool") == "edit"
|
|
and str((tc.get("args") or {}).get("path", "")).endswith(".cc")
|
|
]
|
|
return {
|
|
"pass": not edited_cc,
|
|
"score": 1.0 if not edited_cc else 0.0,
|
|
"reason": "no .cc files edited" if not edited_cc else f"edit used on .cc files: {edited_cc}",
|
|
}
|
|
- type: python
|
|
value: "file://helpers.py:assert_no_compile"
|
|
- type: trajectory:tool-sequence
|
|
value:
|
|
steps:
|
|
- read
|
|
- write
|
|
|
|
- description: "[SKILLS] test-only: runs unit tests through bash"
|
|
vars:
|
|
prompt: "Run the compiled unit tests for Linux Chromium x86 build (out/chrome). Subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report, do not act on your own."
|
|
assert:
|
|
- type: skill-used
|
|
value: "linux-chromium-build"
|
|
- type: javascript
|
|
value: "context.metadata.exitCode === 0"
|
|
- type: python
|
|
value: "file://helpers.py:assert_run_unittests"
|
|
|
|
- description: "[SKILLS] Discover and used lsp-cli.py skill on direct request"
|
|
vars:
|
|
prompt: "With help of language-server build callers callStack of InfoBar::Show method; depth 5"
|
|
assert:
|
|
- type: skill-used
|
|
value: "lsp-cli"
|
|
- type: trajectory:tool-args-match
|
|
value:
|
|
name: "bash"
|
|
args:
|
|
has_binary:
|
|
"./lsp-cli.py": true
|
|
has_word:
|
|
"analyze": true
|
|
"InfoBarManager::AddInfoBarInternal": true
|