113 lines
3.4 KiB
YAML
113 lines
3.4 KiB
YAML
tracing:
|
|
enabled: true
|
|
otlp:
|
|
http:
|
|
enabled: true
|
|
port: 4318
|
|
|
|
prompts:
|
|
- id: test-run
|
|
label: Run tests only
|
|
raw: |
|
|
Run {{task}} available in {{dir}}. For platform:{{platform}}.
|
|
Provide a report in markdown format.
|
|
**DO NOT** run compilation or gn gen.
|
|
**ALWAYS** stop and report on unexpected tools errors.
|
|
- id: lsp-cli-usage
|
|
label: Usage of LSP Server cli
|
|
raw: |
|
|
with help of language server do {{task}}.
|
|
results format is {{format}}.
|
|
- id: write-plan
|
|
label: Write Plan after Research
|
|
raw: |
|
|
/research-init {{task}}. Write to IMPLEMENTATION_PLAN.md.
|
|
|
|
providers:
|
|
- id: "file:///home/vptyp/git/promptfoo-eval-pi/pi_provider.py"
|
|
label: "pi-agent"
|
|
config:
|
|
timeout: 6000000
|
|
pythonExecutable: "/home/vptyp/git/promptfoo-eval-pi/uv_python.sh"
|
|
isolation:
|
|
enabled: false
|
|
strategy: "git-worktree"
|
|
clean: false
|
|
|
|
defaultTest:
|
|
options:
|
|
# Universal Agent-as-a-Judge provider (supports agy, claude, codex, openclaw, etc.)
|
|
provider:
|
|
id: "file:///home/vptyp/git/promptfoo-eval-pi/eval_judge.py"
|
|
config:
|
|
command: "agy -p"
|
|
|
|
tests:
|
|
- description: "[PROMPTS] research-init: reads sources, writes plan, never builds"
|
|
prompts:
|
|
- write-plan
|
|
vars:
|
|
task: "Add a build flag for enabling of infobars visibility through tabs api to a Chromium component."
|
|
assert:
|
|
- type: javascript
|
|
value: "context.metadata.exitCode === 0"
|
|
- type: skill-used
|
|
value: "chromium-docs"
|
|
- type: python
|
|
value: |
|
|
tool_calls = (context.get("metadata") or {}).get("toolCalls") or []
|
|
edited_cc = [
|
|
(tc.get("args") or {}).get("path", "?")
|
|
for tc in tool_calls
|
|
if tc.get("tool") == "edit"
|
|
and str((tc.get("args") or {}).get("path", "")).endswith(".cc")
|
|
]
|
|
return {
|
|
"pass": not edited_cc,
|
|
"score": 1.0 if not edited_cc else 0.0,
|
|
"reason": "no .cc files edited" if not edited_cc else f"edit used on .cc files: {edited_cc}",
|
|
}
|
|
- type: python
|
|
value: "file://helpers.py:assert_no_compile"
|
|
- type: trajectory:tool-sequence
|
|
value:
|
|
steps:
|
|
- read
|
|
- write
|
|
|
|
- description: "[SKILLS] test-only: runs unit tests through bash"
|
|
prompts:
|
|
- test-run
|
|
vars:
|
|
task: "unit_tests with subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report."
|
|
dir: "out/chrome"
|
|
platform: "Linux Chromium x86 build"
|
|
assert:
|
|
- type: skill-used
|
|
value: "linux-chromium-build"
|
|
- type: javascript
|
|
value: "context.metadata.exitCode === 0"
|
|
- type: python
|
|
value: "file://helpers.py:assert_run_unittests"
|
|
|
|
- description: "[SKILLS] Discover and used lsp-cli.py skill on direct request"
|
|
prompts:
|
|
- lsp-cli-usage
|
|
vars:
|
|
task: "build callers callStack of InfoBar::Show method; depth 5"
|
|
format: "mermaid sequence diagram"
|
|
assert:
|
|
- type: skill-used
|
|
value: "lsp-cli"
|
|
- type: trajectory:tool-args-match
|
|
value:
|
|
name: "bash"
|
|
args:
|
|
has_binary:
|
|
"./lsp-cli.py": true
|
|
has_word:
|
|
"analyze": true
|
|
"InfoBarManager::AddInfoBarInternal": true
|
|
- type: contains
|
|
value: "mermaid"
|