tracing:
  enabled: true
  otlp:
    http:
      enabled: true
      port: 4318

prompts:
  - "{{prompt}}"

providers:
  - id: "file:///home/vptyp/git/promptfoo-eval-pi/pi_provider.py"
    label: "pi-agent"
    config:
      timeout: 6000000
      pythonExecutable: "/home/vptyp/git/promptfoo-eval-pi/uv_python.sh"
      isolation:
        enabled: false
        strategy: "git-worktree"
        clean: false

defaultTest:
  options:
    # Universal Agent-as-a-Judge provider (supports agy, claude, codex, openclaw, etc.)
    provider:
      id: "file:///home/vptyp/git/promptfoo-eval-pi/eval_judge.py"
      config:
        command: "agy -p"

tests:
  - description: "[PROMPTS] research-init: reads sources, writes plan, never builds"
    vars:
      prompt: "/research-init Add a build flag for enabling of infobars visibility through tabs api to a Chromium component. Write to IMPLEMENTATION_PLAN_VISIBILITY_BUILD_FLAG.md"
    assert:
      - type: javascript
        value: "context.metadata.exitCode === 0"
      - type: skill-used
        value: "chromium-docs"
      - type: python
        value: |
            tool_calls = (context.get("metadata") or {}).get("toolCalls") or []
            edited_cc = [
                (tc.get("args") or {}).get("path", "?")
                for tc in tool_calls
                if tc.get("tool") == "edit"
                and str((tc.get("args") or {}).get("path", "")).endswith(".cc")
            ]
            return {
                "pass": not edited_cc,
                "score": 1.0 if not edited_cc else 0.0,
                "reason": "no .cc files edited" if not edited_cc else f"edit used on .cc files: {edited_cc}",
            }
      - type: python
        value: "file://helpers.py:assert_no_compile"
      - type: trajectory:tool-sequence
        value:
          steps:
            - read
            - write

  - description: "[SKILLS] test-only: runs unit tests through bash"
    vars:
      prompt: "Run the compiled unit tests for Linux Chromium x86 build (out/chrome). Subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report, do not act on your own."
    assert:
      - type: skill-used
        value: "linux-chromium-build"
      - type: javascript
        value: "context.metadata.exitCode === 0"
      - type: python
        value: "file://helpers.py:assert_run_unittests"

  - description: "[SKILLS] Discover and used lsp-cli.py skill on direct request"
    vars:
      prompt: "With help of language-server build callers callStack of InfoBar::Show method; depth 5"
    assert:
      - type: skill-used
        value: "lsp-cli"
      - type: trajectory:tool-args-match
        value:
          name: "bash"
          args:
            has_binary:
              "./lsp-cli.py": true
            has_word:
              "analyze": true
              "InfoBarManager::AddInfoBarInternal": true