tracing:
  enabled: true
  otlp:
    http:
      enabled: true
      port: 4318

prompts:
  - id: test-run
    label: Run tests only 
    raw: |
      Run {{task}} available in {{dir}}. For platform:{{platform}}. 
      Provide a report in markdown format.
      **DO NOT** run compilation or gn gen.
      **ALWAYS** stop and report on unexpected tools errors.
  - id: lsp-cli-usage
    label: Usage of LSP Server cli
    raw: |
      with help of language server do {{task}}.
      results format is {{format}}.
  - id: write-plan
    label: Write Plan after Research
    raw: |
      /research-init {{task}}. Write to IMPLEMENTATION_PLAN.md.

providers:
  - id: "file:///home/vptyp/git/promptfoo-eval-pi/pi_provider.py"
    label: "pi-agent"
    config:
      timeout: 6000000
      pythonExecutable: "/home/vptyp/git/promptfoo-eval-pi/uv_python.sh"
      isolation:
        enabled: false
        strategy: "git-worktree"
        clean: false

defaultTest:
  options:
    # Universal Agent-as-a-Judge provider (supports agy, claude, codex, openclaw, etc.)
    provider:
      id: "file:///home/vptyp/git/promptfoo-eval-pi/eval_judge.py"
      config:
        command: "agy -p"

tests:
  - description: "[PROMPTS] research-init: reads sources, writes plan, never builds"
    prompts:
      - write-plan
    vars:
      task: "Add a build flag for enabling of infobars visibility through tabs api to a Chromium component."
    assert:
      - type: javascript
        value: "context.metadata.exitCode === 0"
      - type: skill-used
        value: "chromium-docs"
      - type: python
        value: |
            tool_calls = (context.get("metadata") or {}).get("toolCalls") or []
            edited_cc = [
                (tc.get("args") or {}).get("path", "?")
                for tc in tool_calls
                if tc.get("tool") == "edit"
                and str((tc.get("args") or {}).get("path", "")).endswith(".cc")
            ]
            return {
                "pass": not edited_cc,
                "score": 1.0 if not edited_cc else 0.0,
                "reason": "no .cc files edited" if not edited_cc else f"edit used on .cc files: {edited_cc}",
            }
      - type: python
        value: "file://helpers.py:assert_no_compile"
      - type: trajectory:tool-sequence
        value:
          steps:
            - read
            - write

  - description: "[SKILLS] test-only: runs unit tests through bash"
    prompts:
      - test-run
    vars:
      task: "unit_tests with subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report."
      dir: "out/chrome"
      platform: "Linux Chromium x86 build"
    assert:
      - type: skill-used
        value: "linux-chromium-build"
      - type: javascript
        value: "context.metadata.exitCode === 0"
      - type: python
        value: "file://helpers.py:assert_run_unittests"

  - description: "[SKILLS] Discover and used lsp-cli.py skill on direct request"
    prompts:
      - lsp-cli-usage
    vars:
      task: "build callers callStack of InfoBar::Show method; depth 5"
      format: "mermaid sequence diagram"
    assert:
      - type: skill-used
        value: "lsp-cli"
      - type: trajectory:tool-args-match
        value:
          name: "bash"
          args:
            has_binary:
              "./lsp-cli.py": true
            has_word:
              "analyze": true
              "InfoBarManager::AddInfoBarInternal": true
      - type: contains
        value: "mermaid"