update testing/ with prompt separation & small AGENTS.md fix

AGENTS.md

@@ -135,7 +135,7 @@ the user about the reason to skip.
 - Do not attempt a build without first establishing the correct output
   directory and target. If you have not been given them, and you plan on doing
   a build, then stop and ask before starting on any other tasks.
-- Unless otherwise instructed, ask user for build environment details
+- **ALWAYS** Ask user for build environment details
 - Immediately before launching a Chromium build, use the applicable build
   skill's pre-launch procedure to verify the configured locations, environment
   variables, wrapper, output directory, and target.
@@ -148,7 +148,7 @@ the user about the reason to skip.
 
 ## Testing
 
-Unless otherwise instructed, ask user on test running details
+**ALWAYS** ask user on test running details
 
 When using `tools/autotest.py`:
 - Do not invoke `autoninja` beforehand because `autotest.py` automatically

testing/helpers.py

@@ -1,4 +1,29 @@
 #!/usr/bin/env python3
+"""
+Promptfoo Python assertions over parsed bash tool calls emitted by pi_provider.py.
+
+pi_provider.py enriches every bash tool_execution_start event with
+BashCommandParser output before storing it in metadata.toolCalls:
+
+    args.commands   -> list of ParsedCommand dicts (raw, binary, flags, words,
+                       env_vars, wrappers, redirections, operator_after)
+    args.binaries   -> list of executed binary names (pipeline-wide)
+    args.words      -> list of literal words
+    args.has_binary -> dict[str, bool], e.g. {"pytest": true}
+    args.has_word   -> dict[str, bool]
+    args.has_flag   -> dict[str, bool], e.g. {"-q": true, "--tb": true}
+
+In a promptfoo python assertion these are reachable at:
+    context["metadata"]["toolCalls"][*]["args"]
+
+Usage in promptfooconfig.yaml:
+    - type: python
+      value: "file://tests/assert_bash_calls.py:assert_bash_binary"
+      config:
+        binary: pytest
+
+The default get_assert() is equivalent to assert_bash_binary with config.binary.
+"""
 
 from typing import Any, Dict, List
 
@@ -40,5 +65,5 @@ def assert_no_compile(output: str, context: Dict[str, Any]) -> Dict[str, Any]:
 
 def assert_run_unittests(output: str, context: Dict[str, Any]) -> Dict[str, Any]:
     """Assert that unittests has started """
-    expected = ["./out/chrome/unit_tests", "./unit_tests"]
+    expected = ["./out/chrome/unit_tests", "./unit_tests", "unit_tests"]
     return assert_exec(output, context, expected, True)

testing/promptfooconfig.yaml

@@ -6,7 +6,22 @@ tracing:
       port: 4318
 
 prompts:
-  - "{{prompt}}"
+  - id: test-run
+    label: Run tests only 
+    raw: |
+      Run {{task}} available in {{dir}}. For platform:{{platform}}. 
+      Provide a report in markdown format.
+      **DO NOT** run compilation or gn gen.
+      **ALWAYS** stop and report on unexpected tools errors.
+  - id: lsp-cli-usage
+    label: Usage of LSP Server cli
+    raw: |
+      with help of language server do {{task}}.
+      results format is {{format}}.
+  - id: write-plan
+    label: Write Plan after Research
+    raw: |
+      /research-init {{task}}. Write to IMPLEMENTATION_PLAN.md.
 
 providers:
   - id: "file:///home/vptyp/git/promptfoo-eval-pi/pi_provider.py"
@@ -29,8 +44,10 @@ defaultTest:
 
 tests:
   - description: "[PROMPTS] research-init: reads sources, writes plan, never builds"
+    prompts:
+      - write-plan
     vars:
-      prompt: "/research-init Add a build flag for enabling of infobars visibility through tabs api to a Chromium component. Write to IMPLEMENTATION_PLAN_VISIBILITY_BUILD_FLAG.md"
+      task: "Add a build flag for enabling of infobars visibility through tabs api to a Chromium component."
     assert:
       - type: javascript
         value: "context.metadata.exitCode === 0"
@@ -59,8 +76,12 @@ tests:
             - write
 
   - description: "[SKILLS] test-only: runs unit tests through bash"
+    prompts:
+      - test-run
     vars:
-      prompt: "Run the compiled unit tests for Linux Chromium x86 build (out/chrome). Subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report, do not act on your own."
+      task: "unit_tests with subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report."
+      dir: "out/chrome"
+      platform: "Linux Chromium x86 build"
     assert:
       - type: skill-used
         value: "linux-chromium-build"
@@ -70,8 +91,11 @@ tests:
         value: "file://helpers.py:assert_run_unittests"
 
   - description: "[SKILLS] Discover and used lsp-cli.py skill on direct request"
+    prompts:
+      - lsp-cli-usage
     vars:
-      prompt: "With help of language-server build callers callStack of InfoBar::Show method; depth 5"
+      task: "build callers callStack of InfoBar::Show method; depth 5"
+      format: "mermaid sequence diagram"
     assert:
       - type: skill-used
         value: "lsp-cli"
@@ -84,3 +108,5 @@ tests:
             has_word:
               "analyze": true
               "InfoBarManager::AddInfoBarInternal": true
+      - type: contains
+        value: "mermaid"