70 lines
2.8 KiB
Python
70 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Promptfoo Python assertions over parsed bash tool calls emitted by pi_provider.py.
|
|
|
|
pi_provider.py enriches every bash tool_execution_start event with
|
|
BashCommandParser output before storing it in metadata.toolCalls:
|
|
|
|
args.commands -> list of ParsedCommand dicts (raw, binary, flags, words,
|
|
env_vars, wrappers, redirections, operator_after)
|
|
args.binaries -> list of executed binary names (pipeline-wide)
|
|
args.words -> list of literal words
|
|
args.has_binary -> dict[str, bool], e.g. {"pytest": true}
|
|
args.has_word -> dict[str, bool]
|
|
args.has_flag -> dict[str, bool], e.g. {"-q": true, "--tb": true}
|
|
|
|
In a promptfoo python assertion these are reachable at:
|
|
context["metadata"]["toolCalls"][*]["args"]
|
|
|
|
Usage in promptfooconfig.yaml:
|
|
- type: python
|
|
value: "file://tests/assert_bash_calls.py:assert_bash_binary"
|
|
config:
|
|
binary: pytest
|
|
|
|
The default get_assert() is equivalent to assert_bash_binary with config.binary.
|
|
"""
|
|
|
|
from typing import Any, Dict, List
|
|
|
|
PASS = {"pass": True, "score": 1.0, "reason": ""}
|
|
FAIL = {"pass": False, "score": 0.0, "reason": ""}
|
|
|
|
|
|
def _bash_args(context: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""Returns the args dict of every bash tool call in the trajectory."""
|
|
tool_calls = (context.get("metadata") or {}).get("toolCalls") or []
|
|
return [t.get("args", {}) for t in tool_calls if t.get("tool") == "bash"]
|
|
|
|
|
|
def _result(passed: bool, reason: str, score: float = 1.0) -> Dict[str, Any]:
|
|
if passed:
|
|
return {"pass": True, "score": score, "reason": reason}
|
|
return {"pass": False, "score": 0.0, "reason": reason}
|
|
|
|
|
|
def assert_exec(output: str, context: Dict[str, Any], expected, result: bool) -> Dict[str, Any]:
|
|
args_list = _bash_args(context)
|
|
if not args_list:
|
|
return _result(not result, "No bash tool calls were made at all")
|
|
|
|
for args in args_list:
|
|
binaries = args.get("binaries", [])
|
|
per_command = [c.get("binary") for c in args.get("commands", [])]
|
|
for entity in expected:
|
|
if entity in binaries or entity in per_command:
|
|
return _result(result, f"bash invoked '{entity}'")
|
|
|
|
seen = sorted({b for a in args_list for b in a.get("binaries", [])})
|
|
return _result(not result, f"'{expected}' was never invoked. Binaries seen: {seen or 'none'}")
|
|
|
|
def assert_no_compile(output: str, context: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Asserts a specific binaries was not executed in any bash call (config: binary)."""
|
|
expected = ["ninja", "autoninja"]
|
|
return assert_exec(output, context, expected, False)
|
|
|
|
def assert_run_unittests(output: str, context: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Assert that unittests has started """
|
|
expected = ["./out/chrome/unit_tests", "./unit_tests", "unit_tests"]
|
|
return assert_exec(output, context, expected, True)
|