From e0e94a477237ea7c2d2275e74195cd952e2d07f5 Mon Sep 17 00:00:00 2001 From: Artur Mukhamadiev Date: Wed, 2 Sep 2026 22:41:57 +0300 Subject: [PATCH] Added prompts templates, promptfoo config & fixed skills --- prompts/debug-fix.md | 14 ++++ prompts/feature-impl.md | 16 +++++ prompts/post-compact.md | 11 ++++ prompts/research-init.md | 22 +++++++ prompts/research-middle.md | 13 ++++ prompts/test-only.md | 10 +++ skills/linux-chromium-build/SKILL.md | 21 ++++-- skills/lsp-cli/SKILL.md | 99 ++++++++++++++++++++++++++-- skills/webos-ose-build/SKILL.md | 2 +- testing/helpers.py | 44 +++++++++++++ testing/promptfooconfig.yaml | 86 ++++++++++++++++++++++++ 11 files changed, 323 insertions(+), 15 deletions(-) create mode 100644 prompts/debug-fix.md create mode 100644 prompts/feature-impl.md create mode 100644 prompts/post-compact.md create mode 100644 prompts/research-init.md create mode 100644 prompts/research-middle.md create mode 100644 prompts/test-only.md create mode 100644 testing/helpers.py create mode 100644 testing/promptfooconfig.yaml diff --git a/prompts/debug-fix.md b/prompts/debug-fix.md new file mode 100644 index 0000000..a5f6a9d --- /dev/null +++ b/prompts/debug-fix.md @@ -0,0 +1,14 @@ +--- +includes: + - post-compact.md +--- +- **DO NOT** run compilation or `gn gen` +- **DEFINITION OF DONE**: code fix + report. No runtime verification. + +## Definition Of Done +* [BUG / ISSUE] is fixed. +* Language server reports zero errors on all touched files. + +# Task + +$@ diff --git a/prompts/feature-impl.md b/prompts/feature-impl.md new file mode 100644 index 0000000..80eb7de --- /dev/null +++ b/prompts/feature-impl.md @@ -0,0 +1,16 @@ +--- +includes: + - post-compact.md +--- +- **ALWAYS** try to reliably get deep semantic understanding of a code. +- **ALWAYS** anchor to existing implementations if possible. +- **DO NOT** run compilation or `gn gen`. +- **DEFINITION OF DONE**: code written + report. No runtime verification. + +## Definition Of Done +- All Logic from Implementation Plan is implemented. +- Language server reports zero errors on all touched files. + +# Task + +$@ diff --git a/prompts/post-compact.md b/prompts/post-compact.md new file mode 100644 index 0000000..7510603 --- /dev/null +++ b/prompts/post-compact.md @@ -0,0 +1,11 @@ +# PREREQUISITES + +Re-read skills which might be useful at the moment, do not believe your memory, if you think +that something might help: re-read it from scratch +Re-read Implementation plan + +## RULES + +- **ALWAYS** on any unexpected errors or behavior --- **stop and report**. +- **DO NOT** proceed to the next step unless explicitly instructed. + diff --git a/prompts/research-init.md b/prompts/research-init.md new file mode 100644 index 0000000..b947452 --- /dev/null +++ b/prompts/research-init.md @@ -0,0 +1,22 @@ +# RULES + +- **ALWAYS** start from reading chromium docs skill +- **ALWAYS** on any unexpected errors or behavior --- **stop and report**. +- **ALWAYS** Try to find similar existing implementations for parts of your goal. +- **ALWAYS** try to reliably get deep semantic understanding of a code +- **DO NOT** proceed to the next step unless explicitly instructed. +- **DO NOT** run compilation or `gn gen` +- **DEFINITION OF DONE**: Written to the disk plan on review + +## Plan Contents + +Should: +- begin with a section, which is fully reflecting task section contents +- Step-by-step implementation plan **without code** snippets, but with expected locations & signatures. +- contain a sub-section with similar existing implementation to each part of a plan **without code** + snippets, but with links to their location + +# Task +Provide an implementation plan for: + +$@ diff --git a/prompts/research-middle.md b/prompts/research-middle.md new file mode 100644 index 0000000..4dee278 --- /dev/null +++ b/prompts/research-middle.md @@ -0,0 +1,13 @@ +--- +includes: + - post-compact.md +--- +- **ALWAYS** Try to find similar existing implementations for parts of your goal. +- **ALWAYS** try to reliably get deep semantic understanding of a code +- **ALWAYS** read documentation +- **DO NOT** run compilation or `gn gen` +- **DEFINITION OF DONE**: Research results + +# Task + +$@ diff --git a/prompts/test-only.md b/prompts/test-only.md new file mode 100644 index 0000000..e421dca --- /dev/null +++ b/prompts/test-only.md @@ -0,0 +1,10 @@ +--- +includes: + - post-compact.md +--- +- **DO NOT** run compilation or `gn gen` unless explicitly written +- **DEFINITION OF DONE**: report on unit tests runtime run. + +# Task + +$@ diff --git a/skills/linux-chromium-build/SKILL.md b/skills/linux-chromium-build/SKILL.md index bfe5626..24f0784 100644 --- a/skills/linux-chromium-build/SKILL.md +++ b/skills/linux-chromium-build/SKILL.md @@ -72,7 +72,8 @@ TERM=xterm \ ./unit_tests.bash ``` -`unit_tests.bash` defaults to `out/unit_tests`. +`unit_tests.bash` defaults to `out/unit_tests`. To compile it inside `out/chrome` or other dirs +use `build_type= ./unit_tests.bash ` If you pass no target at all, get_all_targets_list() runs and builds every *_unittests target in the tree (filtering out //third_party, //chrome, //tools, //testing, plus blink). @@ -86,6 +87,13 @@ passing to ninja (build_gn_target ${target#//}). So the format is: //content/test:content_unittests ``` +Compilation of all unit_tests/browser_tests is also possible: + +```sh +./unit_tests.bash unit_tests +./unit_tests.bash browser_tests +``` + And example call: ```sh @@ -110,7 +118,7 @@ before the compiler starts. ```sh cd "$CHROMIUM_WRAPPER_DIR" -build_session="codex-build-$(date -u +%Y%m%dT%H%M%SZ)" +build_session="build-session-$(date -u +%Y%m%dT%H%M%SZ)" : > "${CHROMIUM_BUILD_LOG:?set CHROMIUM_BUILD_LOG}" printf '\n=== %s ===\n' "$build_session" >> "$CHROMIUM_BUILD_LOG" nohup env \ @@ -170,17 +178,16 @@ relying on general knowledge. Use targeted `rg` queries; do not use recursive PATH="$CHROMIUM_DEPOT_TOOLS_DIR:$CHROMIUM_WRAPPER_DIR/depot_tools:$PATH" \ TERM=xterm \ ./chrome.bash --configure - ``` - -### For a unit test - - ```sh + + # or for unit tests: cd "$CHROMIUM_WRAPPER_DIR" PATH="$CHROMIUM_DEPOT_TOOLS_DIR:$CHROMIUM_WRAPPER_DIR/depot_tools:$PATH" \ CCACHE_DISABLE=1 \ TERM=xterm \ ./unit_tests.bash --configure ``` +- `--configure` is unwanted flag, because it leads to recompilation of all + sources, so should not be used in normal scenarios. - For an undefined symbol, confirm the providing target appears in `deps` and that `is_component_build` in `args.gn` matches the intended build. - For a visibility error, inspect the dependency target's `visibility` before diff --git a/skills/lsp-cli/SKILL.md b/skills/lsp-cli/SKILL.md index b48af18..07d9f82 100644 --- a/skills/lsp-cli/SKILL.md +++ b/skills/lsp-cli/SKILL.md @@ -26,13 +26,16 @@ report the command and complete failure output, then STOP. Do not guess an endpoint, try an alternate endpoint, or create or replace a session cache without a new user instruction. +**ALWAYS** On any other errors related to `lsp-cli.py` -- report to user and +stop, do not act on your own. + ## Required diagnostic gate After every C/C++ source or header edit, run diagnostics for every modified file before starting a build: ```sh -./lsp-cli.py show-diagnostics path/to/modified/file1.cc path/to/modified/file2.h +./lsp-cli.py diagnostics path/to/modified/file1.cc path/to/modified/file2.h ``` Report all errors and warnings. Zero errors is required to proceed to a build. @@ -41,17 +44,99 @@ run, report the result and STOP. Do not build, retry, or substitute another compiler-diagnostic command until the user gives a new instruction. ## How to get more information + ```sh ./lsp-cli.py --help # to get available arguments and examples -./lsp-cli.py search-symbols --help #Help on how to search for C++ symbols in the codebase -./lsp-cli.py search-class --help #Help on how to search for C++ classes/structs/interfaces (kinds: Class, Struct, Interface) -./lsp-cli.py search-method --help #Help on how to search for C++ methods/functions/constructors (kinds: Method, Function, Constructor) -./lsp-cli.py analyze-symbol --help #Help on how to perform comprehensive analysis of a C++ symbol -./lsp-cli.py get-project-details --help #Help on how to get comprehensive project analysis including build configurations and global compilation database -./lsp-cli.py show-diagnostics --help #Help on how to show clangd diagnostics (errors/warnings/notes) for a source file ``` + `--build-directory` should be inherited from running server (no need to specify explicitly in normal usage) +## Typical usage examples + +All commands run from the project root (`/media/vptyp/images1/webos-ose/chromium/src`). +The server resolves the default build directory from the project-root +`compile_commands.json`, so `--build-directory` is only needed when you want a +different build dir than the default. + +### 1. Orient: project layout and index state + +```sh +./lsp-cli.py project # build dirs, compilation flags + compilation database the server resolved +./lsp-cli.py index # snapshot of clangd indexing progress +./lsp-cli.py index --wait-timeout 300 # block (up to 300s) until clangd reports done +``` + +Note: while clangd is *loading* an existing on-disk index (not building), +`index` may report `progress_percentage: 0.0` even though queries already +work. Trust a successful `search`/`analyze` over the percentage during load. + +### 2. Find symbols + +Search uses clangd fuzzy matching against the *qualified* name, so scoped +queries work: + +```sh +./lsp-cli.py search "InfoBar::Show" # scoped query -> infobars::InfoBar::Show +./lsp-cli.py search "Show" --kind Method # bare name, restrict to methods +./lsp-cli.py search "Buffer" --kind Class Struct # find types only +./lsp-cli.py search "" --files chrome/browser/extensions/api/tabs/tabs_api.cc # list a file's symbols +./lsp-cli.py search "std::" --include-external # include system/library symbols +./lsp-cli.py search "InfoBar" --max-results 20 # cap result count +``` + +Kinds are PascalCase, one or more: `Class Struct Function Method Field +Constructor Interface Module Namespace Enum` (see `search --help`). + +### 3. Deep analysis of one symbol + +`analyze` returns definition, hover, usage examples, and (when applicable) +inheritance + call hierarchy. It auto-selects the analyses that fit the symbol +kind — no flags for inheritance/call hierarchy. + +```sh +./lsp-cli.py analyze "InfoBar::Show" # full context +./lsp-cli.py analyze "InfoBar::Show" --max-examples 5 # cap usage examples +./lsp-cli.py analyze "foo" --location-hint path/to/file.cc:42:15 # disambiguate an overload +``` + +`--location-hint` is `FILE:LINE:COL` and resolves a specific overload when the +name alone is ambiguous. + +### 4. Diagnostic gate (run after every C/C++ edit) + +Accepts one or more files; run it for every modified source/header before a +build and require zero errors: + +```sh +./lsp-cli.py diagnostics chrome/browser/extensions/api/tabs/tabs_api.cc +./lsp-cli.py diagnostics a.cc b.h include/c.h # multi-file in one call +./lsp-cli.py diagnostics file.cc --wait-timeout 60 # give clangd longer to parse +``` + +If a file is still being reparsed, the first call after idle can be slow while +clangd pages its index back in from swap; subsequent calls are fast. Never hide +with grep timed_out or you may receive false results + +### 5. Output formats (apply to every command) + +```sh +./lsp-cli.py --format json search "Show" > show.json # tool result as JSON, for scripting +./lsp-cli.py --format raw diagnostics file.cc # full JSON-RPC response, for debugging +``` + +### Flag placement (important) + +- Global flags go **before** the subcommand: `--format`, `--server-path`, + `--http-url`, `--config`, `--debug`, and the hidden `--attach-timeout` + (default 30s; raise it if the CLI times out attaching to a busy server). +- Subcommand flags go **after** the subcommand: `--wait-timeout`, + `--build-directory`, `--kind`, `--files`, `--max-examples`, `--location-hint`. + +```sh +./lsp-cli.py --format json --attach-timeout 60 search "Show" --wait-timeout 120 +# ^^^^^^^^^^^^^ global ^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^ subcommand +``` + ### Possible CLI locations - Project root diff --git a/skills/webos-ose-build/SKILL.md b/skills/webos-ose-build/SKILL.md index 383a00d..6266b7c 100644 --- a/skills/webos-ose-build/SKILL.md +++ b/skills/webos-ose-build/SKILL.md @@ -80,7 +80,7 @@ recorded byte offset). Do not cite earlier log entries as evidence about the current build. ```sh -build_session="codex-bitbake-$(date -u +%Y%m%dT%H%M%SZ)" +build_session="build-bitbake-$(date -u +%Y%m%dT%H%M%SZ)" : > "${WEBOS_BUILD_LOG:?set WEBOS_BUILD_LOG}" printf '\n=== %s ===\n' "$build_session" >> "$WEBOS_BUILD_LOG" nohup bitbake "$WEBOS_BITBAKE_TARGET" \ diff --git a/testing/helpers.py b/testing/helpers.py new file mode 100644 index 0000000..0c45902 --- /dev/null +++ b/testing/helpers.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +from typing import Any, Dict, List + +PASS = {"pass": True, "score": 1.0, "reason": ""} +FAIL = {"pass": False, "score": 0.0, "reason": ""} + + +def _bash_args(context: Dict[str, Any]) -> List[Dict[str, Any]]: + """Returns the args dict of every bash tool call in the trajectory.""" + tool_calls = (context.get("metadata") or {}).get("toolCalls") or [] + return [t.get("args", {}) for t in tool_calls if t.get("tool") == "bash"] + + +def _result(passed: bool, reason: str, score: float = 1.0) -> Dict[str, Any]: + if passed: + return {"pass": True, "score": score, "reason": reason} + return {"pass": False, "score": 0.0, "reason": reason} + + +def assert_exec(output: str, context: Dict[str, Any], expected, result: bool) -> Dict[str, Any]: + args_list = _bash_args(context) + if not args_list: + return _result(not result, "No bash tool calls were made at all") + + for args in args_list: + binaries = args.get("binaries", []) + per_command = [c.get("binary") for c in args.get("commands", [])] + for entity in expected: + if entity in binaries or entity in per_command: + return _result(result, f"bash invoked '{entity}'") + + seen = sorted({b for a in args_list for b in a.get("binaries", [])}) + return _result(not result, f"'{expected}' was never invoked. Binaries seen: {seen or 'none'}") + +def assert_no_compile(output: str, context: Dict[str, Any]) -> Dict[str, Any]: + """Asserts a specific binaries was not executed in any bash call (config: binary).""" + expected = ["ninja", "autoninja"] + return assert_exec(output, context, expected, False) + +def assert_run_unittests(output: str, context: Dict[str, Any]) -> Dict[str, Any]: + """Assert that unittests has started """ + expected = ["./out/chrome/unit_tests", "./unit_tests"] + return assert_exec(output, context, expected, True) diff --git a/testing/promptfooconfig.yaml b/testing/promptfooconfig.yaml new file mode 100644 index 0000000..d6e2a5b --- /dev/null +++ b/testing/promptfooconfig.yaml @@ -0,0 +1,86 @@ +tracing: + enabled: true + otlp: + http: + enabled: true + port: 4318 + +prompts: + - "{{prompt}}" + +providers: + - id: "file:///home/vptyp/git/promptfoo-eval-pi/pi_provider.py" + label: "pi-agent" + config: + timeout: 6000000 + pythonExecutable: "/home/vptyp/git/promptfoo-eval-pi/uv_python.sh" + isolation: + enabled: false + strategy: "git-worktree" + clean: false + +defaultTest: + options: + # Universal Agent-as-a-Judge provider (supports agy, claude, codex, openclaw, etc.) + provider: + id: "file:///home/vptyp/git/promptfoo-eval-pi/eval_judge.py" + config: + command: "agy -p" + +tests: + - description: "[PROMPTS] research-init: reads sources, writes plan, never builds" + vars: + prompt: "/research-init Add a build flag for enabling of infobars visibility through tabs api to a Chromium component. Write to IMPLEMENTATION_PLAN_VISIBILITY_BUILD_FLAG.md" + assert: + - type: javascript + value: "context.metadata.exitCode === 0" + - type: skill-used + value: "chromium-docs" + - type: python + value: | + tool_calls = (context.get("metadata") or {}).get("toolCalls") or [] + edited_cc = [ + (tc.get("args") or {}).get("path", "?") + for tc in tool_calls + if tc.get("tool") == "edit" + and str((tc.get("args") or {}).get("path", "")).endswith(".cc") + ] + return { + "pass": not edited_cc, + "score": 1.0 if not edited_cc else 0.0, + "reason": "no .cc files edited" if not edited_cc else f"edit used on .cc files: {edited_cc}", + } + - type: python + value: "file://helpers.py:assert_no_compile" + - type: trajectory:tool-sequence + value: + steps: + - read + - write + + - description: "[SKILLS] test-only: runs unit tests through bash" + vars: + prompt: "Run the compiled unit tests for Linux Chromium x86 build (out/chrome). Subset of tests: TabsApiUnitTest.SetSharingInfobarsVisibility*. If unit_tests non exist yet: stop and report, do not act on your own." + assert: + - type: skill-used + value: "linux-chromium-build" + - type: javascript + value: "context.metadata.exitCode === 0" + - type: python + value: "file://helpers.py:assert_run_unittests" + + - description: "[SKILLS] Discover and used lsp-cli.py skill on direct request" + vars: + prompt: "With help of language-server build callers callStack of InfoBar::Show method; depth 5" + assert: + - type: skill-used + value: "lsp-cli" + - type: trajectory:tool-args-match + value: + name: "bash" + args: + has_binary: + "./lsp-cli.py": true + has_word: + "analyze": true + "InfoBarManager::AddInfoBarInternal": true