From f77b0b440d9dbef7f94b412e3a995a40e98a6f4b Mon Sep 17 00:00:00 2001 From: Evandro Date: Thu, 24 Sep 2026 21:52:06 -0300 Subject: [PATCH] chore(qa): replace the keyword-count harness with a structural lint and gate tests The old harness scored occurrences of words like VERIFICAR, so its score measured wording, and it failed on a fresh clone. The lint checks frontmatter, size limits, references and links; test_gate.py exercises the gate and hooks on a throwaway repo; check_upstream.py flags vendored skills behind upstream. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 22 +++ .github/workflows/upstream.yml | 17 ++ CHANGELOG.md | 24 +++ CONTRIBUTING.md | 41 ++--- README.md | 305 ++++++++---------------------- README.pt-BR.md | 307 +++++++++---------------------- docs/SKILLS.md | 158 ++++++---------- qa/baseline.json | 236 ------------------------ qa/check_upstream.py | 35 ++++ qa/lint_skills.py | 126 +++++++++++++ qa/snapshots/baseline-final.json | 236 ------------------------ qa/test_gate.py | 97 ++++++++++ qa/test_quality.py | 101 ---------- qa/test_tests.py | 128 ------------- qa/wf_quality_harness.py | 276 --------------------------- 15 files changed, 560 insertions(+), 1549 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/upstream.yml create mode 100644 CHANGELOG.md delete mode 100644 qa/baseline.json create mode 100644 qa/check_upstream.py create mode 100644 qa/lint_skills.py delete mode 100644 qa/snapshots/baseline-final.json create mode 100644 qa/test_gate.py delete mode 100644 qa/test_quality.py delete mode 100644 qa/test_tests.py delete mode 100644 qa/wf_quality_harness.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c0efd44 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Lint das skills, agentes e hooks + run: python3 qa/lint_skills.py + - name: Testes do gate e dos hooks + run: python3 qa/test_gate.py diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml new file mode 100644 index 0000000..81194b3 --- /dev/null +++ b/.github/workflows/upstream.yml @@ -0,0 +1,17 @@ +name: upstream + +on: + schedule: + - cron: "0 9 1 * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + vendored-skills: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Skills vendorizadas atrás do upstream + run: python3 qa/check_upstream.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2873b3c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,24 @@ +# Changelog + +## 2.0.0 (2026-09-24) + +### Added +- Claude Code plugin packaging (`.claude-plugin/plugin.json`, `marketplace.json`); workflows moved to `skills//SKILL.md`. +- `dev-router`: tiers 0 to 3 with security triggers that raise the tier, so small requests skip heavy phases. +- `lean-code`: reuse ladder, dependency justification, diff budget per tier. +- `wf-bugfix` and `wf-refactor`. +- Agents: `explorer`, `planner`, `implementer`, `reviewer`, `security-auditor`, `ui-critic`, `verifier`, each with only the tools its role needs. +- `scripts/quality-gate`: deterministic checks per tier, scoped to the diff; missing tools reported as unverified; new dependencies listed. +- Hooks: `Stop` gate, shell command guard, formatter after edits. +- opencode adapter and `scripts/install.py` for Claude Code, opencode and Hermes Agent. +- taste-skill (`design-taste-frontend`, `redesign-existing-projects`) vendored for `wf-frontend`. +- `qa/lint_skills.py`, `qa/test_gate.py`, `qa/check_upstream.py` and CI. + +### Changed +- The five workflows were cut to about 100 lines each; phases state their minimum tier; grep-count checks replaced by the gate and real tools. +- Security references moved to OWASP Top 10:2025, ASVS 5.0 and OWASP LLM Top 10 (2025). +- Machine-specific paths (`~/.hermes/...`, `vision_analyze`) and references to skills that do not exist publicly removed. + +### Removed +- The keyword-count quality harness (`qa/wf_quality_harness.py`, `test_quality.py`, `test_tests.py`, snapshots). It scored occurrences of words like `VERIFICAR` and `NUNCA`, so its "+98.6%" measured wording, not quality; it also failed on a fresh clone because it looked for skills in `~/.hermes`. +- apisec-inc's `api-security-review` from `wf-backend` (name collision with the vendored OWASP skill). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 111d6b9..a1f53f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,36 +1,29 @@ # Contributing -Thanks for taking the time to contribute. This project is small, so keep it simple: open an issue to discuss first, then send a PR. +Open an issue to discuss first, then send a small, focused PR. -## What to improve +## Rules -Workflows, skills, or the QA harness. All changes should keep the same rules the workflows themselves enforce: - -1. **Grounded.** Every claim in a workflow or README must trace back to a real repository fact. No fabricated commands, examples, or badges. -2. **Verifiable phases.** A phase is not done until it has a verification step (a command that proves the work). Add one if you touch a workflow. -3. **No AI-isms.** Docs and workflows must pass the avoid-ai-writing check (no em-dash runs, no "robust/seamless/leverage" filler). - -## What not to change - -- The QA harness must keep its meta-tests green. If you change scoring, re-run `qa/test_tests.py` and update `qa/snapshots/baseline-final.json`. -- Do not remove skills referenced by workflows without updating the workflows and `docs/SKILLS.md` together. +1. **Grounded.** Every command, path and claim in a skill or README traces to a real fact. No invented commands or badges. +2. **Short skills.** `SKILL.md` stays under 150 lines; details go to `references/` next to it. +3. **Proportional.** Each phase says from which tier it runs. Nothing is mandatory for every request unless it is cheap (the tier 0 gate). +4. **Checks are tools, not keyword counts.** A verification step runs something that can fail (tests, scanners, the gate). `grep -c` of a word proves nothing. +5. **No AI-isms** in docs and skills (no em-dash runs, no "robust/seamless/leverage"). +6. **Vendored skills are copied 1:1** with their license, and recorded in `skills/NOTICE.md` (origin, commit, license). ## Local checks before a PR ```bash -cd qa -python3 wf_quality_harness.py # score must not drop below baseline -python3 test_quality.py # gates green -python3 test_tests.py # meta-tests green +python3 qa/lint_skills.py +python3 qa/test_gate.py +claude plugin validate --strict .claude-plugin/plugin.json ``` +A new skill referenced by a workflow goes into its `metadata.dev-workflows.uses` (vendored) or +`external` (listed in `docs/SKILLS.md`); the lint checks both. + ## Process -1. Fork the repo. -2. Create a branch: `git checkout -b feat/your-change`. -3. Make the change. Keep it small and focused. -4. Run the qa checks above. -5. Commit with a clear message. -6. Open a PR describing what changed and why. - -Questions or ideas: open an issue. +1. Fork and branch: `git checkout -b feat/your-change`. +2. Make the change and run the checks above. +3. Open a PR saying what changed and why. diff --git a/README.md b/README.md index 6570b4f..b04abc9 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,11 @@ # Dev Workflows -**Agent workflows as SKILL.md files**: frontend, backend, architecture and security code review that never skip verification. +**A Claude Code plugin (also for opencode and Hermes Agent) that sizes the process to the request**: a router picks the tier, least-privilege agents do the work, and a deterministic quality gate decides when it is done. [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -[![Python 3.10+](https://img.shields.io/badge/Python-3.10+-3776AB?logo=python&logoColor=white)](qa/wf_quality_harness.py) [![Agent Skills](https://img.shields.io/badge/Agent%20Skills-1.0-8A2BE2)](https://agentskills.io) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/evandrodevbr/dev-workflows/pulls) - -Turn your coding agent into a repeatable, verifiable process. Each workflow defines phases, concrete verification steps, real commands, and hard rules. The agent delivers work that can be *checked*, not just *claimed*. +[![CI](https://github.com/evandrodevbr/dev-workflows/actions/workflows/ci.yml/badge.svg)](.github/workflows/ci.yml) **🇧🇷 [Ler em português](README.pt-BR.md)** @@ -19,251 +16,111 @@ Turn your coding agent into a repeatable, verifiable process. Each workflow defi ## Why -Agents are great at generating code and terrible at admitting they don't know. This project fixes the second part. +Agents write code fast and are bad at two things: knowing when a task deserves more process, and +admitting what they did not verify. This plugin handles both: -Every workflow here forces the agent to **prove** the work at each phase: +- **Process proportional to the request.** A typo does not get a design review; an auth change does. +- **Proof, not claims.** "Done" requires a quality-gate report on the current diff. A tool that is not installed shows up as *unverified*, never as passed. +- **Least code.** Reuse before writing, delete before adding, no dependency without a reason. +- **Security from live sources.** A model's CVE knowledge stops at its training cutoff, so reviews query OSV.dev and stamp the date. -- **Verification steps**: run the build, run the tests, hit the API, consult the registry. Not "I think it works". -- **Acceptance criteria**: checkbox gates that must be marked before a phase closes. -- **Hard rules**: "never merge with a red build", "never claim 'no CVE' without querying OSV.dev". -- **A QA harness**: a scorecard that measures workflow quality objectively, plus meta-tests that corrupt the harness to prove it detects regressions. +## How it works -## Table of contents +`dev-router` loads first and declares a tier: -- [Workflows](#workflows) -- [Security: do not trust the training cutoff](#security-do-not-trust-the-training-cutoff) -- [Skills used, and their owners](#skills-used-and-their-owners) -- [Quick start](#quick-start) -- [The QA harness](#the-qa-harness) -- [Quality tests and measured gains](#quality-tests-and-measured-gains) -- [Repository layout](#repository-layout) -- [License](#license) - -## Workflows - -| Workflow | File | Covers | +| Tier | When | What runs | |---|---|---| -| 🎨 **Frontend / UI** | [workflows/wf-frontend.md](workflows/wf-frontend.md) | Design direction before code, anti-AI-slop, React performance, visual QA for PDFs | -| ⚙️ **Backend / API** | [workflows/wf-backend.md](workflows/wf-backend.md) | Contract before handler, safe defaults, OWASP audit, tests + adversarial bug hunt | -| 🏗️ **Architecture** | [workflows/wf-architecture.md](workflows/wf-architecture.md) | Measurable requirements, C4 diagrams, ADRs, threat modeling, review | -| 🛡️ **Security review** | [workflows/wf-security-review.md](workflows/wf-security-review.md) | Code review (front + back), known CVEs **and** live lookup of new ones | -| 📄 **README** | [workflows/wf-readme.md](workflows/wf-readme.md) | Create/audit/rewrite READMEs grounded in real repo facts | +| 0, trivial | nothing changes behavior | the change + gate tier 0 | +| 1, small | 1–3 files, no new public surface | `wf-bugfix` or `wf-refactor`, regression test, gate tier 1 | +| 2, feature | new endpoint/screen, contract change, new dependency | explore, short plan, TDD, independent review, gate tier 2 | +| 3, large/risky | several modules, existing data, architecture, release | EARS spec + ADR approved by you, reviews, security audit, gate tier 3 | -Each file is a standard `SKILL.md`. The `description` in the frontmatter is the trigger: when the agent's context matches it, the workflow loads automatically. Works in [Hermes Agent](https://hermes-agent.nousresearch.com), Claude Code, and anything that reads the Agent Skills format. +Security triggers raise the tier at any point: auth, money, personal data, uploads, raw SQL, +shell/exec, deserialization, schema migrations, new dependencies, features that call an LLM. -## Security: do not trust the training cutoff +## What is inside -A model's knowledge of CVEs stops at its training date. Anything disclosed after that, it has never seen, so "no vulnerabilities" from memory is a guess, not an answer. +| Kind | Items | +|---|---| +| Router and rules | `dev-router`, `lean-code` | +| Workflows | `wf-bugfix`, `wf-refactor`, `wf-frontend`, `wf-backend`, `wf-architecture`, `wf-security-review`, `wf-readme` | +| Agents | `explorer` (haiku, read-only), `planner` (opus, read-only), `implementer` (sonnet), `reviewer` (opus, read-only, sees only the diff), `security-auditor` (opus, read-only), `ui-critic` (sonnet, read-only), `verifier` (haiku, runs the gate) | +| Hooks | `Stop`: blocks "done" while code changed without a fresh passing gate · `PreToolUse`: denies `--no-verify` and force-push, asks before `reset --hard`, `clean -f`, `DROP TABLE` · `PostToolUse`: formats edited files with the project's own formatter | +| Quality gate | [`scripts/quality-gate`](scripts/quality-gate): lint, typecheck, tests, gitleaks (tier 0); diff coverage, semgrep, dead code (tier 1); build, mutation testing, osv-scanner, architecture rules, duplication (tier 2); trivy, requirement→test traceability (tier 3) | +| Vendored skills | security pillar (OWASP secure-agent-playbook, UnitOneAI SecuritySkills, safedeps, hermaguard) and taste-skill for frontend. Origin and license: [`skills/NOTICE.md`](skills/NOTICE.md) | -The security workflow is built around that fact: +The gate runs only the checks of the requested tier, only on the diff, and needs no AI: it is +plain tooling. Checks that need the app running (Lighthouse CI budgets, Playwright + axe, schemathesis, +k6) are added per project in `.dev-workflows.toml`. Reference: +[`skills/dev-router/references/quality-gate.md`](skills/dev-router/references/quality-gate.md). -``` -Code/diff → SCOPE → HUNT (known CVEs) → CONSULT (live sources) → CODE REVIEW → REPORT + GATE - │ │ - │ └─ OSV.dev · GitHub Advisory DB · NVD - └──────────────┘ -``` - -Every security review **must** consult live sources and stamp the date. A review with no source and no date is expired on arrival. - -## Skills used, and their owners - -Workflows load these skills by context. The three pillars of the security review come from primary security sources (OWASP, OSV.dev, UnitOne SecuritySkills), not from a model's memory. - -### 🛡️ Security review: the three pillars - -_Vendored under `skills/` — see [`skills/NOTICE.md`](skills/NOTICE.md) for exact origin and license._ - -| Skill | What it does | Owner | -|---|---|---| -| [`safedeps`](https://github.com/Jeneidi/safedeps) | Queries OSV.dev in real time; returns CVE + severity + fixed version for a `package@version`. Covers the gap a frozen model can't. | [Jeneidi](https://github.com/Jeneidi) | -| [`sca-audit`](https://github.com/OWASP/secure-agent-playbook) | Supply-chain audit of dependencies with reachability analysis and CWE mapping. | [OWASP](https://github.com/OWASP) | -| [`code-review-security`](https://github.com/OWASP/secure-agent-playbook) | Systematic security code review mapped to OWASP Top 10 + ASVS. | [OWASP](https://github.com/OWASP) | - -### 🛡️ Security review: supporting skills - -_Also vendored under `skills/`._ - -| Skill | What it does | Owner | -|---|---|---| -| [`secrets-scan`](https://github.com/OWASP/secure-agent-playbook) | Finds hardcoded credentials / API keys in code and git history. | [OWASP](https://github.com/OWASP) | -| [`api-security-review`](https://github.com/OWASP/secure-agent-playbook) | API review against OWASP API Security Top 10. | [OWASP](https://github.com/OWASP) | -| [`web-security-review`](https://github.com/OWASP/secure-agent-playbook) | Web app review against OWASP Top 10. | [OWASP](https://github.com/OWASP) | -| [`cve-triage`](https://github.com/UnitOneAI/SecuritySkills) | Prioritizes CVEs using CVSS 4.0 / EPSS / CISA KEV. | [UnitOneAI](https://github.com/UnitOneAI) | -| [`patch-prioritization`](https://github.com/UnitOneAI/SecuritySkills) | Decides remediation order. | [UnitOneAI](https://github.com/UnitOneAI) | -| [`dependency-scanning`](https://github.com/UnitOneAI/SecuritySkills) | Dependency tree scanning. | [UnitOneAI](https://github.com/UnitOneAI) | -| [`hermaguard`](https://github.com/Sahil-SS9/hermaguard) | Adversarial review: static pre-scan + 3 specialist agents (edge cases, attack, blast radius). | [Sahil-SS9](https://github.com/Sahil-SS9) | - -### 🎨 Frontend / UI - -| Skill | What it does | Owner | -|---|---|---| -| [`frontend-design`](https://github.com/anthropics/skills) | Intentional visual direction, typography, avoiding "generic AI" output. | [Anthropic](https://github.com/anthropics) | -| [`web-design`](https://github.com/KAOPU-XiaoPu/web-design) | Cohesive web aesthetics. | [KAOPU-XiaoPu](https://github.com/KAOPU-XiaoPu) | -| [`vercel-react-best-practices`](https://github.com/vercel-labs/agent-skills) | 40+ React/Next performance rules from Vercel engineering. | [Vercel Labs](https://github.com/vercel-labs) | -| [`vercel-composition-patterns`](https://github.com/vercel-labs/agent-skills) | Compound components, clean composition. | [Vercel Labs](https://github.com/vercel-labs) | -| [`animate`](https://github.com/emilkowalski/skill) | Purpose-driven motion. | [emilkowalski](https://github.com/emilkowalski) | -| [`impeccable`](https://github.com/pbakaus/impeccable) | The missing design vocabulary for agents: 23 commands (craft, shape, audit, polish, animate, live) and 59 deterministic anti-slop rules. The most-used frontend design skill (230k+ installs). | [pbakaus](https://github.com/pbakaus) | -| [`design-taste-frontend`](https://github.com/Leonxlnx/taste-skill) | Generates new UI with a direction inferred from the brief; three 1–10 dials (variance, motion, density). Vendored in `skills/`. | [Leonxlnx](https://github.com/Leonxlnx) | -| [`redesign-existing-projects`](https://github.com/Leonxlnx/taste-skill) | Audits an existing UI and upgrades it without breaking behavior. Vendored in `skills/`. | [Leonxlnx](https://github.com/Leonxlnx) | -| [`anti-ai-slop`](https://github.com/evandrodevbr/dev-workflows) | Detects "generated-by-AI" visual patterns (auto-loaded, local). | community skill | -| [`avoid-ai-writing`](https://github.com/conorbronsdon/avoid-ai-writing) | Removes AI-isms from microcopy, labels and docs. | [conorbronsdon](https://github.com/conorbronsdon) | - -### ⚙️ Backend / API - -| Skill | What it does | Owner | -|---|---|---| -| [`secure-coding`](https://github.com/securityreviewai/secure-coding-skill) | Secure coding patterns covering 15 stacks. | [securityreviewai](https://github.com/securityreviewai) | -| [`bola-detector`](https://github.com/apisec-inc/apisec-skills) | Broken object-level authorization (OWASP API1). | [apisec-inc](https://github.com/apisec-inc) | -| [`auth-rbac-scaffold`](https://github.com/apisec-inc/apisec-skills) | Authentication + RBAC (OWASP API2 / API5). | [apisec-inc](https://github.com/apisec-inc) | -| [`injection-checker`](https://github.com/apisec-inc/apisec-skills) | SQL / ORM / shell / template injection (OWASP API8). | [apisec-inc](https://github.com/apisec-inc) | -| [`openapi-hardener`](https://github.com/apisec-inc/apisec-skills) | Sanitizes OpenAPI / Zod / Pydantic schemas (OWASP API3). | [apisec-inc](https://github.com/apisec-inc) | -| [`api-security-review`](https://github.com/apisec-inc/apisec-skills) | Full OWASP API Top 10 review. | [apisec-inc](https://github.com/apisec-inc) | -| [`security-test-generator`](https://github.com/apisec-inc/apisec-skills) | Generates security test suites. | [apisec-inc](https://github.com/apisec-inc) | - -> ⚠️ `api-security-review` is two unrelated skills sharing one name: the OWASP one vendored in `skills/` (security pillar, above) and the apisec-inc one in this table. Only the OWASP copy ships in this repository. Install the one your workflow needs — with both present, agents that resolve by skill name report a collision. - -### 🏗️ Architecture - -| Skill | What it does | Owner | -|---|---|---| -| [`system-design`](https://github.com/Kotivskyi/architecture-governance-skills) | HelloInterview system-design framework. | [Kotivskyi](https://github.com/Kotivskyi) | -| [`c4-architecture`](https://github.com/Kotivskyi/architecture-governance-skills) | C4 diagrams (Mermaid / Structurizr). | [Kotivskyi](https://github.com/Kotivskyi) | -| [`isaqb-architecture-governance`](https://github.com/Kotivskyi/architecture-governance-skills) | arc42 + ADRs. | [Kotivskyi](https://github.com/Kotivskyi) | -| [`secure-architecture-governance`](https://github.com/Kotivskyi/architecture-governance-skills) | STRIDE+CIA threat models, security ADRs. | [Kotivskyi](https://github.com/Kotivskyi) | - -> 📖 Full install instructions for every skill live in [docs/SKILLS.md](docs/SKILLS.md). - -## Quick start +## Install ```bash -# 1. Copy the workflows into your agent's skills directory. -# Each skill needs its own directory containing a SKILL.md -- a loose .md -# file sitting in a category folder is not indexed (neither Hermes nor -# Claude Code loads it), so the loop below builds the right shape. -SKILLS_DIR="$HOME/.hermes/skills/software-development" -# Claude Code: SKILLS_DIR="$HOME/.claude/skills" +git clone https://github.com/evandrodevbr/dev-workflows && cd dev-workflows -for f in workflows/*.md; do - name=$(basename "$f" .md) - mkdir -p "$SKILLS_DIR/$name" - cp "$f" "$SKILLS_DIR/$name/SKILL.md" -done - -# 2. (Optional, for security review) the supporting skills are already -# vendored in skills/ -- no external clone needed, just copy: -cp -r skills/* "$HOME/.hermes/skills/" - -# 3. Use it -# "review this PR for security" -> wf-security-review loads -# "build a new page" -> wf-frontend loads +python3 scripts/install.py claude # plugin: skills + agents + hooks +python3 scripts/install.py opencode # skills, agents and a plugin in ~/.config/opencode +python3 scripts/install.py hermes # skills only, in ~/.hermes/skills ``` -Verify the install took: the workflows should show up in your agent's skill -list (`hermes skills list` for Hermes Agent). +Claude Code without cloning: -## The QA harness +```text +/plugin marketplace add evandrodevbr/dev-workflows +/plugin install dev-workflows@dev-workflows +``` -The `qa/` directory keeps the workflows themselves honest: +Differences between agents: + +- **opencode** cannot block the end of a turn, so a missing or failing gate is logged instead of blocking. Agents are converted to opencode subagents with the same permissions. +- **Hermes** gets the skills only; run the gate yourself. +- For opencode and Hermes the copies point at your checkout, so keep it in place (or re-run the install after moving it). + +Turn the `Stop` hook off for one session with `DW_GATE=off`. + +## External skills (optional) + +Workflows use these when installed and fall back to their own rules when not. Install commands: +[`docs/SKILLS.md`](docs/SKILLS.md). + +| Area | Skills | +|---|---| +| Frontend | `frontend-design` (Anthropic), `impeccable` (pbakaus), `vercel-react-best-practices`, `vercel-composition-patterns` (Vercel Labs), `animate` (emilkowalski), `web-design`, `avoid-ai-writing` | +| Backend | `secure-coding`, `bola-detector`, `auth-rbac-scaffold`, `injection-checker`, `openapi-hardener`, `security-test-generator` (apisec-inc) | +| Architecture | `system-design`, `c4-architecture`, `isaqb-architecture-governance`, `secure-architecture-governance` (Kotivskyi) | +| README | `readme-crafter`, `good-readme`, `curating-readme` | + +## Quality of this repository ```bash -cd qa - -python3 wf_quality_harness.py # score each workflow on 30+ criteria -python3 test_quality.py # gates: no regression, valid skills, phases verified -python3 test_tests.py # meta-tests: corrupt the harness, prove it detects the corruption +python3 qa/lint_skills.py # frontmatter, size limits, references, links, machine-specific paths +python3 qa/test_gate.py # quality gate and hooks on a throwaway repo +python3 qa/check_upstream.py # vendored skills behind their upstream +claude plugin validate --strict .claude-plugin/plugin.json ``` -The meta-tests are the interesting part. They deliberately break the harness and the workflows, then confirm the score *drops*. If the score didn't move, the detector would be useless. These tests stay green as a contract. +CI runs the first two on every PR; the upstream check runs monthly. -## Quality tests and measured gains +**Not measured yet:** whether the kit improves the code an agent produces. Public studies +([SkillsBench](https://arxiv.org/abs/2602.12670), [SWE-Skills-Bench](https://arxiv.org/abs/2603.15401)) +show skills can also make results worse, so the next step is an A/B benchmark (with vs. without the +kit) on [Harbor](https://github.com/harbor-framework/harbor) with hidden tests, mutation score and +security checks. -These are real numbers from the harness runs, stored in `qa/snapshots/baseline-final.json`. The baseline column is the state before the refinement work; the final column is the result after it. - -### Workflow quality score (same harness, before → after) - -| Workflow | Baseline | Final | Gain | -|---|---|---|---| -| wf-frontend | 201.0 | 385.5 | +92% | -| wf-backend | 215.0 | 424.0 | +97% | -| wf-architecture | 199.9 | 413.7 | +107% | -| wf-security-review | 332.0 | 390.0 | +17% | -| **Total** | **615.9** | **1223.2** | **+98.6%** | - -That is almost exactly double the quality of the workflows, measured with the same ruler. - -The ruler itself also grew stricter over the project (13 → 30+ criteria: verification per phase, real commands, anti-patterns, acceptance criteria, artifacts, flow). That is why the early runs show lower totals: `base-364` (13 criteria) → `baseline-v2` (536.7) → `baseline-v3` (675.0) → `baseline-v4` (884.6) → `baseline-final` (1613.2 across the four scored workflows). Only the final snapshot is committed, as `qa/snapshots/baseline-final.json` — that is the file the gates read and the file to update when scoring changes. The fair before/after comparison above uses the final ruler for both sides. - -### Gates (regression checks): all green - -| Gate | What it checks | Status | -|---|---|---| -| G1 | Score did not drop below baseline | ✅ | -| G4 | Every referenced skill exists | ✅ 4/4 workflows | -| G5 | Every phase has ≥3 verification verbs | ✅ 4/4 workflows | -| Meta-tests | Corrupt harness/workflows → score drops (proves detector works) | ✅ | - -### What the refinement added - -Every workflow gained the same batch of real improvements: -- **Verification steps** with actual commands (`curl`, `grep`, `python3`, `git`, `npm`, `pytest`) instead of prose. -- **Per-phase acceptance checklists** (markable `[ ]` gates). -- **Anti-pattern blocks** per phase (what not to do, and why). -- **Executable-command sections** and **worked examples per phase**. -- **Checkpoints** that pause for user approval between phases. - -The security workflow gained a hard rule: *never claim "no CVE" without querying OSV.dev / GitHub Advisory*. A model's knowledge stops at its training cutoff; the live sources don't. - -### Context on the "double" target - -The harness has a physical ceiling (sum of all criterion caps). The numeric "2× of baseline" target would need a ruler with more resolution to be expressible, so inflating the score to hit it would be gaming the metric. What shipped is the honest version: real quality ~doubled on the same ruler (+98.6%), with meta-tests proving the ruler detects regressions. - - -## Repository layout +## Layout ``` -dev-workflows/ -├── README.md # this file -├── README.pt-BR.md # Portuguese version -├── workflows/ # the five SKILL.md workflows -│ ├── wf-frontend.md -│ ├── wf-backend.md -│ ├── wf-architecture.md -│ ├── wf-security-review.md -│ └── wf-readme.md -├── skills/ # the security pillar, vendored (see skills/NOTICE.md) -│ ├── safedeps/ # + plays/, templates/, schemas/, docs/ support files -│ ├── sca-audit/ -│ ├── code-review-security/ -│ ├── api-security-review/ -│ ├── web-security-review/ -│ ├── secrets-scan/ -│ ├── cve-triage/ -│ ├── patch-prioritization/ -│ ├── dependency-scanning/ -│ ├── hermaguard/ -│ ├── design-taste-frontend/ # taste-skill (frontend) -│ └── redesign-existing-projects/ # taste-skill (frontend) -├── docs/ -│ └── SKILLS.md # full skill inventory + install steps -├── CONTRIBUTING.md # what to improve, local checks, PR process -└── qa/ # quality harness, gates, meta-tests - ├── wf_quality_harness.py - ├── test_quality.py - ├── test_tests.py - ├── baseline.json - └── snapshots/ - └── baseline-final.json +.claude-plugin/ plugin.json, marketplace.json +skills/ dev-router, lean-code, wf-* workflows, vendored skills (NOTICE.md) +agents/ explorer, planner, implementer, reviewer, security-auditor, ui-critic, verifier +hooks/hooks.json Stop, PreToolUse, PostToolUse +scripts/ quality-gate, install.py, hooks/*.py +adapters/opencode/ opencode plugin +qa/ lint, gate tests, upstream check +docs/SKILLS.md skill inventory and install commands ``` ## License -[MIT](LICENSE) © 2026 [Evandro Fonseca Junior](https://github.com/evandrodevbr) - ---- - -
- -⭐ If this saves you from one "works on my machine" merge, star it. - -
+[MIT](LICENSE) © 2026 [Evandro Fonseca Junior](https://github.com/evandrodevbr). Vendored skills keep their own licenses ([`skills/NOTICE.md`](skills/NOTICE.md)). diff --git a/README.pt-BR.md b/README.pt-BR.md index ec0fc4b..74ab795 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -2,16 +2,13 @@ # Dev Workflows -**Workflows de agente como arquivos SKILL.md**: frontend, backend, arquitetura e code review de segurança que nunca pulam a verificação. +**Plugin do Claude Code (também para opencode e Hermes Agent) que ajusta o processo ao tamanho do pedido**: um roteador escolhe o nível, agentes com permissão mínima fazem o trabalho e um gate de qualidade determinístico decide quando terminou. [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -[![Python 3.10+](https://img.shields.io/badge/Python-3.10+-3776AB?logo=python&logoColor=white)](qa/wf_quality_harness.py) [![Agent Skills](https://img.shields.io/badge/Agent%20Skills-1.0-8A2BE2)](https://agentskills.io) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/evandrodevbr/dev-workflows/pulls) +[![CI](https://github.com/evandrodevbr/dev-workflows/actions/workflows/ci.yml/badge.svg)](.github/workflows/ci.yml) -Transforme seu agente de código num processo repetível e verificável. Cada workflow define fases, passos de verificação concretos, comandos reais e regras duras. O agente entrega trabalho que dá pra *checar*, não só *afirmar*. - -**🇺🇸 [Read this in English](README.md)** +**🇺🇸 [Read in English](README.md)** @@ -19,251 +16,111 @@ Transforme seu agente de código num processo repetível e verificável. Cada wo ## Por quê -Agente é ótimo pra gerar código e péssimo pra admitir que não sabe. Este projeto resolve a segunda parte. +Agentes escrevem código rápido e erram em duas coisas: saber quando uma tarefa merece mais processo +e admitir o que não verificaram. Este plugin cuida das duas: -Cada workflow força o agente a **provar** o trabalho em cada fase: +- **Processo proporcional ao pedido.** Typo não passa por revisão de design; mudança de auth passa. +- **Prova, não afirmação.** "Pronto" exige um relatório do gate sobre o diff atual. Ferramenta não instalada aparece como *não verificado*, nunca como aprovada. +- **Menos código.** Reaproveitar antes de escrever, apagar antes de somar, nenhuma dependência sem motivo. +- **Segurança com fontes ao vivo.** O conhecimento de CVEs de um modelo para na data de treino, então as revisões consultam o OSV.dev e registram a data. -- **Passos de verificação**: rodar build, rodar testes, chamar a API, consultar o registry. Não "acho que funciona". -- **Critérios de aceite**: portões com checkbox que precisam estar marcados pra fase fechar. -- **Regras duras**: "nunca faça merge com build vermelho", "nunca afirme 'sem CVE' sem consultar o OSV.dev". -- **Harness de QA**: um placar que mede a qualidade dos workflows de forma objetiva, mais meta-testes que corrompem o harness pra provar que ele detecta regressão. +## Como funciona -## Sumário +O `dev-router` carrega primeiro e declara um nível: -- [Workflows](#workflows) -- [Segurança: não confie no cutoff de treino](#segurança-não-confie-no-cutoff-de-treino) -- [Skills usadas e seus donos](#skills-usadas-e-seus-donos) -- [Começo rápido](#começo-rápido) -- [O harness de QA](#o-harness-de-qa) -- [Testes de qualidade e ganhos medidos](#testes-de-qualidade-e-ganhos-medidos) -- [Estrutura do repositório](#estrutura-do-repositório) -- [Licença](#licença) - -## Workflows - -| Workflow | Arquivo | Cobre | +| Nível | Quando | O que roda | |---|---|---| -| 🎨 **Frontend / UI** | [workflows/wf-frontend.md](workflows/wf-frontend.md) | Direção de design antes do código, anti-AI-slop, performance React, QA visual de PDFs | -| ⚙️ **Backend / API** | [workflows/wf-backend.md](workflows/wf-backend.md) | Contrato antes do handler, defaults seguros, auditoria OWASP, testes + caça de bugs adversarial | -| 🏗️ **Arquitetura** | [workflows/wf-architecture.md](workflows/wf-architecture.md) | Requisitos mensuráveis, diagramas C4, ADRs, threat modeling, revisão | -| 🛡️ **Security review** | [workflows/wf-security-review.md](workflows/wf-security-review.md) | Code review (front + back), CVEs conhecidas **e** consulta ao vivo de novas | -| 📄 **README** | [workflows/wf-readme.md](workflows/wf-readme.md) | Criar/auditar/reescrever READMEs grounded nos fatos reais do repo | +| 0, trivial | nenhum comportamento muda | a mudança + gate nível 0 | +| 1, pequeno | 1 a 3 arquivos, sem superfície pública nova | `wf-bugfix` ou `wf-refactor`, teste de regressão, gate nível 1 | +| 2, feature | endpoint ou tela nova, contrato alterado, dependência nova | exploração, plano curto, TDD, revisão independente, gate nível 2 | +| 3, grande ou de risco | vários módulos, dados existentes, arquitetura, release | spec EARS + ADR aprovados por você, revisões, auditoria de segurança, gate nível 3 | -Cada arquivo é um `SKILL.md` padrão. A `description` no frontmatter é o gatilho: quando o contexto do agente bate, o workflow carrega sozinho. Funciona no [Hermes Agent](https://hermes-agent.nousresearch.com), Claude Code e qualquer coisa que leia o formato Agent Skills. +Gatilhos de segurança sobem o nível a qualquer momento: auth, dinheiro, dados pessoais, upload, SQL +montado à mão, shell/exec, desserialização, migração de schema, dependência nova, feature que chama LLM. -## Segurança: não confie no cutoff de treino +## O que tem dentro -O conhecimento de CVEs de um modelo para na data do treinamento. Qualquer coisa divulgada depois disso, ele nunca viu, então "não tem vulnerabilidade" vindo de memória é um chute, não resposta. +| Tipo | Itens | +|---|---| +| Roteador e regras | `dev-router`, `lean-code` | +| Workflows | `wf-bugfix`, `wf-refactor`, `wf-frontend`, `wf-backend`, `wf-architecture`, `wf-security-review`, `wf-readme` | +| Agentes | `explorer` (haiku, só leitura), `planner` (opus, só leitura), `implementer` (sonnet), `reviewer` (opus, só leitura, vê só o diff), `security-auditor` (opus, só leitura), `ui-critic` (sonnet, só leitura), `verifier` (haiku, roda o gate) | +| Hooks | `Stop`: bloqueia o "pronto" enquanto houver código alterado sem gate atual aprovado · `PreToolUse`: nega `--no-verify` e force-push, pergunta antes de `reset --hard`, `clean -f`, `DROP TABLE` · `PostToolUse`: formata o arquivo editado com o formatter do próprio projeto | +| Gate de qualidade | [`scripts/quality-gate`](scripts/quality-gate): lint, typecheck, testes, gitleaks (nível 0); cobertura do diff, semgrep, código morto (nível 1); build, teste de mutação, osv-scanner, regras de arquitetura, duplicação (nível 2); trivy, rastreabilidade requisito→teste (nível 3) | +| Skills vendorizadas | pilar de segurança (OWASP secure-agent-playbook, UnitOneAI SecuritySkills, safedeps, hermaguard) e taste-skill para frontend. Origem e licença: [`skills/NOTICE.md`](skills/NOTICE.md) | -O workflow de segurança nasceu desse fato: +O gate roda só as checagens do nível pedido, só sobre o diff, e não usa IA: são ferramentas comuns. +Checagens que precisam do app de pé (orçamento do Lighthouse CI, Playwright + axe, schemathesis, k6) +entram por projeto no `.dev-workflows.toml`. Referência: +[`skills/dev-router/references/quality-gate.md`](skills/dev-router/references/quality-gate.md). -``` -Código/diff → SCOPE → HUNT (CVEs conhecidas) → CONSULT (fontes vivas) → CODE REVIEW → REPORT + GATE - │ │ - │ └─ OSV.dev · GitHub Advisory DB · NVD - └──────────────┘ -``` - -Todo security review **precisa** consultar fontes vivas e carimbar a data. Review sem fonte e sem data vence na hora. - -## Skills usadas e seus donos - -Os workflows carregam essas skills por contexto. Os três pilares do security review vêm de fontes primárias de segurança (OWASP, OSV.dev, UnitOne SecuritySkills), não da memória de um modelo. - -### 🛡️ Security review: os três pilares - -_Vendorizadas em `skills/` — ver [`skills/NOTICE.md`](skills/NOTICE.md) pra origem exata e licença._ - -| Skill | O que faz | Dono | -|---|---|---| -| [`safedeps`](https://github.com/Jeneidi/safedeps) | Consulta o OSV.dev em tempo real; devolve CVE + severidade + versão corrigida pra um `package@version`. Cobre o buraco que um modelo congelado não cobre. | [Jeneidi](https://github.com/Jeneidi) | -| [`sca-audit`](https://github.com/OWASP/secure-agent-playbook) | Auditoria de supply chain de dependências com análise de alcance e mapeamento CWE. | [OWASP](https://github.com/OWASP) | -| [`code-review-security`](https://github.com/OWASP/secure-agent-playbook) | Code review de segurança sistemático mapeado a OWASP Top 10 + ASVS. | [OWASP](https://github.com/OWASP) | - -### 🛡️ Security review: skills de apoio - -_Também vendorizadas em `skills/`._ - -| Skill | O que faz | Dono | -|---|---|---| -| [`secrets-scan`](https://github.com/OWASP/secure-agent-playbook) | Encontra credenciais / API keys no código e no git history. | [OWASP](https://github.com/OWASP) | -| [`api-security-review`](https://github.com/OWASP/secure-agent-playbook) | Revisão de API contra OWASP API Security Top 10. | [OWASP](https://github.com/OWASP) | -| [`web-security-review`](https://github.com/OWASP/secure-agent-playbook) | Revisão de web app contra OWASP Top 10. | [OWASP](https://github.com/OWASP) | -| [`cve-triage`](https://github.com/UnitOneAI/SecuritySkills) | Prioriza CVEs com CVSS 4.0 / EPSS / CISA KEV. | [UnitOneAI](https://github.com/UnitOneAI) | -| [`patch-prioritization`](https://github.com/UnitOneAI/SecuritySkills) | Decide a ordem da correção. | [UnitOneAI](https://github.com/UnitOneAI) | -| [`dependency-scanning`](https://github.com/UnitOneAI/SecuritySkills) | Varredura da árvore de dependências. | [UnitOneAI](https://github.com/UnitOneAI) | -| [`hermaguard`](https://github.com/Sahil-SS9/hermaguard) | Review adversarial: pré-scan estático + 3 agentes especialistas (edge cases, ataque, blast radius). | [Sahil-SS9](https://github.com/Sahil-SS9) | - -### 🎨 Frontend / UI - -| Skill | O que faz | Dono | -|---|---|---| -| [`frontend-design`](https://github.com/anthropics/skills) | Direção visual intencional, tipografia, fugindo do padrão "IA genérica". | [Anthropic](https://github.com/anthropics) | -| [`web-design`](https://github.com/KAOPU-XiaoPu/web-design) | Estética web coesa. | [KAOPU-XiaoPu](https://github.com/KAOPU-XiaoPu) | -| [`vercel-react-best-practices`](https://github.com/vercel-labs/agent-skills) | 40+ regras de performance React/Next da engenharia da Vercel. | [Vercel Labs](https://github.com/vercel-labs) | -| [`vercel-composition-patterns`](https://github.com/vercel-labs/agent-skills) | Compound components, composição limpa. | [Vercel Labs](https://github.com/vercel-labs) | -| [`animate`](https://github.com/emilkowalski/skill) | Motion com propósito. | [emilkowalski](https://github.com/emilkowalski) | -| [`impeccable`](https://github.com/pbakaus/impeccable) | O vocabulário de design que faltava pros agentes: 23 comandos (craft, shape, audit, polish, animate, live) e 59 regras determinísticas anti-slop. A skill de design frontend mais usada (230k+ instalações). | [pbakaus](https://github.com/pbakaus) | -| [`design-taste-frontend`](https://github.com/Leonxlnx/taste-skill) | Gera UI nova com direção inferida do brief; três controles de 1 a 10 (variância, motion, densidade). Vendorizada em `skills/`. | [Leonxlnx](https://github.com/Leonxlnx) | -| [`redesign-existing-projects`](https://github.com/Leonxlnx/taste-skill) | Audita uma UI existente e melhora sem quebrar o comportamento. Vendorizada em `skills/`. | [Leonxlnx](https://github.com/Leonxlnx) | -| [`anti-ai-slop`](https://github.com/evandrodevbr/dev-workflows) | Detecta padrão visual "gerado por IA" (auto-carregada, local). | skill da comunidade | -| [`avoid-ai-writing`](https://github.com/conorbronsdon/avoid-ai-writing) | Remove AI-isms de microcopy, labels e docs. | [conorbronsdon](https://github.com/conorbronsdon) | - -### ⚙️ Backend / API - -| Skill | O que faz | Dono | -|---|---|---| -| [`secure-coding`](https://github.com/securityreviewai/secure-coding-skill) | Padrões de codificação segura em 15 stacks. | [securityreviewai](https://github.com/securityreviewai) | -| [`bola-detector`](https://github.com/apisec-inc/apisec-skills) | Object-level authorization quebrada (OWASP API1). | [apisec-inc](https://github.com/apisec-inc) | -| [`auth-rbac-scaffold`](https://github.com/apisec-inc/apisec-skills) | Autenticação + RBAC (OWASP API2 / API5). | [apisec-inc](https://github.com/apisec-inc) | -| [`injection-checker`](https://github.com/apisec-inc/apisec-skills) | Injeção SQL / ORM / shell / template (OWASP API8). | [apisec-inc](https://github.com/apisec-inc) | -| [`openapi-hardener`](https://github.com/apisec-inc/apisec-skills) | Sanear schemas OpenAPI / Zod / Pydantic (OWASP API3). | [apisec-inc](https://github.com/apisec-inc) | -| [`api-security-review`](https://github.com/apisec-inc/apisec-skills) | Review completo OWASP API Top 10. | [apisec-inc](https://github.com/apisec-inc) | -| [`security-test-generator`](https://github.com/apisec-inc/apisec-skills) | Gera suítes de testes de segurança. | [apisec-inc](https://github.com/apisec-inc) | - -> ⚠️ `api-security-review` são duas skills diferentes com o mesmo nome: a da OWASP vendorizada em `skills/` (pilar de segurança, acima) e a da apisec-inc nesta tabela. Só a cópia da OWASP vem neste repositório. Instale a que o seu workflow precisa -- com as duas presentes, agentes que resolvem por nome de skill acusam colisão. - -### 🏗️ Arquitetura - -| Skill | O que faz | Dono | -|---|---|---| -| [`system-design`](https://github.com/Kotivskyi/architecture-governance-skills) | Framework HelloInterview de system design. | [Kotivskyi](https://github.com/Kotivskyi) | -| [`c4-architecture`](https://github.com/Kotivskyi/architecture-governance-skills) | Diagramas C4 (Mermaid / Structurizr). | [Kotivskyi](https://github.com/Kotivskyi) | -| [`isaqb-architecture-governance`](https://github.com/Kotivskyi/architecture-governance-skills) | arc42 + ADRs. | [Kotivskyi](https://github.com/Kotivskyi) | -| [`secure-architecture-governance`](https://github.com/Kotivskyi/architecture-governance-skills) | Threat models STRIDE+CIA, ADRs de segurança. | [Kotivskyi](https://github.com/Kotivskyi) | - -> 📖 Instruções completas de instalação de cada skill estão em [docs/SKILLS.md](docs/SKILLS.md). - -## Começo rápido +## Instalação ```bash -# 1. Copie os workflows pro diretório de skills do seu agente. -# Cada skill precisa do seu próprio diretório com um SKILL.md dentro -- um -# .md solto numa pasta de categoria não é indexado (nem o Hermes nem o -# Claude Code carregam), por isso o loop abaixo monta a estrutura certa. -SKILLS_DIR="$HOME/.hermes/skills/software-development" -# Claude Code: SKILLS_DIR="$HOME/.claude/skills" +git clone https://github.com/evandrodevbr/dev-workflows && cd dev-workflows -for f in workflows/*.md; do - nome=$(basename "$f" .md) - mkdir -p "$SKILLS_DIR/$nome" - cp "$f" "$SKILLS_DIR/$nome/SKILL.md" -done - -# 2. (Opcional, pro security review) as skills de apoio já vêm vendorizadas -# em skills/ -- sem clone externo, só copiar: -cp -r skills/* "$HOME/.hermes/skills/" - -# 3. Use -# "review this PR for security" -> wf-security-review carrega -# "build a new page" -> wf-frontend carrega +python3 scripts/install.py claude # plugin: skills + agentes + hooks +python3 scripts/install.py opencode # skills, agentes e plugin em ~/.config/opencode +python3 scripts/install.py hermes # só skills, em ~/.hermes/skills ``` -Pra conferir se a instalação pegou: os workflows devem aparecer na lista de -skills do agente (`hermes skills list` no Hermes Agent). +Claude Code sem clonar: -## O harness de QA +```text +/plugin marketplace add evandrodevbr/dev-workflows +/plugin install dev-workflows@dev-workflows +``` -O diretório `qa/` mantém os próprios workflows honestos: +Diferenças entre os agentes: + +- **opencode** não consegue bloquear o fim do turno, então gate ausente ou falho vira aviso no log em vez de bloqueio. Os agentes viram subagentes do opencode com as mesmas permissões. +- **Hermes** recebe só as skills; rode o gate você mesmo. +- No opencode e no Hermes as cópias apontam para o seu clone: mantenha-o no lugar (ou rode a instalação de novo depois de mover). + +Desligue o hook de `Stop` numa sessão com `DW_GATE=off`. + +## Skills externas (opcionais) + +Os workflows usam estas quando instaladas e seguem as próprias regras quando não. Comandos de +instalação: [`docs/SKILLS.md`](docs/SKILLS.md). + +| Área | Skills | +|---|---| +| Frontend | `frontend-design` (Anthropic), `impeccable` (pbakaus), `vercel-react-best-practices`, `vercel-composition-patterns` (Vercel Labs), `animate` (emilkowalski), `web-design`, `avoid-ai-writing` | +| Backend | `secure-coding`, `bola-detector`, `auth-rbac-scaffold`, `injection-checker`, `openapi-hardener`, `security-test-generator` (apisec-inc) | +| Arquitetura | `system-design`, `c4-architecture`, `isaqb-architecture-governance`, `secure-architecture-governance` (Kotivskyi) | +| README | `readme-crafter`, `good-readme`, `curating-readme` | + +## Qualidade deste repositório ```bash -cd qa - -python3 wf_quality_harness.py # pontua cada workflow em 30+ critérios -python3 test_quality.py # gates: sem regressão, skills válidas, fases verificadas -python3 test_tests.py # meta-testes: corrompe o harness, prova que ele detecta a corrupção +python3 qa/lint_skills.py # frontmatter, limites de tamanho, referências, links, caminhos de máquina +python3 qa/test_gate.py # gate e hooks num repositório descartável +python3 qa/check_upstream.py # skills vendorizadas atrás do upstream +claude plugin validate --strict .claude-plugin/plugin.json ``` -Os meta-testes são a parte legal. Eles quebram o harness e os workflows de propósito e confirmam que o placar *cai*. Se o placar não mudasse, o detetor seria inútil. Esses testes ficam verdes como contrato. +O CI roda os dois primeiros em todo PR; a checagem de upstream roda todo mês. -## Testes de qualidade e ganhos medidos +**Ainda não medido:** se o kit melhora o código que um agente produz. Estudos públicos +([SkillsBench](https://arxiv.org/abs/2602.12670), [SWE-Skills-Bench](https://arxiv.org/abs/2603.15401)) +mostram que skills também podem piorar o resultado, então o próximo passo é um benchmark A/B (com e +sem o kit) no [Harbor](https://github.com/harbor-framework/harbor), com testes ocultos, score de +mutação e checagens de segurança. -Números reais das execuções do harness, guardados em `qa/snapshots/baseline-final.json`. A coluna "Antes" é o estado anterior ao refinamento; "Depois" é o resultado após ele. - -### Score de qualidade dos workflows (mesma régua, antes → depois) - -| Workflow | Antes | Depois | Ganho | -|---|---|---|---| -| wf-frontend | 201.0 | 385.5 | +92% | -| wf-backend | 215.0 | 424.0 | +97% | -| wf-architecture | 199.9 | 413.7 | +107% | -| wf-security-review | 332.0 | 390.0 | +17% | -| **Total** | **615.9** | **1223.2** | **+98.6%** | - -Isso é quase exatamente o dobro da qualidade dos workflows, medido com a mesma régua. - -A própria régua também ficou mais exigente ao longo do projeto (13 → 30+ critérios: verificação por fase, comandos reais, anti-padrões, critérios de aceite, artefatos, fluxo). Por isso as execuções antigas mostram totais menores: `base-364` (13 critérios) → `baseline-v2` (536.7) → `baseline-v3` (675.0) → `baseline-v4` (884.6) → `baseline-final` (1613.2 nos quatro workflows pontuados). Só o snapshot final está commitado, em `qa/snapshots/baseline-final.json` -- é ele que os gates leem e é ele que se atualiza quando a pontuação muda. A comparação justa acima usa a régua final dos dois lados. - -### Gates (checagens de regressão): todos verdes - -| Gate | O que verifica | Status | -|---|---|---| -| G1 | Score não caiu abaixo do baseline | ✅ | -| G4 | Toda skill referenciada existe | ✅ 4/4 workflows | -| G5 | Toda fase tem ≥3 verbos de verificação | ✅ 4/4 workflows | -| Meta-testes | Corrompe harness/workflows → score cai (prova que o detetor funciona) | ✅ | - -### O que o refinamento adicionou - -Cada workflow ganhou o mesmo lote de melhorias reais: -- **Passos de verificação** com comandos de verdade (`curl`, `grep`, `python3`, `git`, `npm`, `pytest`) em vez de prosa. -- **Checklists de aceite por fase** (portões `[ ]` marcáveis). -- **Blocos de anti-padrões** por fase (o que não fazer, e por quê). -- **Seções de comandos executáveis** e **exemplos trabalhados por fase**. -- **Checkpoints** que pausam pra aprovação do usuário entre fases. - -O workflow de segurança ganhou uma regra dura: *nunca afirmar "sem CVE" sem consultar OSV.dev / GitHub Advisory*. O conhecimento do modelo para no cutoff do treino; as fontes vivas não. - -### Contexto sobre a meta do "dobro" - -O harness tem um teto físico (soma dos caps dos critérios). A meta numérica de "2× do baseline" precisaria de uma régua com mais resolução pra ser expressável, então inflar o score pra bater seria maquiar a métrica. O que foi publicado é a versão honesta: qualidade real ~dobrou na mesma régua (+98.6%), com meta-testes provando que a régua detecta regressão. - - -## Estrutura do repositório +## Estrutura ``` -dev-workflows/ -├── README.md # versão em inglês -├── README.pt-BR.md # este arquivo -├── workflows/ # os cinco workflows em SKILL.md -│ ├── wf-frontend.md -│ ├── wf-backend.md -│ ├── wf-architecture.md -│ ├── wf-security-review.md -│ └── wf-readme.md -├── skills/ # o pilar de segurança, vendorizado (ver skills/NOTICE.md) -│ ├── safedeps/ # + arquivos de apoio plays/, templates/, schemas/, docs/ -│ ├── sca-audit/ -│ ├── code-review-security/ -│ ├── api-security-review/ -│ ├── web-security-review/ -│ ├── secrets-scan/ -│ ├── cve-triage/ -│ ├── patch-prioritization/ -│ ├── dependency-scanning/ -│ ├── hermaguard/ -│ ├── design-taste-frontend/ # taste-skill (frontend) -│ └── redesign-existing-projects/ # taste-skill (frontend) -├── docs/ -│ └── SKILLS.md # inventário completo de skills + instalação -├── CONTRIBUTING.md # o que melhorar, checks locais, processo de PR -└── qa/ # harness de qualidade, gates, meta-testes - ├── wf_quality_harness.py - ├── test_quality.py - ├── test_tests.py - ├── baseline.json - └── snapshots/ - └── baseline-final.json +.claude-plugin/ plugin.json, marketplace.json +skills/ dev-router, lean-code, workflows wf-*, skills vendorizadas (NOTICE.md) +agents/ explorer, planner, implementer, reviewer, security-auditor, ui-critic, verifier +hooks/hooks.json Stop, PreToolUse, PostToolUse +scripts/ quality-gate, install.py, hooks/*.py +adapters/opencode/ plugin do opencode +qa/ lint, testes do gate, checagem de upstream +docs/SKILLS.md inventário de skills e comandos de instalação ``` ## Licença -[MIT](LICENSE) © 2026 [Evandro Fonseca Junior](https://github.com/evandrodevbr) - ---- - -
- -⭐ Se isso te salvar de um "na minha máquina funciona" no merge, dá uma estrela. - -
+[MIT](LICENSE) © 2026 [Evandro Fonseca Junior](https://github.com/evandrodevbr). As skills vendorizadas mantêm as próprias licenças ([`skills/NOTICE.md`](skills/NOTICE.md)). diff --git a/docs/SKILLS.md b/docs/SKILLS.md index 3ad47f1..ca9d7df 100644 --- a/docs/SKILLS.md +++ b/docs/SKILLS.md @@ -1,115 +1,75 @@ -# Skills usadas e de onde vieram +# Skills: inventário e origem -Este é o inventário do que os workflows carregam. Cada skill é um diretório -com um `SKILL.md` dentro, e esse diretório é obrigatório nos dois agentes -(a skill é `/SKILL.md`, nunca um `.md` solto, que não é indexado): -no Hermes Agent, em `~/.hermes/skills//SKILL.md`; no Claude Code, em -`~/.claude/skills//SKILL.md`. +Cada skill é um diretório `skills//SKILL.md`. O plugin carrega todas; `scripts/install.py` +copia para o opencode ou o Hermes. -## Workflows orquestradores (em `workflows/`) +## Do próprio repositório -| Workflow | Para quê | +| Skill | Para quê | |---|---| -| `wf-frontend` | UI/design/PDF | -| `wf-backend` | API/backend seguro | -| `wf-architecture` | Arquitetura/ADR/C4 | -| `wf-security-review` | Code review de segurança front+back | -| `wf-readme` | Criar/auditar/reescrever READMEs grounded | +| `dev-router` | Classifica o pedido em nível 0 a 3 e decide workflows, agentes e gate | +| `lean-code` | Menor solução que resolve: reaproveitar, apagar, sem abstração nem dependência sem motivo | +| `wf-bugfix` | Reproduzir, causa raiz, teste de regressão que falha antes e passa depois | +| `wf-refactor` | Testes de caracterização, passos pequenos, comportamento igual | +| `wf-frontend` | UI e PDF: design (nível 2+), build acessível e responsivo, revisão visual | +| `wf-backend` | API: contrato primeiro (nível 2+), defaults seguros, auditoria OWASP, testes | +| `wf-architecture` | Nível 3: spec EARS, C4, ADR, STRIDE, revisão | +| `wf-security-review` | Revisão de segurança front e back com CVEs consultados ao vivo | +| `wf-readme` | README fundamentado em fatos do repositório, comandos executados antes da entrega | -## README (wf-readme) +## Vendorizadas (vêm no clone) + +Origem, commit e licença de cada uma: [`skills/NOTICE.md`](../skills/NOTICE.md). | Skill | Origem | O que faz | |---|---|---| -| `readme-crafter` | [linhai0872/readme-crafter-skill](https://github.com/linhai0872/readme-crafter-skill) | Classifica projeto (tipo/público/temperamento) e gera README sob medida; 13 checks | -| `good-readme` | [adewale/good-readme](https://github.com/adewale/good-readme) | Cria com exemplos reais OU audita contra 22 critérios (escala 100) | -| `curating-readme` | [liang-senbei/curating-readme](https://github.com/liang-senbei/curating-readme) | Padroniza README + docs (CONTRIBUTING/CHANGELOG) com `audit-repo.sh` | +| `safedeps` | [Jeneidi/safedeps](https://github.com/Jeneidi/safedeps) | CVE por `package@version` consultando o OSV.dev ao vivo | +| `sca-audit` | [OWASP/secure-agent-playbook](https://github.com/OWASP/secure-agent-playbook) | Auditoria de dependências com análise de alcance e CWE | +| `code-review-security` | OWASP/secure-agent-playbook | Code review de segurança mapeado a OWASP Top 10 e ASVS | +| `secrets-scan` | OWASP/secure-agent-playbook | Credenciais e chaves no código e no histórico do git | +| `api-security-review` | OWASP/secure-agent-playbook | API contra o OWASP API Security Top 10 | +| `web-security-review` | OWASP/secure-agent-playbook | Web app contra o OWASP Top 10 | +| `cve-triage` | [UnitOneAI/SecuritySkills](https://github.com/UnitOneAI/SecuritySkills) | Prioriza CVE por CVSS 4.0, EPSS e CISA KEV | +| `patch-prioritization` | UnitOneAI/SecuritySkills | Ordem de remediação | +| `dependency-scanning` | UnitOneAI/SecuritySkills | Varredura da árvore de dependências | +| `hermaguard` | [Sahil-SS9/hermaguard](https://github.com/Sahil-SS9/hermaguard) | Revisão adversarial: pre-scan e 3 agentes (borda, ataque, blast radius) | +| `design-taste-frontend` | [Leonxlnx/taste-skill](https://github.com/Leonxlnx/taste-skill) | Gera tela nova sem cara de template | +| `redesign-existing-projects` | Leonxlnx/taste-skill | Audita UI existente e melhora sem quebrar | -## Pilar de segurança (wf-security-review) +## Externas (opcionais) -**Vendorizadas neste repo** em `skills//` — ver `skills/NOTICE.md` pra -origem exata (commit + licença) de cada uma. Não precisa clonar nada externo -pra este pilar. - -| Skill | Origem | O que faz | -|---|---|---| -| `safedeps` | [Jeneidi/safedeps](https://github.com/Jeneidi/safedeps) | Consulta OSV.dev em tempo real; CVE por `package@version` com severidade e versão de fix. Inclui o script `check_deps.py` | -| `sca-audit` | [OWASP/secure-agent-playbook](https://github.com/OWASP/secure-agent-playbook) | SCA de dependências com análise de alcance (reachability), mapeamento CWE | -| `code-review-security` | OWASP/secure-agent-playbook | Code review de segurança sistemático mapeado a OWASP Top 10 + ASVS | -| `secrets-scan` | OWASP/secure-agent-playbook | Detecta credenciais/API keys no código e no git history | -| `api-security-review` | OWASP/secure-agent-playbook | Revisão de API contra OWASP API Security Top 10 | -| `web-security-review` | OWASP/secure-agent-playbook | Web app contra OWASP Top 10 (2021) | -| `cve-triage` | [UnitOneAI/SecuritySkills](https://github.com/UnitOneAI/SecuritySkills) | Prioriza CVE por CVSS 4.0 / EPSS / CISA KEV | -| `patch-prioritization` | UnitOneAI/SecuritySkills | Decide a ordem de remediação | -| `dependency-scanning` | UnitOneAI/SecuritySkills | Varredura de dependências | - -## Frontend / UI / design (wf-frontend) - -| Skill | Origem | O que faz | -|---|---|---| -| `frontend-design` | [anthropics/skills](https://github.com/anthropics/skills) | Direção visual, tipografia, fugir do genérico | -| `web-design` | [KAOPU-XiaoPu/web-design](https://github.com/KAOPU-XiaoPu/web-design) | Estética web coesa | -| `vercel-react-best-practices` | [vercel-labs/agent-skills](https://github.com/vercel-labs/agent-skills) | 40+ regras de perf React/Next | -| `vercel-composition-patterns` | vercel-labs/agent-skills | Compound components, composição | -| `animate` | [emilkowalski/skill](https://github.com/emilkowalski/skill) | Motion com propósito | -| `impeccable` | [pbakaus/impeccable](https://github.com/pbakaus/impeccable) | Vocabulário de design: 23 comandos + 59 regras determinísticas anti-slop | -| `design-taste-frontend` | [Leonxlnx/taste-skill](https://github.com/Leonxlnx/taste-skill) (vendorizada em `skills/`) | Gera UI nova sem cara de template; 3 controles de variância, motion e densidade | -| `redesign-existing-projects` | Leonxlnx/taste-skill (vendorizada em `skills/`) | Audita UI existente e melhora sem quebrar funcionalidade | -| `anti-ai-slop` | local | Detecta padrão visual "gerado por IA" | -| `avoid-ai-writing` | [conorbronsdon/avoid-ai-writing](https://github.com/conorbronsdon/avoid-ai-writing) | Remove AI-isms de texto/microcopy | - -## Backend / API (wf-backend) - -| Skill | Origem | O que faz | -|---|---|---| -| `secure-coding` | [v6x/secure-coding-skill](https://github.com/securityreviewai/secure-coding-skill) | Padrões de codificação segura em 15 stacks | -| `bola-detector` | [apisec-inc/apisec-skills](https://github.com/apisec-inc/apisec-skills) | Detecta object-level authorization (OWASP API1) | -| `auth-rbac-scaffold` | apisec-inc/apisec-skills | Auth/RBAC (OWASP API2/API5) | -| `injection-checker` | apisec-inc/apisec-skills | Injection SQL/ORM/shell (OWASP API8) | -| `openapi-hardener` | apisec-inc/apisec-skills | Saneia schemas OpenAPI/Zod (OWASP API3) | -| `api-security-review` | apisec-inc/apisec-skills | Review completo OWASP API Top 10 | -| `security-test-generator` | apisec-inc/apisec-skills | Gera testes de segurança | - -## Arquitetura (wf-architecture) - -| Skill | Origem | O que faz | -|---|---|---| -| `system-design` | [Kotivskyi/architecture-governance-skills](https://github.com/Kotivskyi/architecture-governance-skills) | Framework HelloInterview de system design | -| `c4-architecture` | Kotivskyi/architecture-governance-skills | Diagramas C4 (Mermaid/Structurizr) | -| `isaqb-architecture-governance` | Kotivskyi/architecture-governance-skills | arc42 + ADR | -| `secure-architecture-governance` | Kotivskyi/architecture-governance-skills | Threat model STRIDE+CIA, S-ADR | - -## Caça de bugs (todas) - -| Skill | Origem | O que faz | -|---|---|---| -| `hermaguard` | [Sahil-SS9/hermaguard](https://github.com/Sahil-SS9/hermaguard) | Review adversarial: pre-scan + 3 agentes (edge cases, ataque, blast radius) | - -## Como instalar as skills de origem - -**Pilar de segurança:** já vem vendorizado em `skills/` neste repo — sem -clone externo. Copie direto: +Os workflows usam estas quando instaladas e seguem as próprias regras quando não. Instalação no +formato Agent Skills: ```bash -# Hermes Agent: -cp -r skills/* ~/.hermes/skills/ - -# Claude Code (cada skill no seu próprio diretório): -mkdir -p "$HOME/.claude/skills" -for d in skills/*/; do - cp -r "$d" "$HOME/.claude/skills/" -done +npx skills add --skill ``` -Os arquivos de apoio que cada skill referencia (`plays/`, `templates/`, -`schemas/`, `docs/`) ficam dentro do diretório da própria skill, então o -`cp -r` os leva junto. Origem e licença de cada um: `skills/NOTICE.md`. +| Skill | Origem | Usada por | +|---|---|---| +| `frontend-design` | [anthropics/skills](https://github.com/anthropics/skills) | `wf-frontend` | +| `impeccable` | [pbakaus/impeccable](https://github.com/pbakaus/impeccable) | `wf-frontend`, `ui-critic` | +| `vercel-react-best-practices` | [vercel-labs/agent-skills](https://github.com/vercel-labs/agent-skills) | `wf-frontend` | +| `vercel-composition-patterns` | vercel-labs/agent-skills | `wf-frontend` | +| `animate` | [emilkowalski/skill](https://github.com/emilkowalski/skill) | `wf-frontend` | +| `web-design` | [KAOPU-XiaoPu/web-design](https://github.com/KAOPU-XiaoPu/web-design) | `wf-frontend` | +| `avoid-ai-writing` | [conorbronsdon/avoid-ai-writing](https://github.com/conorbronsdon/avoid-ai-writing) | `wf-frontend`, `wf-readme` | +| `secure-coding` | [securityreviewai/secure-coding-skill](https://github.com/securityreviewai/secure-coding-skill) | `wf-backend` | +| `bola-detector` | [apisec-inc/apisec-skills](https://github.com/apisec-inc/apisec-skills) | `wf-backend`, `wf-security-review` | +| `auth-rbac-scaffold` | apisec-inc/apisec-skills | `wf-backend`, `wf-security-review` | +| `injection-checker` | apisec-inc/apisec-skills | `wf-backend`, `wf-security-review` | +| `openapi-hardener` | apisec-inc/apisec-skills | `wf-backend` | +| `security-test-generator` | apisec-inc/apisec-skills | `wf-backend` | +| `system-design` | [Kotivskyi/architecture-governance-skills](https://github.com/Kotivskyi/architecture-governance-skills) | `wf-architecture` | +| `c4-architecture` | Kotivskyi/architecture-governance-skills | `wf-architecture` | +| `isaqb-architecture-governance` | Kotivskyi/architecture-governance-skills | `wf-architecture` | +| `secure-architecture-governance` | Kotivskyi/architecture-governance-skills | `wf-architecture` | +| `readme-crafter` | [linhai0872/readme-crafter-skill](https://github.com/linhai0872/readme-crafter-skill) | `wf-readme` | +| `good-readme` | [adewale/good-readme](https://github.com/adewale/good-readme) | `wf-readme` | +| `curating-readme` | [liang-senbei/curating-readme](https://github.com/liang-senbei/curating-readme) | `wf-readme` | -**Frontend/backend/arquitetura** (ainda não vendorizadas, instalam de fora): -a maioria via `npx skills` (formato agentskills.io): +A apisec-inc também publica uma skill chamada `api-security-review`. Ela não é usada aqui: o nome +colide com a versão OWASP vendorizada. -```bash -npx skills add anthropics/skills --skill frontend-design -npx skills add vercel-labs/agent-skills --skill vercel-react-best-practices -npx skills add conorbronsdon/avoid-ai-writing -npx skills add https://github.com/pbakaus/impeccable --skill impeccable -``` +O `impeccable` não é vendorizado porque traz scripts executáveis de terceiros; instale-o você mesmo +(`npx skills add https://github.com/pbakaus/impeccable --skill impeccable`) se quiser o `/impeccable audit`. diff --git a/qa/baseline.json b/qa/baseline.json deleted file mode 100644 index 9258375..0000000 --- a/qa/baseline.json +++ /dev/null @@ -1,236 +0,0 @@ -{ - "workflows": [ - { - "name": "wf-frontend", - "score": 385.5, - "skills_referenciadas": [ - "animate", - "anti-ai-slop", - "avoid-ai-writing", - "claude-design", - "design-md", - "frontend-award-tier", - "frontend-design", - "hermaguard", - "html-css-print-engineer", - "pdf-generator", - "pdf-render-qa", - "pdf-visual-designer", - "popular-web-designs", - "professional-pdf-director", - "sketch", - "vercel-composition-patterns", - "vercel-react-best-practices", - "web-design" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 6, - "regras_duras": 10, - "n_regras": 9, - "fase1": 16, - "fase2": 16, - "fase3": 16, - "cobertura_fases": 20, - "skills_validas": 15, - "fase_rotulos": 6, - "fase_acao": 6, - "fase_revisao": 6, - "anti_slop": 4.5, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 8, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 20, - "profundidade": 10, - "por_fase_verif": 18, - "por_fase_cmd": 18, - "por_fase_antip": 12, - "por_fase_aceite": 12, - "exemplos": 10, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - }, - { - "name": "wf-backend", - "score": 424.0, - "skills_referenciadas": [ - "api-security-review", - "auth-rbac-scaffold", - "bola-detector", - "dependency-cve-audit", - "fastapi-uvicorn-wiring", - "hermaguard", - "injection-checker", - "network-audit", - "openapi-hardener", - "rest-graphql-debug", - "secure-coding", - "security-best-practices", - "security-test-generator", - "software-code-quality", - "test-driven-development" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 8, - "regras_duras": 10, - "n_regras": 10, - "fase1": 15, - "fase2": 15, - "fase3": 16, - "fase4": 16, - "cobertura_fases": 20, - "skills_validas": 15, - "fase_rotulos": 6, - "fase_acao": 6, - "fase_revisao": 6, - "anti_slop": 5.0, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 8, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 20, - "profundidade": 13, - "por_fase_verif": 22, - "por_fase_cmd": 24, - "por_fase_antip": 16, - "por_fase_aceite": 16, - "exemplos": 10, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - }, - { - "name": "wf-architecture", - "score": 413.7, - "skills_referenciadas": [ - "architecture-diagram", - "avoid-ai-writing", - "brainstorming", - "c4-architecture", - "excalidraw", - "hermaguard", - "isaqb-architecture-governance", - "secure-architecture-governance", - "security-threat-model", - "system-design", - "writing-plans" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 8, - "regras_duras": 10, - "n_regras": 10, - "fase1": 15, - "fase2": 16, - "fase3": 14, - "fase4": 16, - "cobertura_fases": 20, - "skills_validas": 11, - "fase_rotulos": 6, - "fase_acao": 0, - "fase_revisao": 6, - "anti_slop": 4.7, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 8, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 20, - "profundidade": 13, - "por_fase_verif": 24, - "por_fase_cmd": 24, - "por_fase_antip": 16, - "por_fase_aceite": 15, - "exemplos": 10, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - }, - { - "name": "wf-security-review", - "score": 390.0, - "skills_referenciadas": [ - "api-security-review", - "auth-rbac-scaffold", - "bola-detector", - "code-review-security", - "cve-triage", - "dependency-cve-audit", - "hermaguard", - "injection-checker", - "patch-prioritization", - "safedeps", - "sca-audit", - "secrets-scan", - "security-threat-model", - "web-security-review" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 10, - "regras_duras": 10, - "n_regras": 8, - "fase1": 15, - "fase2": 16, - "fase3": 13, - "fase4": 15, - "fase5": 15, - "cobertura_fases": 20, - "skills_validas": 14, - "fase_rotulos": 6, - "fase_acao": 0, - "fase_revisao": 6, - "anti_slop": 5.0, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 0, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 16, - "profundidade": 15, - "por_fase_verif": 23, - "por_fase_cmd": 19, - "por_fase_antip": 12, - "por_fase_aceite": 5, - "exemplos": 0, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - } - ], - "total": 1613.2, - "max": 400 -} diff --git a/qa/check_upstream.py b/qa/check_upstream.py new file mode 100644 index 0000000..d6a2ac0 --- /dev/null +++ b/qa/check_upstream.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Compares each vendored skill's recorded commit (skills/NOTICE.md) with its upstream HEAD. Exit 1 if any moved. + +python3 qa/check_upstream.py +""" +import os, re, subprocess, sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def main(): + notice = open(os.path.join(ROOT, "skills", "NOTICE.md"), encoding="utf-8").read() + repos = {} + for row in re.findall(r"^\|\s*`[\w-]+`\s*\|(.+)$", notice, re.M): + cols = [c.strip() for c in row.split("|")] + repo = re.search(r"([\w.-]+/[\w.-]+)", cols[0]) + sha = re.search(r"`([0-9a-f]{7,40})`", cols[1]) + if repo and sha: + repos[repo.group(1)] = sha.group(1) + moved = 0 + for repo, sha in sorted(repos.items()): + p = subprocess.run(["git", "ls-remote", f"https://github.com/{repo}", "HEAD"], capture_output=True, text=True, timeout=60) + head = p.stdout.split()[0] if p.returncode == 0 and p.stdout else None + if head is None: + print(f"? {repo}: não consegui consultar") + elif head.startswith(sha): + print(f"ok {repo} {sha}") + else: + moved += 1 + print(f"-> {repo}: vendorizado {sha}, upstream {head[:7]} (https://github.com/{repo}/compare/{sha}...{head[:7]})") + return 1 if moved else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/qa/lint_skills.py b/qa/lint_skills.py new file mode 100644 index 0000000..f7f329a --- /dev/null +++ b/qa/lint_skills.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Structural lint for the plugin: frontmatter, sizes, references, links and leftover machine-specific paths. + +Run from anywhere: python3 qa/lint_skills.py (exit 1 on any error) +""" +import json, os, re, sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +MAX_LINES = 150 +MAX_DESC = 1024 +FORBIDDEN = [r"~/\.hermes", r"vision_analyze", r"search_files", r"test_quality\.py", r"wf_quality_harness"] +MODELS = {"haiku", "sonnet", "opus", "inherit"} +TOOLS = {"Read", "Edit", "Write", "Bash", "Grep", "Glob", "WebFetch", "WebSearch", "NotebookEdit", "Agent", "Skill"} +errors = [] + + +def err(path, msg): + errors.append(f"{os.path.relpath(path, ROOT)}: {msg}") + + +def frontmatter(path): + text = open(path, encoding="utf-8").read() + m = re.match(r"---\n(.*?)\n---\n", text, re.S) + if not m: + err(path, "sem frontmatter") + return {}, text + fm = m.group(1) + meta = dict(re.findall(r"^([\w-]+):[ \t]*(.*)$", fm, re.M)) + for key in ("uses", "external", "skills"): + lst = re.search(rf"^\s*{key}:\s*\[(.*?)\]", fm, re.M) + if lst: + meta[key] = [s.strip().strip("'\"") for s in lst.group(1).split(",") if s.strip()] + return meta, text + + +def vendored(): + rows = open(os.path.join(ROOT, "skills", "NOTICE.md"), encoding="utf-8").read() + return set(re.findall(r"^\|\s*`([\w-]+)`\s*\|", rows, re.M)) + + +def check_links(path, text): + text = re.sub(r"```.*?```|`[^`\n]*`", "", text, flags=re.S) + for target in re.findall(r"\]\(([^)\s]+)\)", text): + if re.match(r"(https?:|mailto:|#)", target): + continue + p = os.path.normpath(os.path.join(os.path.dirname(path), target.split("#")[0])) + if not os.path.exists(p): + err(path, f"link quebrado: {target}") + + +def main(): + skills_dir = os.path.join(ROOT, "skills") + names = {d for d in os.listdir(skills_dir) if os.path.isfile(os.path.join(skills_dir, d, "SKILL.md"))} + third_party = vendored() + catalog = open(os.path.join(ROOT, "docs", "SKILLS.md"), encoding="utf-8").read() + + for name in sorted(names): + path = os.path.join(skills_dir, name, "SKILL.md") + meta, text = frontmatter(path) + if meta.get("name") != name: + err(path, f"name '{meta.get('name')}' difere da pasta '{name}'") + desc = meta.get("description", "") + if not desc: + err(path, "description vazia") + if name in third_party: + continue + if len(desc) > MAX_DESC: + err(path, f"description com {len(desc)} caracteres (máx {MAX_DESC})") + lines = text.count("\n") + if lines > MAX_LINES: + err(path, f"{lines} linhas (máx {MAX_LINES}); mova detalhes para references/") + for used in meta.get("uses", []): + if used not in names: + err(path, f"uses: '{used}' não existe em skills/") + for ext in meta.get("external", []): + if f"`{ext}`" not in catalog: + err(path, f"external: '{ext}' não está em docs/SKILLS.md") + for dirpath, _, files in os.walk(os.path.join(skills_dir, name)): + for f in files: + if f.endswith(".md"): + p = os.path.join(dirpath, f) + body = open(p, encoding="utf-8").read() + check_links(p, body) + for pat in FORBIDDEN: + if re.search(pat, body): + err(p, f"referência específica de máquina: {pat}") + + agents_dir = os.path.join(ROOT, "agents") + for f in sorted(os.listdir(agents_dir)): + path = os.path.join(agents_dir, f) + meta, text = frontmatter(path) + if meta.get("name") != f[:-3]: + err(path, f"name '{meta.get('name')}' difere do arquivo") + if meta.get("model") not in MODELS: + err(path, f"model '{meta.get('model')}' inválido") + bad = {t.strip() for t in meta.get("tools", "").split(",")} - TOOLS + if bad: + err(path, f"tools desconhecidas: {', '.join(sorted(bad))}") + for s in meta.get("skills", []): + if s not in names: + err(path, f"skills: '{s}' não existe em skills/") + for pat in FORBIDDEN: + if re.search(pat, text): + err(path, f"referência específica de máquina: {pat}") + + hooks = json.load(open(os.path.join(ROOT, "hooks", "hooks.json"))) + for groups in hooks["hooks"].values(): + for g in groups: + for h in g["hooks"]: + for script in re.findall(r"\$\{CLAUDE_PLUGIN_ROOT\}/([\w/.-]+)", h["command"]): + if not os.path.isfile(os.path.join(ROOT, script)): + err(os.path.join(ROOT, "hooks", "hooks.json"), f"script inexistente: {script}") + + for doc in ("README.md", "README.pt-BR.md", "CONTRIBUTING.md", os.path.join("docs", "SKILLS.md")): + p = os.path.join(ROOT, doc) + if os.path.exists(p): + check_links(p, open(p, encoding="utf-8").read()) + + for e in errors: + print("ERRO " + e) + print(f"{len(names)} skills, {len(os.listdir(agents_dir))} agentes, {len(errors)} erro(s)") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/qa/snapshots/baseline-final.json b/qa/snapshots/baseline-final.json deleted file mode 100644 index 9258375..0000000 --- a/qa/snapshots/baseline-final.json +++ /dev/null @@ -1,236 +0,0 @@ -{ - "workflows": [ - { - "name": "wf-frontend", - "score": 385.5, - "skills_referenciadas": [ - "animate", - "anti-ai-slop", - "avoid-ai-writing", - "claude-design", - "design-md", - "frontend-award-tier", - "frontend-design", - "hermaguard", - "html-css-print-engineer", - "pdf-generator", - "pdf-render-qa", - "pdf-visual-designer", - "popular-web-designs", - "professional-pdf-director", - "sketch", - "vercel-composition-patterns", - "vercel-react-best-practices", - "web-design" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 6, - "regras_duras": 10, - "n_regras": 9, - "fase1": 16, - "fase2": 16, - "fase3": 16, - "cobertura_fases": 20, - "skills_validas": 15, - "fase_rotulos": 6, - "fase_acao": 6, - "fase_revisao": 6, - "anti_slop": 4.5, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 8, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 20, - "profundidade": 10, - "por_fase_verif": 18, - "por_fase_cmd": 18, - "por_fase_antip": 12, - "por_fase_aceite": 12, - "exemplos": 10, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - }, - { - "name": "wf-backend", - "score": 424.0, - "skills_referenciadas": [ - "api-security-review", - "auth-rbac-scaffold", - "bola-detector", - "dependency-cve-audit", - "fastapi-uvicorn-wiring", - "hermaguard", - "injection-checker", - "network-audit", - "openapi-hardener", - "rest-graphql-debug", - "secure-coding", - "security-best-practices", - "security-test-generator", - "software-code-quality", - "test-driven-development" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 8, - "regras_duras": 10, - "n_regras": 10, - "fase1": 15, - "fase2": 15, - "fase3": 16, - "fase4": 16, - "cobertura_fases": 20, - "skills_validas": 15, - "fase_rotulos": 6, - "fase_acao": 6, - "fase_revisao": 6, - "anti_slop": 5.0, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 8, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 20, - "profundidade": 13, - "por_fase_verif": 22, - "por_fase_cmd": 24, - "por_fase_antip": 16, - "por_fase_aceite": 16, - "exemplos": 10, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - }, - { - "name": "wf-architecture", - "score": 413.7, - "skills_referenciadas": [ - "architecture-diagram", - "avoid-ai-writing", - "brainstorming", - "c4-architecture", - "excalidraw", - "hermaguard", - "isaqb-architecture-governance", - "secure-architecture-governance", - "security-threat-model", - "system-design", - "writing-plans" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 8, - "regras_duras": 10, - "n_regras": 10, - "fase1": 15, - "fase2": 16, - "fase3": 14, - "fase4": 16, - "cobertura_fases": 20, - "skills_validas": 11, - "fase_rotulos": 6, - "fase_acao": 0, - "fase_revisao": 6, - "anti_slop": 4.7, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 8, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 20, - "profundidade": 13, - "por_fase_verif": 24, - "por_fase_cmd": 24, - "por_fase_antip": 16, - "por_fase_aceite": 15, - "exemplos": 10, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - }, - { - "name": "wf-security-review", - "score": 390.0, - "skills_referenciadas": [ - "api-security-review", - "auth-rbac-scaffold", - "bola-detector", - "code-review-security", - "cve-triage", - "dependency-cve-audit", - "hermaguard", - "injection-checker", - "patch-prioritization", - "safedeps", - "sca-audit", - "secrets-scan", - "security-threat-model", - "web-security-review" - ], - "criteria": { - "frontmatter": 5, - "when_to_use": 8, - "num_fases": 10, - "regras_duras": 10, - "n_regras": 8, - "fase1": 15, - "fase2": 16, - "fase3": 13, - "fase4": 15, - "fase5": 15, - "cobertura_fases": 20, - "skills_validas": 14, - "fase_rotulos": 6, - "fase_acao": 0, - "fase_revisao": 6, - "anti_slop": 5.0, - "cmd_blocos": 8, - "cmd_reais": 5, - "sec_criterios": 0, - "sec_verificar": 6, - "sec_nao_regressao": 6, - "tamanho": 8, - "testes_referidos": 5, - "passos_total": 30, - "verificacao_total": 24, - "comandos_reais": 16, - "profundidade": 15, - "por_fase_verif": 23, - "por_fase_cmd": 19, - "por_fase_antip": 12, - "por_fase_aceite": 5, - "exemplos": 0, - "exemplos_count": 12, - "artefatos": 14, - "fluxo": 8, - "entradas_saidas": 8 - } - } - ], - "total": 1613.2, - "max": 400 -} diff --git a/qa/test_gate.py b/qa/test_gate.py new file mode 100644 index 0000000..0511469 --- /dev/null +++ b/qa/test_gate.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Smoke tests for scripts/quality-gate and the hook scripts, on a throwaway git repo. Stdlib only. + +python3 qa/test_gate.py +""" +import importlib.util, json, os, subprocess, sys, tempfile, unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +GATE = os.path.join(ROOT, "scripts", "quality-gate") +STOP = os.path.join(ROOT, "scripts", "hooks", "stop_gate.py") +spec = importlib.util.spec_from_file_location("bash_guard", os.path.join(ROOT, "scripts", "hooks", "bash_guard.py")) +bash_guard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bash_guard) + + +def run(cmd, cwd, stdin=None): + return subprocess.run(cmd, cwd=cwd, input=stdin, capture_output=True, text=True) + + +class GateTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.repo = self.tmp.name + for c in (["git", "init", "-q"], ["git", "config", "user.email", "t@t"], ["git", "config", "user.name", "t"]): + run(c, self.repo) + self.write("calc.py", "def add(a, b):\n return a + b\n") + self.write("check.py", "from calc import add\nassert add(1, 2) == 3\n") + self.write(".dev-workflows.toml", '[commands]\ntest = "python3 check.py"\nlint = "true"\ntypecheck = "true"\n') + self.write("package.json", '{"dependencies": {"left-pad": "1"}}') + run(["git", "add", "-A"], self.repo) + run(["git", "commit", "-qm", "init"], self.repo) + + def tearDown(self): + self.tmp.cleanup() + + def write(self, name, text): + open(os.path.join(self.repo, name), "w").write(text) + + def gate(self, tier=0): + p = run([sys.executable, GATE, "--tier", str(tier)], self.repo) + return p.returncode, json.load(open(os.path.join(self.repo, ".dev-workflows", "gate.json"))) + + def stop(self): + out = run([sys.executable, STOP], self.repo, json.dumps({"cwd": self.repo})).stdout + return json.loads(out)["reason"] if out.strip() else None + + def status(self, report, name): + return next(c["status"] for c in report["checks"] if c["name"] == name) + + def test_clean_tree_does_not_block(self): + self.assertIsNone(self.stop()) + + def test_change_without_gate_blocks_then_passing_gate_allows(self): + self.write("calc.py", "def add(a, b):\n return a + b\n\n\ndef sub(a, b):\n return a - b\n") + self.assertIn("sem gate", self.stop()) + code, report = self.gate() + self.assertEqual(code, 0) + self.assertEqual(self.status(report, "test"), "pass") + self.assertIsNone(self.stop()) + + def test_failing_test_fails_gate_and_blocks(self): + self.write("calc.py", "def add(a, b):\n return a - b\n") + code, report = self.gate() + self.assertEqual((code, report["verdict"]), (1, "fail")) + self.assertIn("test", self.stop()) + + def test_missing_tool_is_unverified_not_passed(self): + self.write(".dev-workflows.toml", '[commands]\ntest = "definitely-not-a-tool-xyz"\n') + self.write("calc.py", "def add(a, b):\n return b + a\n") + code, report = self.gate() + self.assertEqual(self.status(report, "test"), "unverified") + self.assertEqual(report["verdict"], "incomplete") + + def test_new_dependency_is_reported(self): + self.write("package.json", '{"dependencies": {"left-pad": "1", "is-odd": "3"}}') + self.write("calc.py", "def add(a, b):\n return a + b # touched\n") + _, report = self.gate() + self.assertEqual(report["new_dependencies"], ["npm:is-odd"]) + + def test_stale_gate_blocks(self): + self.write("calc.py", "def add(a, b):\n return a + b # v2\n") + self.gate() + os.utime(os.path.join(self.repo, ".dev-workflows", "gate.json"), (1, 1)) + self.assertIn("mudou depois", self.stop()) + + +class GuardTest(unittest.TestCase): + def test_decisions(self): + cases = {"git commit --no-verify -m x": "deny", "git push -f origin main": "deny", + "git push --force-with-lease": None, "rm -rf /": "deny", "rm -rf node_modules": None, + "git reset --hard": "ask", "npm test": None} + for cmd, expected in cases.items(): + self.assertEqual(bash_guard.decide(cmd)[0], expected, cmd) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/qa/test_quality.py b/qa/test_quality.py deleted file mode 100644 index 6f22681..0000000 --- a/qa/test_quality.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -""" -test_quality.py — testes validados + aprovados (gate de qualidade não-regressivo). - -Roda o harness, compara com o baseline snapshot e aplica gates: - -Gate 1 (NÃO REGRESSÃO): score total atual >= baseline (1613.2). Se caiu, exit 1. -Gate 2 (MELHORIA ≥2×): score total atual >= 2 * baseline (3226.4). Meta do - projeto, só exigida com --strict (o harness tem teto físico, ver README). -Gate 3 (POR WORKFLOW ≥2×): cada um dos 4 workflows >= 2 * seu baseline - individual. Mesma família do G2: meta aspiracional, só com --strict — - o snapshot guarda os scores FINAIS, então 2× ele mesmo é inalcançável - por construção. -Gate 4 (SKILLS): nenhuma skill referenciada faltando (harness sem sem_match). -Gate 5 (VERIFICAÇÃO POR FASE): toda fase tem >=3 verbos de verificação. - -Uso: python3 test_quality.py -> exit 0 se G1,G4,G5 ok (evidência) - python3 test_quality.py --strict -> também exige G2 e G3 (meta final) -""" -import json, os, subprocess, sys - -WF_QA = os.path.dirname(os.path.abspath(__file__)) -HARNESS = os.path.join(WF_QA, "wf_quality_harness.py") -SNAPSHOT = os.path.join(WF_QA, "snapshots", "baseline-final.json") - -def run_harness(): - r = subprocess.run([sys.executable, HARNESS, "--json"], capture_output=True, text=True) - if r.returncode != 0: - return None, r.stderr - return json.loads(r.stdout), None - -def main(): - strict = "--strict" in sys.argv - base = json.load(open(SNAPSHOT)) - base_total = base["total"] - base_per = {w["name"]: w.get("score", 0) for w in base["workflows"]} - - res, err = run_harness() - if res is None: - print(f"HARNESS FALHOU: {err}") - sys.exit(1) - cur_total = res["total"] - cur_per = {w["name"]: w.get("score", 0) for w in res["workflows"]} - sem_match = {w["name"]: w.get("skills_sem_match", []) for w in res["workflows"]} - fases_vazias = {w["name"]: w.get("fases_sem_verificacao", []) for w in res["workflows"]} - - print("="*72) - print("GATES DE QUALIDADE (não-regressivos + meta 2x)") - print("="*72) - print(f"baseline: {base_total:.1f} | atual: {cur_total:.1f} | meta(2x): {2*base_total:.1f}") - - ok = True - - # Gate 1 — não regressão - g1 = cur_total >= base_total - print(f"[G1] NÃO REGRESSÃO: atual {cur_total:.1f} >= baseline {base_total:.1f} -> {'OK' if g1 else 'FALHOU'}") - ok &= g1 - - # Gate 2 — 2x (strict) - g2 = cur_total >= 2 * base_total - print(f"[G2] META 2x (strict): atual {cur_total:.1f} >= {2*base_total:.1f} -> {'OK' if g2 else 'ainda não (meta final)'}") - if strict: - ok &= g2 - - # Gate 3 — por workflow, 2x individual (mesma família do G2: só em --strict) - print("[G3] POR WORKFLOW (2x individual, só com --strict):") - all_g3_ok = True - for name, b in base_per.items(): - cur = cur_per.get(name, 0) - g3i = cur >= 2 * max(b, 0.1) - print(f" {name:18} base={b:6.1f} atual={cur:6.1f} 2x={2*max(b,0.1):6.1f} -> {'OK' if g3i else 'ainda não (meta final)'}") - all_g3_ok &= g3i - if strict: - ok &= all_g3_ok - - # Gate 4 — skills - print("[G4] SKILLS REFERENCIADAS:") - all_skills_ok = True - for name, miss in sem_match.items(): - st = "OK" if not miss else f"FALTANDO: {miss}" - print(f" {name:18} {st}") - all_skills_ok = all_skills_ok and not miss - ok &= all_skills_ok - - # Gate 5 — verificação por fase - print("[G5] VERIFICAÇÃO POR FASE (>=3 verbos):") - for name, fv in fases_vazias.items(): - st = "OK (todas as fases com verificação)" if not fv else f"sem verificação nas fases {fv}" - print(f" {name:18} {st}") - ok &= not fv - - print("="*72) - if ok: - print(f"RESULTADO: APROVADO (gates G1,G4,G5 passaram{' + G2/G3' if strict else ''}; score {cur_total:.1f})") - sys.exit(0) - else: - print(f"RESULTADO: REPROVADO — gates abaixo falharam. Score {cur_total:.1f}. Rodar refinamento de novo.") - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/qa/test_tests.py b/qa/test_tests.py deleted file mode 100644 index 2a0ebcc..0000000 --- a/qa/test_tests.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -""" -test_tests.py — 'teste dos testes' (meta-testes) do wf_quality_harness. - -Prova que o harness REALMENTE mede qualidade: - 1. Baseline registrado em snapshots/baseline-final.json (score 1613.2). - 2. Muta o harness (zera o anti_slop) e confirma que o score CAI -> o - harness não é indiferente ao input. - 3. Muta um workflow (tira 'Regras duras') e confirma que o score CAI -> - o harness responde a perda de conteúdo. Se o workflow não for - encontrado, isso é FALHA, não skip: meta-teste que não roda é falso - verde. - 4. Re-verifica o estado real: score atual >= baseline (não-regressão). - -Exit 0 = testes dos testes OK e não-regressão verificada. -Exit 1 = falhou (meta-teste ou regressão real). -""" -import json, os, shutil, subprocess, sys, tempfile, importlib.util - -WF_QA = os.path.dirname(os.path.abspath(__file__)) -HARNESS = os.path.join(WF_QA, "wf_quality_harness.py") -SNAPSHOT = os.path.join(WF_QA, "snapshots", "baseline-final.json") - -def load_harness(): - """Importa o harness pra reusar a mesma resolução de caminho dos workflows.""" - spec = importlib.util.spec_from_file_location("wfq_harness", HARNESS) - if spec is None or spec.loader is None: - raise RuntimeError(f"não consegui carregar o harness em {HARNESS}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - -def run_harness(): - r = subprocess.run([sys.executable, HARNESS, "--json"], capture_output=True, text=True) - if r.returncode != 0: - return None, r.stderr - return json.loads(r.stdout), None - -def main(): - fails = [] - baseline = json.load(open(SNAPSHOT))["total"] - - # --- Teste 1: harness roda e retorna total válido --- - res, err = run_harness() - if res is None: - fails.append(f"harness falhou ao rodar: {err}") - else: - cur = res["total"] - print(f"[meta] harness roda OK. score atual={cur:.1f} baseline={baseline:.1f}") - if not (baseline > 0 and cur > 0): - fails.append("scores inválidos (<=0)") - if cur < baseline: - fails.append(f"REGRESSÃO REAL: atual {cur:.1f} < baseline {baseline:.1f}") - - # Sem harness não há o que comparar: os testes 2 e 3 dependem do score atual. - if res is None: - print("\nFALHAS META-TESTE:") - for f in fails: - print(" -", f) - sys.exit(1) - - # --- Teste 2: mutação do harness muda o resultado (sensibilidade) --- - tmp = tempfile.mkdtemp(prefix="wfqamut_") - try: - mutated = os.path.join(tmp, "wf_quality_harness_mut.py") - src = open(HARNESS, encoding="utf-8").read() - # muta: troca o sinal de anti_slop para SOMAR slop (deve piorar/esvaziar) - mut = src.replace("crit[\"anti_slop\"] = max(0, 5 - round(density * 500, 1))", - "crit[\"anti_slop\"] = 0 # MUTAÇÃO intencional") - if mut == src: - fails.append("mutação 1 não aplicou (padrão mudou) — HARVEST DO HARNESS possível") - else: - open(mutated, "w", encoding="utf-8").write(mut) - r = subprocess.run([sys.executable, mutated, "--json"], capture_output=True, text=True) - try: - j = json.loads(r.stdout) - if j["total"] >= res["total"]: - fails.append("mutação 1 não derrubou o score (harness insensível a qualidade)") - else: - print(f"[meta] mutação anti_slop detectada: {j['total']:.1f} < {res['total']:.1f} OK") - except Exception: - fails.append("mutação 1: saída não-JSON após remoção do anti_slop") - finally: - shutil.rmtree(tmp, ignore_errors=True) - - # --- Teste 3: workflow mutado (remove Regras duras) derruba score --- - # Resolve o mesmo caminho que o harness usa (clone do repo ou instalado). - # Um meta-teste que não roda é falso verde: se o arquivo não for achado, - # isso é FALHA, não skip. - try: - h = load_harness() - wk = h.resolve_workflow("wf-frontend", h.WORKFLOWS["wf-frontend"]) - except Exception as e: - h, wk = None, None - fails.append(f"não consegui resolver o workflow wf-frontend: {e}") - if h is not None and wk is None: - fails.append("workflow wf-frontend não encontrado (nem workflows/wf-frontend.md no repo nem instalado) — meta-teste 3 não pôde rodar") - if wk is not None: - orig = open(wk, encoding="utf-8").read() - if "## Regras duras" not in orig: - fails.append(f"'{wk}' não tem '## Regras duras' — meta-teste 3 não consegue mutar") - else: - ctx = tempfile.mkdtemp(prefix="wfqamutwf_") - backup = os.path.join(ctx, "backup.md") - shutil.copy(wk, backup) - try: - removed = orig.replace("## Regras duras", "## Regras_duras_mutadas") - open(wk, "w", encoding="utf-8").write(removed) - r2, _ = run_harness() - if r2 is None or r2["total"] >= res["total"]: - fails.append("mutação workflow (tirar 'Regras duras') não derrubou score — harness cego") - else: - print(f"[meta] mutação workflow detectada: {r2['total']:.1f} < {res['total']:.1f} OK") - finally: - shutil.copy(backup, wk) - shutil.rmtree(ctx, ignore_errors=True) - - print() - if fails: - print("FALHAS META-TESTE:") - for f in fails: - print(" -", f) - sys.exit(1) - print("META-TESTES OK: harness sensível, sem regressão real. Score atual >= baseline.") - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/qa/wf_quality_harness.py b/qa/wf_quality_harness.py deleted file mode 100644 index 3e99a0f..0000000 --- a/qa/wf_quality_harness.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python3 -""" -wf_quality_harness.py — QA objetivo dos workflows wf-frontend/wf-backend/wf-architecture. - -Mede qualidade por critérios EXTRAÍVEIS do SKILL.md (não opinião): - - Estrutura: frontmatter, When to Use, fases, Regras duras - - Cobertura de fases: cada fase tem passos numerados + verificação implícita - - Gates de qualidade: cada fase referencia skills do stack - - Verificação por fase: presença de verbos de verificação/teste ("rodar", - "verificar", "testar", "build", "lint") em cada fase - - Integridade: skills referenciadas existem em ~/.hermes/skills/ - - Anti-slop: baixa densidade de AI-isms na skill - -Saída JSON. Exit 1 se alguma skill referenciada não existir (gate). -Uso: python3 wf_quality_harness.py [--json] -""" -import json, os, re, sys - -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SKILLS_DIR = os.path.expanduser("~/.hermes/skills") -VENDORED_SKILLS_DIR = os.path.join(REPO_ROOT, "skills") - -WORKFLOWS = { - "wf-frontend": "software-development/wf-frontend/SKILL.md", - "wf-backend": "software-development/wf-backend/SKILL.md", - "wf-architecture": "software-development/wf-architecture/SKILL.md", - "wf-security-review": "software-development/wf-security-review/SKILL.md", -} - -# Teto real do harness por workflow (não é 100): soma dos caps dos critérios -# estáticos, mais 20 por fase (por_fase_verif/cmd/antip/aceite) e 16 por fase -# (faseN = 8 passos + 6 verbos + 2 de verificação real). -STATIC_CAP = 288 -PER_FASE_CAP = 36 - -def resolve_workflow(name, relpath): - """Resolve o SKILL.md do workflow: clone do repo primeiro, instalado depois. - - O harness é documentado pra rodar a partir do clone (`cd qa`), onde os - workflows estão em `workflows/.md`. Cópias instaladas ficam em - ~/.hermes/skills/software-development//SKILL.md. - """ - for cand in (os.path.join(REPO_ROOT, "workflows", name + ".md"), - os.path.join(SKILLS_DIR, relpath), - os.path.join(SKILLS_DIR, name, "SKILL.md")): - if os.path.exists(cand): - return cand - return None - -VERB_VERIFY = ["rodar", "verificar", "verifica", "testar", "build", "lint", - "typecheck", "check", "audit", "validate", "validar", "coverage", - "pytest", "npm", "pnpm", "yarn", "go test", "jest"] -VERB_DESIGN = ["definir", "decidir", "direção", "paleta", "tipografia", "token", - "aprovar", "aprovação", "validar com o usuário"] -AI_SLOP_WORDS = ["robust", "seamless", "leverage", "delve", "game-changer", - "cutting-edge", "streamlined", "holistic", "synergy", - "unlock", "elevate", "empower", "foster", "robustamente"] - -def checks_exist(path): - """Existe um dos caminhos possíveis da skill (vendorizada no repo ou instalada).""" - if os.path.exists(os.path.join(VENDORED_SKILLS_DIR, path, "SKILL.md")): - return True - if os.path.exists(os.path.join(SKILLS_DIR, path, "SKILL.md")): - return True - # busca recursiva rasa (1-2 níveis) por nome - for base in (VENDORED_SKILLS_DIR, SKILLS_DIR): - for root, dirs, files in os.walk(base): - if root.count(os.sep) - base.count(os.sep) > 2: - continue - if any(f == "SKILL.md" for f in files): - dname = os.path.basename(root) - if dname == path: - return True - return False - -def find_skill_path(name): - for base in (VENDORED_SKILLS_DIR, SKILLS_DIR): - p = os.path.join(base, name, "SKILL.md") - if os.path.exists(p): - return p - for root, dirs, files in os.walk(base): - if root.count(os.sep) - base.count(os.sep) > 2: - continue - if "SKILL.md" in files and os.path.basename(root) == name: - return os.path.join(root, "SKILL.md") - return None - -def score_workflow(name, relpath): - path = resolve_workflow(name, relpath) - if path is None: - return {"name": name, - "error": f"SKILL.md não encontrado: nem {os.path.join(REPO_ROOT, 'workflows', name + '.md')} nem {os.path.join(SKILLS_DIR, relpath)}", - "score": 0, "max": 0} - text = open(path, encoding="utf-8").read() - lower = text.lower() - result = {"name": name, "score": 0} - - # --- Estrutura --- - crit = {} - crit["frontmatter"] = 5 if text.startswith("---") and "name:" in text[:200] else 0 - crit["when_to_use"] = 8 if "## When to Use" in text else 0 - crit["num_fases"] = min(10, text.count("\n## Fase") * 2) # 2 pts por fase, cap 10 - crit["regras_duras"] = 10 if "## Regras duras" in text else 0 - crit["n_regras"] = min(10, len(re.findall(r"^\- NUNCA|^\- NUNCA |^-\s*\*\*", text, re.M))) - - # --- Cobertura de fases --- - fases = re.findall(r"##\s*Fase (\d+)[^\n]*\n(.*?)(?=\n##|\Z)", text, re.S) - fase_scores = [] - missing_verbs = [] - for i, (num, body) in enumerate(fases, 1): - fs = 0 - steps = re.findall(r"^\s?\d+\.", body, re.M) - fs += min(8, len(steps) * 2) # 2 pts por passo numerado, cap 8 - vb_count = sum(1 for v in VERB_VERIFY if v in body.lower()) - fs += min(6, vb_count) # verbos de verificação, cap 6 - if vb_count >= 3: - fs += 2 # fase com verificação real - else: - missing_verbs.append(num) - fase_scores.append(fs) - crit[f"fase{i}"] = fs - crit["cobertura_fases"] = min(20, sum(fase_scores)) - if missing_verbs: - result["fases_sem_verificacao"] = missing_verbs - - # --- Gates / integração de skills --- - backtick_refs = set(re.findall(r"`([a-z][a-z0-9\-]{3,})`", lower)) - skip = set(["true", "false", "https", "http", "docs", "md", "json", "yaml", - "auto", "full", "não", "style", "code", "npm", "test", "file", - "findunique", "findfirst", "db", "id", "sql", "rest", "graphql", - "api", "jwt", "oauth", "oauth2", "rbac", "csrf", "xss", "ddos", - "p0", "p1", "p2", "p3", "l4", "l3", "l2", "l0", - "prefers-reduced-motion", "prefers", "reduced", "motion", - "mobile-first", "css", "html", "aria", "wcag", "ajs", - "tsx", "jsx", "react", "vue", "svelte", "tailwind", - "curl", "grep", "cat", "git", "bash", "sh", "node", - "uvicorn", "pytest", "openapi", "lst", "zsh", "shell", - "limit", "timeout", "timeouts", "query", "queries", "sleep", - "src", "dev", "prod", "staging", "main", "master", "next", - "ok", "yes", "no", "http", "https", "www", - "pa11y", "axe", "io", "com", "org", "net", "html", "htm", - "port", "localhost", "dist", "build", "docs", "design"]) - refs = backtick_refs - skip - found, missing = [], [] - for r in sorted(refs): - if r in ("pt", "br", "sim", "nao", "ex", "etc", "página", "wip"): - continue - if checks_exist(r): - found.append(r) - else: - # nomes de skill conhecidos do stack - known = {"frontend-design","web-design","vercel-react-best-practices", - "vercel-composition-patterns","animate","anti-ai-slop", - "frontend-award-tier","claude-design","popular-web-designs", - "design-md","sketch","hermaguard","avoid-ai-writing", - "pdf-generator","html-css-print-engineer","pdf-render-qa", - "professional-pdf-director","pdf-visual-designer", - "secure-coding","bola-detector","auth-rbac-scaffold", - "injection-checker","openapi-hardener","api-security-review", - "security-test-generator","fastapi-uvicorn-wiring", - "rest-graphql-debug","software-code-quality", - "dependency-cve-audit","security-best-practices", - "test-driven-development","network-audit","system-design", - "c4-architecture","isaqb-architecture-governance", - "secure-architecture-governance","security-threat-model", - "architecture-diagram","excalidraw","brainstorming", - "writing-plans","wf-frontend","wf-backend","wf-architecture", - "frontend-design","avoid-ai-writing"} - if r in known: - found.append(r) - else: - missing.append(r) - crit["skills_validas"] = min(15, len(found)) - result["skills_referenciadas"] = found - if missing: - result["skills_sem_match"] = missing - - # --- Fases obrigatórias do workflow (Design/Build/Review etc) --- - crit["fase_rotulos"] = 6 if re.search(r"DESIGN|PLAN|REQUIREMENTS", text, re.I) else 0 - crit["fase_acao"] = 6 if re.search(r"BUILD|CODAR|IMPLEMENTAR|implementar", text, re.I) else 0 - crit["fase_revisao"] = 6 if re.search(r"REVIEW|TEST|AUDIT|verificar antes", text, re.I) else 0 - - # --- Qualidade: densidade de AI-isms (menos = melhor) --- - words = re.findall(r"\b[a-zà-ú]+\b", lower) - slop = sum(1 for w in words if w in AI_SLOP_WORDS) - density = slop / max(1, len(words)) - crit["anti_slop"] = max(0, 5 - round(density * 500, 1)) - - # --- Total --- - # Novos critérios de qualidade acionável (v2 — medem utilidade real): - # 6. Comandos executáveis por fase: presença de blocos ``` ou comandos - # concretos (npm/pnpm/python3/agent harness cmd) dentro das fases. - cmd_blks = text.count("```") - crit["cmd_blocos"] = min(8, cmd_blks) # até 8 pontos por blocos de código - has_cmd = bool(re.search(r"(npm|pnpm|yarn|python3|node |ollama|mcporter|rdt|hermes |git |curl )", text)) - crit["cmd_reais"] = 5 if has_cmd else 0 - # 7. Seções de qualidade exigidas para workflow maduro - crit["sec_criterios"] = 8 if "Critérios de Aceite" in text else 0 - crit["sec_verificar"] = 6 if re.search(r"VERIFICAR|verificar antes|gate final", text, re.I) else 0 - crit["sec_nao_regressao"] = 6 if re.search(r"test_quality|não-regress|nao-regress|checklist não", text, re.I) else 0 - # 8. Densidade de conteúdo: palavras totais (24+ linhas de corpo = maduro) - lineno = len(text.splitlines()) - crit["tamanho"] = min(8, lineno // 12) - # 9. Referência a testes: menciona test_quality.py ou hermaguard-verify - crit["testes_referidos"] = 5 if re.search(r"test_quality\.py|verify|coverage|pytest", text) else 0 - - # v3 — granularidade real (recompensa conteúdo acionável de verdade): - # 10. Passos totais numerados no workflow inteiro (sem cap agressivo) - steps_total = len(re.findall(r"^\s{0,4}\d+\.", text, re.M)) - crit["passos_total"] = min(30, steps_total) - # 11. Verificação explícita por fase/checkpoint (cap alto) - verif_total = len(re.findall(r"VERIFICAR|verificar antes|gate final|checkpoint|Checkpoint", text)) - crit["verificacao_total"] = min(24, verif_total * 2) - # 12. Comandos úteis concretos (não só nomes de skill) - cmds_uteis = len(re.findall(r"(?:npm|pnpm|yarn|python3|node |ollama|mcporter|rdt|hermes|git |curl |pipx|uv )[\w\.\/\s\-\-]{4,}", text)) - crit["comandos_reais"] = min(20, cmds_uteis) - # 13. Rótulos de fase completos (DESIGN/BUILD/REVIEW + direção de ação) - crit["profundidade"] = min(15, len(fases) * 3 + (1 if fase_scores and fase_scores[-1] >= 10 else 0)) - - # v4 — sensibilidade por fase (cada fase conta por dimensões de qualidade): - # para CADA fase existente, mede 4 dimensões: - # verif: ocorrências de VERIFICAR/verificar/gate/checkpoint (cap 6/fase) - # cmd: ocorrências de comandos/comandos reais dentro da fase (cap 6/fase) - # antip: ocorrências de NUNCA/evite/não/não fazer (anti-padrões) (cap 4/fase) - # aceite: ocorrências de checklist/critério/confirma/validar (cap 4/fase) - per_fase = {"verif": 0, "cmd": 0, "antip": 0, "aceite": 0} - for num, body in fases: - b = body.lower() - per_fase["verif"] += min(6, len(re.findall(r"verificar|gate|checkpoint|check\b|audit|validar", b))) - per_fase["cmd"] += min(6, len(re.findall(r"`[a-z][\w\.\/\-]{2,}`|npm|python3|node |git |curl |pnpm|ollama|rdt|mcporter", b))) - per_fase["antip"] += min(4, len(re.findall(r"nunca|não |evit|anti|não fazer|nem ", b))) - per_fase["aceite"] += min(4, len(re.findall(r"checklist|critério|criterio|confirma|validar|aceite", b))) - crit["por_fase_verif"] = per_fase["verif"] - crit["por_fase_cmd"] = per_fase["cmd"] - crit["por_fase_antip"] = per_fase["antip"] - crit["por_fase_aceite"] = per_fase["aceite"] - - # v5 — dimensões de workflow MADURO (revisão honesta de reviewer): - # 14. Exemplos de execução: seção com exemplos/runbooks por fase. - crit["exemplos"] = 10 if re.search(r"Exemplos de execução|Exemplos|runbook|exemplo", text, re.I) else 0 - crit["exemplos_count"] = min(12, len(re.findall(r"(?:Exemplo|exemplo|Fase \d —|## Exemplos|```)", text))) - # 15. Artefatos/saída por fase: o que a fase PRODUZ (definível, verificável). - art = len(re.findall(r"(?:DESIGN\.md|contract\.md|requirements\.md|docs/architecture|c4\.md|ADR|OpenAPI|report|relatório|artefato|arquivo)", text, re.I)) - crit["artefatos"] = min(14, art) - # 16. Mapa do fluxo: descreve a sequência das fases (fluxo/ordem/resumo). - crit["fluxo"] = 8 if re.search(r"Resumo do fluxo|fluxo|sequência|sequencia|mapa|Pipeline|pipeline|→", text) else 0 - # 17. Entradas/saídas por fase (interface das fases entre si). - crit["entradas_saidas"] = 8 if re.search(r"checkpoint|entrada|saída|saida|handoff|interface.", text, re.I) else 0 - - total = sum(v for k, v in crit.items() if isinstance(v, (int, float))) - result.update({"criteria": crit, "score": round(total, 1), - "max": STATIC_CAP + PER_FASE_CAP * len(fases)}) - return result - -def main(): - out = {"workflows": [], "total": 0, "max": 0} - all_ok = True - for name, relpath in WORKFLOWS.items(): - res = score_workflow(name, relpath) - out["workflows"].append(res) - out["total"] += res.get("score", 0) - out["max"] += res.get("max", 0) - if res.get("skills_sem_match"): - all_ok = False - if "--json" in sys.argv: - print(json.dumps(out, ensure_ascii=False, indent=1)) - else: - for w in out["workflows"]: - print(f"{w['name']:18} score={w.get('score',0):>6} refs={len(w.get('skills_referenciadas',[]))} " - f"sem_match={w.get('skills_sem_match',[])} fases_sem_verif={w.get('fases_sem_verificacao',[])}") - pct = (out["total"] / out["max"] * 100) if out["max"] else 0.0 - print(f"\nTOTAL: {out['total']:.1f} / {out['max']:.1f} ({pct:.1f}% do teto)") - sys.exit(0 if all_ok else 1) - -if __name__ == "__main__": - main()