1{2 "skill_name": "consulting-adrs",3 "notes": "Structure + behavior eval set. Offline cases point the skill at the synthetic fixture ADR catalog at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr (treated as a local bitwarden/contributing-docs checkout). The source-call case provides no local checkout so the skill must reach contributing.bitwarden.com; its assertions are graded on output content, not the run's tool-call log. Prompts are intentionally blind: they state the change neutrally and never name the expected classification.",4 "fixtures_dir": "plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr",5 "evals": [6 {7 "id": 1,8 "name": "conflict-throws-instead-of-result",
10 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: OrganizationService methods will throw a NotFoundException when an entity is missing and throw a ValidationException on bad input, instead of returning a result object.",
11 "assertions": [
12 "Each finding is labeled with its type (conflict, gap, stale-reference, or aligned); how the label is rendered does not matter.",
13 "Output ends with a roll-up line giving counts per finding type or stating no in-force ADR was relevant.",
14 "Emits a CONFLICT finding.",
15 "Cites ADR-9001 (Result types for expected service errors) as the conflicting decision.",
16 "Does not classify this change as Aligned."
17 ]
18 },
19 {
20 "id": 2,
21 "name": "gap-new-event-bus",
22 "type": ["behavior"],
23 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: introduce a new cross-service event bus built on Redis pub/sub to broadcast domain events between the API and background workers.",
24 "assertions": [
25 "Emits a GAP finding for the event bus decision.",
26 "Explains why it is significant (new cross-service pattern / precedent / blast radius).",
27 "Does not invent or cite a specific ADR number, title, or URL for the event bus."
28 ]
29 },
30 {
31 "id": 3,
32 "name": "aligned-returns-result",
33 "type": ["behavior"],
34 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: a new BillingService returns a Result<T> carrying a typed error for expected validation failures, and reserves exceptions for unrecoverable states.",
35 "assertions": [
36 "Classifies the change as Aligned with ADR-9001 (or otherwise reports no conflict/gap).",
37 "Does not emit a CONFLICT or GAP finding for this change.",
38 "Keeps the aligned note brief (does not manufacture findings)."
39 ]
40 },
41 {
42 "id": 4,
43 "name": "stale-reference-superseded-adr",
44 "type": ["behavior"],
45 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: our updated testing guide tells engineers to write all new client unit tests in Vitest, citing ADR-9002.",
46 "assertions": [
47 "Emits a STALE-REFERENCE finding (or flags that ADR-9002 is superseded).",
48 "Points to ADR-9005 (Jest) as the in-force decision.",
49 "Does not treat ADR-9002 (Vitest) as a current, binding constraint."
50 ]
51 },
52 {
53 "id": 5,
54 "name": "skip-no-architectural-surface",
55 "type": ["behavior"],
56 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: fix a typo in a log message and correct an off-by-one error in a loop bound in the export routine.",
57 "assertions": [
58 "Reports no relevant in-force ADR / no architectural surface.",
59 "Does not manufacture a CONFLICT, GAP, or STALE-REFERENCE finding.",
60 "Does not invent an ADR."
61 ]
62 },
63 {
64 "id": 6,
65 "name": "no-adr-found-do-not-invent",
66 "type": ["behavior"],
67 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: adjust the sync client retry backoff constant from 200ms to 500ms.",
68 "assertions": [
69 "Reports that no in-force ADR governs the change (none found).",
70 "Does not invent an ADR number, title, or URL.",
78 "prompt": "Validate this proposed change against Bitwarden's ADRs. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr. Proposed change: add a single 400-line BillingManager class that handles billing, invoicing, tax calculation, and email receipts together.",
79 "assertions": [
80 "Does not classify this as an ADR CONFLICT (no fixture ADR governs class size or separation of concerns).",
81 "If mentioned at all, frames it as not ADR-governed rather than a recorded-decision conflict."
82 ]
83 },
84 {
85 "id": 8,
86 "name": "source-call-live-contributing-site",
87 "type": ["behavior", "source"],
88 "prompt": "Validate this proposed change against Bitwarden's ADRs. No local checkout of bitwarden/contributing-docs is available. Proposed change: adopt npm as the package manager for the client monorepo.",
89 "assertions": [
90 "Output references the real package-manager ADR (adopt-pnpm / ADR-0030).",
91 "Cites a contributing.bitwarden.com ADR path.",
92 "Does not fabricate an ADR: any cited ADR path resolves under contributing.bitwarden.com/architecture/adr/."
93 ]
94 },
95 {
96 "id": 9,
97 "name": "summarize-adr-catalog",
98 "type": ["behavior"],
99 "prompt": "Summarize the accepted ADRs for onboarding. A local checkout of the bitwarden/contributing-docs ADR catalog is at plugins/bitwarden-adr-tools/skills/consulting-adrs/evals/fixtures/adr.",
100 "assertions": [
101 "Returns a summary/list of ADRs (title + status), not a conflict/gap/stale finding report.",