Setting the file. One moment. - Number
- 7.108
- Position
- 108 of 155
- Type
- JavaScript
- Size
- 7 KB
- Lines
- 155
lib/grade-recommendation.mjs
JavaScript·155 lines·7 KB
[-*]\s
+|
\d
+
[.)]\s
+|
[*_]
+
)
?
(?:add
|
set
|
enable
|
disable
|
replace
|
remove
|
move
|
wrap
|
cache
|
defer
|
parallelize
|
introduce
|
configure
|
update
|
change
|
switch
|
opt
[-\s]
?
in
|
opt
[-\s]
?
out
|
export
|
import
|
install
|
run
|
delete
|
rename)/
im
;
6const COUNT_WORDS_RE = /\b(errors?|queries|invocations|requests|reads|writes|bytes|fetch(?:es)?|calls?|hits?|misses?|seconds?|images?|deployments?|cold[- ]?starts?|users?)\b/gi;
7const UNIT_RE = /\b\d[\d.,]*\s*(?:%|ms|s|sec|seconds?|min|minutes?|h|hours?|GB|MB|KB|K|M|B|rps|qps|req\/s|reqs?\/min)\b/gi;
8const CODE_FENCE_RE = /```[\s\S]*?```/g;
9const INLINE_CODE_RE = /`[^`\n]{10,}`/g;
10const FILE_LINE_RE = /[\w/.\-()\[\]]+\.\w+:\d+/g;
11
12// Grounding + evidence are lie-detectors — weighted higher than specificity/actionability, which LLMs can game with fluff.
13const W = { grounding: 0.35, evidence: 0.30, specificity: 0.20, actionability: 0.15 };
14
15export function gradeRecommendation(rec, ctx = {}) {
16 const accountScope = isAccountScope(rec);
17 const specificity = scoreSpecificity(rec);
18 const actionability = scoreActionability(rec);
19 const grounding = accountScope ? scoreGroundingAccount(rec) : scoreGrounding(rec, ctx);
20 const evidence = accountScope ? scoreEvidenceAccount(rec) : scoreEvidence(rec);
21 const overall = roundTo(
22 grounding * W.grounding + evidence * W.evidence + specificity * W.specificity + actionability * W.actionability,
23 4,
24 );
25 return {
26 specificity, actionability, grounding, evidence, overall,
27 grade: grade(overall),
28 scope: accountScope ? 'account' : 'route',
29 };
30}
31
32function isAccountScope(rec) {
33 if (rec?.scope === 'account') return true;
34 const ref = rec?.candidateRef;
35 if (typeof ref === 'string' && ref.startsWith('platform_')) return true;
36 return false;
37}
38
39function grade(overall) {
40 if (overall >= 0.85) return 'Excellent';
41 if (overall >= 0.70) return 'Good';
42 if (overall >= 0.55) return 'Fair';
43 return 'Poor';
44}
45
46function scoreSpecificity(rec) {
47 let s = 0;
48 const codeText = [rec.fix, rec.currentBehavior, rec.desiredBehavior].filter((x) => typeof x === 'string').join('\n');
49 const hasFence = CODE_FENCE_RE.test(codeText);
50 CODE_FENCE_RE.lastIndex = 0;
51 if (hasFence) s += 0.5;
52 if (INLINE_CODE_RE.test(codeText)) s += 0.2;
53 INLINE_CODE_RE.lastIndex = 0;
54 if (Array.isArray(rec.affectedFiles) && rec.affectedFiles.length > 0) s += 0.2;
55 if (Array.isArray(rec.findingRefs) && rec.findingRefs.some((r) => /:\d+/.test(r))) s += 0.3;
56 return Math.min(1, roundTo(s, 4));
57}
58
59function scoreActionability(rec) {
60 const text = typeof rec.fix === 'string' ? rec.fix : '';
61 if (!text) return 0;
62 let s = 0;
63 if (VERB_OPENERS.test(text)) s += 0.35;
64 const stepCount = (text.match(/(?:^|\n)\s*(?:\d+[.)]\s+|[-*]\s+)/g) ?? []).length;
65 if (stepCount >= 2) s += 0.35;
66 else if (stepCount === 1) s += 0.15;
67 const hedges = (text.match(HEDGE_WORDS) ?? []).length;
68 HEDGE_WORDS.lastIndex = 0;
69 s -= Math.min(0.3, hedges * 0.1);
70 // Baseline so a verb-only one-liner still scores.
71 s += 0.3;
72 return Math.max(0, Math.min(1, roundTo(s, 4)));
73}
74
75function scoreGrounding(rec, ctx) {
76 let s = 0;
77 const knownFindings = Array.isArray(ctx.knownFindings) ? ctx.knownFindings : [];
78 const findingKeys = new Set(knownFindings.map((f) => `${f.file}:${f.line}`));
79 const refs = Array.isArray(rec.findingRefs) ? rec.findingRefs : [];
80 const matched = refs.filter((r) => findingKeys.has(r));
81 if (matched.length > 0) s += 0.5;
82 else if (refs.length > 0) s += 0.25;
83 if (Array.isArray(rec.affectedFiles) && rec.affectedFiles.length > 0) s += 0.25;
84 const fenceText = [rec.currentBehavior, rec.desiredBehavior].filter((x) => typeof x === 'string').join('\n');
85 if (CODE_FENCE_RE.test(fenceText)) s += 0.25;
86 CODE_FENCE_RE.lastIndex = 0;
87 if (typeof rec.candidateRef === 'string' && rec.candidateRef.length > 0) s += 0.1;
88 return Math.min(1, roundTo(s, 4));
89}
90
91function scoreEvidence(rec) {
92 const text = [rec.what, rec.why, rec.fix, rec.verify]
93 .filter((x) => typeof x === 'string').join('\n');
94 if (!text) return 0;
95 const counts = (text.match(COUNT_WORDS_RE) ?? []).length;
96 COUNT_WORDS_RE.lastIndex = 0;
97 const units = (text.match(UNIT_RE) ?? []).length;
98 UNIT_RE.lastIndex = 0;
99 const filelines = (text.match(FILE_LINE_RE) ?? []).length;
100 FILE_LINE_RE.lastIndex = 0;
101 // file:line is the gold standard.
102 let s = Math.min(0.5, filelines * 0.2)
103 + Math.min(0.3, units * 0.075)
104 + Math.min(0.2, counts * 0.05);
105 return Math.min(1, roundTo(s, 4));
106}
107
108// No findingRefs/code fences possible — grade structural tie to gate + signal-quoting.
109function scoreGroundingAccount(rec) {
110 let s = 0;
111 if (typeof rec.candidateRef === 'string' && rec.candidateRef.startsWith('platform_')) s += 0.4;
112 else if (typeof rec.candidateRef === 'string' && rec.candidateRef.length > 0) s += 0.2;
113 // Quoting deep-dive data in why/fix is the account-scope equivalent of citing file:line.
114 const text = [rec.why, rec.fix, rec.verify].filter((x) => typeof x === 'string').join('\n');
115 const units = (text.match(UNIT_RE) ?? []).length;
116 UNIT_RE.lastIndex = 0;
117 if (units >= 3) s += 0.4;
118 else if (units >= 1) s += 0.2;
119 const citations = Array.isArray(rec.citations) ? rec.citations.length : 0;
120 if (citations >= 2) s += 0.2;
121 else if (citations >= 1) s += 0.1;
122 return Math.min(1, roundTo(s, 4));
123}
124
125// Heavily weighted toward magnitude quoting — vague platform recs should score low.
126function scoreEvidenceAccount(rec) {
127 const text = [rec.what, rec.why, rec.fix, rec.verify]
128 .filter((x) => typeof x === 'string').join('\n');
129 if (!text) return 0;
130 const counts = (text.match(COUNT_WORDS_RE) ?? []).length;
131 COUNT_WORDS_RE.lastIndex = 0;
132 const units = (text.match(UNIT_RE) ?? []).length;
133 UNIT_RE.lastIndex = 0;
134 // Higher weight than route-scope variant — file:line gold standard isn't available.
135 let s = Math.min(0.55, units * 0.15) + Math.min(0.35, counts * 0.08);
136 if (typeof rec.o11ySignal === 'string' && rec.o11ySignal.length > 0) s += 0.1;
137 return Math.min(1, roundTo(s, 4));
138}
139
140function roundTo(n, d) {
141 const f = 10 ** d;
142 return Math.round(n * f) / f;
143}
144
145// 0.55 = Poor/Fair boundary. Recommending Poor-graded items erodes trust faster than the marginal recall benefit.
146export function applyQualityFloor(recs, floor = 0.55) {
147 const kept = [];
148 const dropped = [];
149 for (const rec of recs) {
150 const o = rec?.quality?.overall ?? 0;
151 if (o < floor) dropped.push({ rec, reason: `quality.overall=${o} < floor=${floor}` });
152 else kept.push(rec);
153 }
154 return { kept, dropped };
155}