1import { canonicalizeJson, sha256 } from "./extraction-contract.mjs";
2import { validateGeneratedArtifact } from "./extraction-validators.generated.mjs";
3
4export const DETECTOR_ENSEMBLE_VERSION = "0083.1";
5export const HIGH_CONFIDENCE_MIN_PRECISION = 0.95;
6
7const KNOWN_KINDS = new Set([
8  "accordion",

extraction-detectors.mjs on GitHub

"banner"

,

"button"

,

"button-group"

,

"card"

,

"card-collection"

,

"category-strip", "cta-strip", "footer", "form", "gallery", "header", "hero",

"marquee", "menu", "navigation", "promo-band", "repeater", "reviews", "rich-text",

"stat-group", "static-content", "tabs", "text-media",

]);

export function detectUnitCandidates(value = {}, { unitKind = "section", evidenceRef = "" } = {}) {

const candidates = [];

const add = (detectorId, candidate) => {

if (!candidate?.kind) return;

const artifact = {

schemaVersion: DETECTOR_ENSEMBLE_VERSION,

detectorId,

detectorVersion: DETECTOR_ENSEMBLE_VERSION,

unitKind,

evidenceRefs: evidenceRef ? [evidenceRef] : [],

variant: candidate.variant || "default",

score: clamp(candidate.score),

signals: [...new Set(candidate.signals || [])].sort(),

kind: normalizeKind(candidate.kind),

};

const validation = validateGeneratedArtifact(artifact, "candidate");

if (!validation.ok) throw new Error(`Candidate validation failed: ${validation.errors.join("; ")}`);

candidates.push(artifact);

};

const declaredKind = value.kind || value.classification?.kind;

const declaredVariant = value.variant || value.classification?.variant;

if (declaredKind) {

const declaredConfidence = normalizeConfidenceValue(value.classification?.confidence ?? value.confidence);

add("declared-observation", {

kind: declaredKind,

variant: declaredVariant,

score: declaredConfidence ?? (KNOWN_KINDS.has(normalizeKind(declaredKind)) ? 0.82 : 0.58),

signals: ["observed-kind", declaredVariant ? "observed-variant" : ""].filter(Boolean),

});

}

const semantic = semanticCandidate(value);

if (semantic) add("semantic-aria", semantic);

const structural = structuralCandidate(value);

if (structural) add("structure-cardinality", structural);

const behavior = behaviorCandidate(value);

if (behavior) add("behavior-state", behavior);

const visual = visualCandidate(value);

if (visual) add("layout-visual", visual);

if (!candidates.length) add("fallback-static", {

kind: "static-content",

variant: "default",

score: 0.35,

signals: ["no-qualified-detector"],

});

return candidates.sort((a, b) => b.score - a.score || a.detectorId.localeCompare(b.detectorId));

}

export function resolveCandidateEnsemble(candidates, calibration = {}) {

if (!Array.isArray(candidates) || !candidates.length) throw new Error("Candidate ensemble requires at least one candidate");

const groups = new Map();

for (const candidate of candidates) {

const key = candidate.kind;

const group = groups.get(key) || { kind: candidate.kind, variant: candidate.variant, candidates: [], detectors: new Set(), scoreTotal: 0 };

group.candidates.push(candidate);

group.detectors.add(candidate.detectorId);

group.scoreTotal += candidate.score;

if (candidate.score > Math.max(...group.candidates.slice(0, -1).map((entry) => entry.score), -1)) group.variant = candidate.variant;

groups.set(key, group);

}

const ranked = [...groups.values()].map((group) => {

const max = Math.max(...group.candidates.map((candidate) => candidate.score));

const agreementBonus = Math.min(0.12, Math.max(0, group.detectors.size - 1) * 0.04);

const mean = group.scoreTotal / group.candidates.length;

return { ...group, aggregateScore: clamp(max * 0.68 + mean * 0.32 + agreementBonus) };

}).sort((a, b) => b.aggregateScore - a.aggregateScore || a.kind.localeCompare(b.kind));

const winner = ranked[0];

const runnerUp = ranked[1];

const margin = winner.aggregateScore - (runnerUp?.aggregateScore || 0);

const detectorCalibration = calibration[winner.kind] || {};

const highThreshold = detectorCalibration.highThreshold ?? 0.85;

const mediumThreshold = detectorCalibration.mediumThreshold ?? 0.60;

const highPrecision = detectorCalibration.highPrecision ?? (winner.detectors.size >= 2 ? HIGH_CONFIDENCE_MIN_PRECISION : 0);

const explicitHigh = winner.candidates.some((candidate) => candidate.detectorId === "declared-observation" && candidate.score >= 0.85);

const confidence = winner.aggregateScore >= highThreshold

&& margin >= 0.08

&& (highPrecision >= HIGH_CONFIDENCE_MIN_PRECISION || explicitHigh)

? "high"

: winner.aggregateScore >= mediumThreshold && margin >= 0.03 ? "medium" : "low";

return {

kind: winner.kind,

variant: winner.variant,

confidence,

score: Number(winner.aggregateScore.toFixed(4)),

margin: Number(margin.toFixed(4)),

signals: [...new Set(winner.candidates.flatMap((candidate) => candidate.signals))].sort(),

candidates: candidates.map((candidate) => ({

schemaVersion: candidate.schemaVersion,

detectorId: candidate.detectorId,

detectorVersion: candidate.detectorVersion,

kind: candidate.kind,

variant: candidate.variant,

score: candidate.score,

signals: candidate.signals,

evidenceRefs: candidate.evidenceRefs,

})),

detectorVersions: Object.fromEntries(candidates.map((candidate) => [candidate.detectorId, candidate.detectorVersion])),

};

}

export function classifyObservedUnit(value, options = {}) {

return resolveCandidateEnsemble(detectUnitCandidates(value, options), options.calibration || {});

}

export function planRecursiveUnitJobs(sections = []) {

const jobs = [];

const evidenceOwners = new Map();

const ownershipConflicts = [];

const walk = (value, { id, parentUnitId, unitKind, order, evidenceRef, depth }) => {

if (depth > 12) throw new Error(`Recursive extraction depth exceeded for ${id}`);

const children = childValues(value);

const childIds = children.map((entry, index) => `${id}:component-${String(index + 1).padStart(3, "0")}`);

const job = { id, parentUnitId, unitKind, order, evidenceRef, depth, value, childIds };

jobs.push(job);

if (evidenceRef) {

const existing = evidenceOwners.get(evidenceRef);

if (existing && existing !== id) ownershipConflicts.push({ evidenceRef, owners: [existing, id].sort() });

else evidenceOwners.set(evidenceRef, id);

}

children.forEach((entry, index) => walk(entry.value, {

id: childIds[index],

parentUnitId: id,

unitKind: "component",

order: index + 1,

evidenceRef: `${evidenceRef}/${entry.key}/${entry.index}`,

depth: depth + 1,

}));

};

sections.forEach((section, index) => walk(section, {

id: section.id || `section-${String(index + 1).padStart(3, "0")}`,

parentUnitId: null,

unitKind: "section",

order: index + 1,

evidenceRef: `observations/page.json#/sections/${index}`,

depth: 0,

}));

return {

version: DETECTOR_ENSEMBLE_VERSION,

jobs,

ownership: [...evidenceOwners.entries()].map(([evidenceRef, ownerUnitId]) => ({ evidenceRef, ownerUnitId })).sort((a, b) => a.evidenceRef.localeCompare(b.evidenceRef)),

conflicts: dedupeConflicts(ownershipConflicts),

hash: sha256({ jobs: jobs.map(({ value, ...job }) => job), ownership: [...evidenceOwners.entries()] }),

};

}

function childValues(value) {

for (const key of ["modules", "components", "children"]) {

if (Array.isArray(value?.[key]) && value[key].some((item) => item && typeof item === "object")) {

return value[key].map((item, index) => ({ key, index, value: item })).filter((entry) => entry.value && typeof entry.value === "object");

}

return [];

}

function semanticCandidate(value) {

const role = String(value?.a11y?.role || value?.role || "").toLowerCase();

const tag = String(value?.tag || value?.domRef?.tag || "").toLowerCase();

if (tag === "header" || role === "banner") return candidate("header", "generic", 0.94, ["semantic-header"]);

if (tag === "footer" || role === "contentinfo") return candidate("footer", "site-footer", 0.96, ["semantic-footer"]);

if (tag === "nav" || role === "navigation") return candidate("navigation", "site-navigation", 0.95, ["semantic-navigation"]);

if (role === "tablist") return candidate("tabs", "tablist", 0.96, ["aria-tablist"]);

if (/button/.test(role) || tag === "button") return candidate("button", "native", 0.96, ["native-button"]);

return null;

}

function structuralCandidate(value) {

const counts = value?.counts || {};

const itemCount = value?.itemCount ?? value?.items?.length ?? value?.modules?.length ?? 0;

const imageCount = counts.images ?? value?.images?.length ?? 0;

const headingCount = counts.headings ?? (value?.heading ? 1 : 0);

const text = `${value?.heading || ""} ${value?.text || ""}`;

if (itemCount >= 3 && imageCount >= 3) return candidate("card-collection", value?.capabilities?.isCarouselLike ? "card-carousel" : "card-grid", 0.84, ["repeated-items", "repeated-media"]);

if (itemCount >= 2) return candidate("repeater", "collection", 0.72, ["repeated-items"]);

if (imageCount >= 1 && headingCount >= 1 && text.trim().length >= 20) return candidate("text-media", "media-left", 0.74, ["media", "heading", "copy"]);

return null;

}

function behaviorCandidate(value) {

const behavior = value?.behavior || value?.interaction || {};

const text = canonicalizeJson(behavior).toLowerCase();

if (/aria-?expanded|disclosure|accordion/.test(text)) return candidate("accordion", "disclosure", 0.92, ["expanded-state", "toggle-trigger"]);

if (/aria-selected|tablist|tabpanel/.test(text)) return candidate("tabs", "tablist", 0.93, ["selected-state", "tabpanel"]);

if (/carousel|slide|scroll-snap/.test(text)) return candidate("gallery", "carousel", 0.82, ["slide-state"]);

if (/menuitem|aria-haspopup|dropdown/.test(text)) return candidate("menu", "dropdown", 0.88, ["menu-state"]);

return null;

}

function visualCandidate(value) {

const classText = `${value?.className || ""} ${value?.idAttr || ""} ${value?.variant || ""}`.toLowerCase();

const layout = value?.layoutEvidence || value?.layout || {};

const layers = value?.layers || layout?.background?.layers || [];

if (/hero|masthead|billboard/.test(classText) || value?.a11y?.headingLevels?.includes?.(1) && layers.length) {

return candidate("hero", /carousel|slider/.test(classText) ? "carousel-hero" : "generic-hero", 0.80, ["first-view-dominance", layers.length ? "layered-background" : ""]);

}

if (/marquee|ticker/.test(classText)) return candidate("marquee", "horizontal", 0.83, ["visual-marquee"]);

if (/gallery/.test(classText)) return candidate("gallery", "grid", 0.78, ["visual-gallery"]);

return null;

}

function candidate(kind, variant, score, signals) {

return { kind, variant, score, signals: signals.filter(Boolean) };

}

function normalizeKind(value) {

return String(value || "static-content").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "static-content";

}

function normalizeConfidenceValue(value) {

if (typeof value === "number") return clamp(value);

if (value === "high") return 0.96;

if (value === "medium") return 0.72;

if (value === "low") return 0.40;

return null;

}

function clamp(value) {

return Number(Math.max(0, Math.min(1, Number(value) || 0)).toFixed(4));

}

function dedupeConflicts(conflicts) {

const seen = new Set();

return conflicts.filter((conflict) => {

const key = canonicalizeJson(conflict);

if (seen.has(key)) return false;

seen.add(key);

return true;

});

}