/** * ATS fidelity metrics — score a PDF's extracted text against the fixture's * ground truth. * * The design principle: every metric maps to a real ATS failure mode that * distinguishes Chromium print-to-PDF from purpose-built PDF engines. * "Are these two strings similar" is not a metric — "did the company name * survive extraction intact, the way an ATS needs it to" is. * * All metrics return a number in [0, 1] where 1 = perfect fidelity. * `compareMetrics` then takes Chromium's score and WeasyPrint's score for * each (fixture, template, metric) triple and labels the gap: * ≤2% ✅ negligible * ≤5% ⚠️ material * >5% ❌ significant * * These thresholds are the same ones in the plan — they're the decision * boundaries for "is Chromium-only good enough to ship." */ import { UNICODE_TOKENS, CROSS_PAGE_MARKER } from "../ats-fixtures"; import type { ResumeData } from "../../src/components/builder/use-resume"; import { SECTION_TITLES } from "../../src/components/builder/templates/shared"; export interface MetricResult { /** Stable identifier — used as the JSON key and the report row label. */ id: string; /** Human-readable explanation of what this metric proves. */ description: string; /** Score for the Chromium PDF, in [0, 1]. */ chromium: number; /** Score for the WeasyPrint PDF, in [0, 1]. */ weasyprint: number; /** Optional human-readable detail (failing tokens, counts, etc.). */ detail?: string; } export interface FixtureReport { fixture: string; template: string; metrics: MetricResult[]; } export type Verdict = "pass" | "warn" | "fail"; /** Compute the verdict from a Chromium vs WeasyPrint gap. */ export function gapVerdict(chromium: number, weasyprint: number): Verdict { const gap = Math.abs(chromium - weasyprint); if (gap <= 0.02) return "pass"; if (gap <= 0.05) return "warn"; return "fail"; } // --------------------------------------------------------------------------- // Token extraction helpers — pull the ground-truth strings out of a fixture // so the metrics have something to check the extracted text against. // --------------------------------------------------------------------------- /** Every word in the fixture's "interesting" text fields, lowercased, as a * Set. Used by the word-integrity metric. We deliberately exclude generic * English ("the", "and") by only pulling from structured fields (names, * company, role, skills, bullets) — those are the ATS-relevant tokens. * * Tokenization: split on whitespace, then strip leading/trailing * punctuation (commas, periods, colons, parens, quotes). ATS parsers do * the same — "TypeScript," in the PDF text becomes "typescript" for * indexing. Without this normalization the metric would falsely report a * skill as missing because the extracted text has "typescript," (trailing * comma) and the ground truth has "typescript". */ function groundTruthWords(data: ResumeData): Set { const words = new Set(); const add = (s: string | null | undefined) => { if (!s) return; for (const w of s.toLowerCase().split(/\s+/)) { // Strip leading/trailing punctuation. Keep internal hyphens/apostrophes // ("full-time", "d'expérience") since those are part of the word. const stripped = w.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""); if (stripped.length >= 2) words.add(stripped); } }; add(data.firstName); add(data.lastName); add(data.headline); add(data.summary); add(data.contact.email); add(data.contact.phone); add(data.contact.location); for (const link of data.contact.links ?? []) add(link.label); for (const e of data.experience) { add(e.role); add(e.company); add(e.location); for (const b of e.bullets) add(b); } for (const ed of data.education) { add(ed.institution); add(ed.degree); add(ed.field); } for (const s of data.skills) { add(s.category); for (const item of s.items) add(item); } for (const p of data.projects) { add(p.name); add(p.description); for (const b of p.bullets) add(b); } for (const c of data.certifications) add(c.name); for (const l of data.languages) { add(l.name); add(l.level); } return words; } /** Specific tokens (proper nouns, technical terms, years) the metrics check * verbatim. These are ATS load-bearing: an ATS that fails to extract * "Anthropic" or "2023" can't match the candidate against a job posting. */ function groundTruthTokens(data: ResumeData): string[] { const tokens: string[] = []; // Identity — always checked. tokens.push(`${data.firstName} ${data.lastName}`.trim()); if (data.headline) tokens.push(data.headline); // Every company + every year mentioned. for (const e of data.experience) { if (e.company) tokens.push(e.company); if (e.startDate) tokens.push(e.startDate); if (e.endDate) tokens.push(e.endDate); } // Skill items — the highest-value ATS keywords. for (const s of data.skills) { for (const item of s.items) tokens.push(item); } // Certifications — high-signal keywords. for (const c of data.certifications) tokens.push(c.name); return tokens.filter(Boolean); } // --------------------------------------------------------------------------- // Normalization — pdftotext output has whitespace noise (column breaks // become multiple spaces, soft hyphens, form feeds between pages). Normalize // before comparing so the metrics measure CONTENT loss, not formatting noise. // --------------------------------------------------------------------------- /** Normalize extracted text for comparison: collapse runs of whitespace to a * single space, strip form feeds (page breaks), preserve case (case-matters * for proper nouns like "Anthropic" vs "anthropic"). */ function normalize(s: string): string { return s .replace(/\f/g, " ") // form feed (page break) .replace(/[\u00ad\u200b\u2060]/g, "") // soft hyphen, zero-width space, word joiner .replace(/\s+/g, " ") .trim(); } // --------------------------------------------------------------------------- // Individual metrics. Each takes the extracted text + the fixture and // returns a score in [0, 1] plus optional detail. // --------------------------------------------------------------------------- /** METRIC 1 — Word integrity. * * Of the ground-truth words, what fraction appear intact in the extracted * text? This is the most direct ATS proxy: an ATS indexes the document by * words; a missing word = an unmatchable keyword. * * "Intact" means the word appears as a standalone token (whitespace on * both sides — or punctuation), NOT as a substring of a longer run. The * classic Chromium failure mode is to merge adjacent words into one run; * a substring check would falsely pass on "antseattle" containing "anthropic". */ function wordIntegrityScore( extracted: string, data: ResumeData, ): { score: number; detail: string } { const truth = groundTruthWords(data); if (truth.size === 0) return { score: 1, detail: "no ground-truth words" }; const normalized = normalize(extracted.toLowerCase()); // Build a Set of extracted tokens using the SAME tokenization as ground // truth (split on whitespace, strip leading/trailing punctuation). A // mismatched tokenizer would measure the tokenization difference, not an // extraction difference. const extractedTokens = new Set( normalized .split(/\s+/) .map((w) => w.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")) .filter((w) => w.length >= 2), ); let found = 0; const missing: string[] = []; for (const w of truth) { if (extractedTokens.has(w)) { found++; } else { missing.push(w); } } const score = found / truth.size; const detail = missing.length === 0 ? `all ${truth.size} ground-truth words found` : `missing ${missing.length}/${truth.size}: ${missing.slice(0, 8).join(", ")}${missing.length > 8 ? "…" : ""}`; return { score, detail }; } /** METRIC 2 — Section title recognizability. * * ATS parsers use the seven canonical section titles to locate * Experience/Skills/Education blocks. If a title is missing from the * extracted text, the parser may not segment the resume correctly and will * misattribute bullets to the wrong section. * * We check the seven titles from SECTION_TITLES (the shared source of * truth the templates themselves use). All seven are present in every * fixture by construction, so any miss is a real extraction failure. * * Comparison is CASE-INSENSITIVE because (a) ATS parsers are * case-insensitive — a recruiter searching "experience" matches "EXPERIENCE", * "Experience", and "experience" — and (b) the templates apply * `text-transform: uppercase` to section titles, which Chromium honors in * the text layer (so its PDF extracts "EXPERIENCE") and WeasyPrint does not * (its PDF extracts "Experience"). A case-sensitive comparison would * therefore give a false 0% to whichever engine doesn't match the source * string's case — measuring a CSS-text-transform implementation difference, * not an ATS-fidelity difference. ATS-fidelity is the question, so we * compare case-insensitively. */ function sectionTitleScore(extracted: string): { score: number; detail: string } { const normalized = normalize(extracted).toLowerCase(); const titles = Object.values(SECTION_TITLES); let found = 0; const missing: string[] = []; for (const title of titles) { if (normalized.includes(title.toLowerCase())) { found++; } else { missing.push(title); } } const score = found / titles.length; return { score, detail: missing.length === 0 ? `all ${titles.length} section titles found` : `missing: ${missing.join(", ")}`, }; } /** METRIC 3 — Contact info extraction (email, phone, URLs). * * ATS uses regexes to pull email and phone. If the regex doesn't match in * the extracted text, the candidate's contact info is silently dropped. * This is the highest-friction ATS failure: a candidate whose email can't * be parsed may not even show up in the recruiter's CRM. */ function contactInfoScore( extracted: string, data: ResumeData, ): { score: number; detail: string } { const normalized = normalize(extracted); const checks: Array<{ label: string; pass: boolean }> = []; if (data.contact.email) { // Email regex — forgiving around case, exact on the local/domain parts. const re = new RegExp( data.contact.email.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i", ); checks.push({ label: `email(${data.contact.email})`, pass: re.test(normalized) }); } if (data.contact.phone) { // Phone — strip the fixture's formatting and check the digits appear in // order in the extracted text. ATS parsers normalize phones, so a // missing parens or different separator shouldn't fail the check; a // missing digit should. const digits = data.contact.phone.replace(/\D/g, ""); // Pull digit runs from extracted text and check one contains them. const extractedDigitRuns = normalized.match(/\d[\d\s().-]{4,}\d/g) ?? []; const extractedDigitStrings = extractedDigitRuns.map((r) => r.replace(/\D/g, "")); const phonePass = extractedDigitStrings.some((s) => s.includes(digits)); checks.push({ label: `phone(${data.contact.phone})`, pass: phonePass }); } for (const link of data.contact.links ?? []) { // What does ATS actually see? Browser PDFs (Chromium, WeasyPrint) put // the visible link LABEL in the text layer, not the href. The // templates render `GitHub` — only "GitHub" is // extractable. Real ATS parsers see the same: they get the visible text. // Some ATS (and recruiters) do click through to the URL, but that's a // separate concern from text-layer fidelity. Check the label here. const label = link.label ?? new URL(link.url).hostname; checks.push({ label: `link(${label})`, pass: normalize(extracted).toLowerCase().includes(label.toLowerCase()), }); } const passed = checks.filter((c) => c.pass).length; const score = checks.length === 0 ? 1 : passed / checks.length; const failed = checks.filter((c) => !c.pass).map((c) => c.label); return { score, detail: failed.length === 0 ? `all ${checks.length} contact fields matched` : `missing: ${failed.join(", ")}`, }; } /** METRIC 4 — Key token preservation (proper nouns, skills, years). * * Distinct from word-integrity: this checks that multi-word tokens * ("Anthropic", "AWS Solutions Architect — Professional", "Stanford * University", "2023") survive as contiguous substrings. Word-integrity * would pass even if "Stanford University" became "Stanford" + "University" * on separate lines; this metric catches the case where a multi-word * proper noun got split across runs and lost its phrase identity. */ function keyTokenScore( extracted: string, data: ResumeData, ): { score: number; detail: string } { const normalized = normalize(extracted); const tokens = groundTruthTokens(data); if (tokens.length === 0) return { score: 1, detail: "no key tokens" }; let found = 0; const missing: string[] = []; for (const t of tokens) { // For multi-word tokens, normalize the token the same way and check // substring. For single words, check word-boundary match. const normalizedToken = normalize(t.toLowerCase()); const haystack = normalized.toLowerCase(); if (normalizedToken.includes(" ")) { if (haystack.includes(normalizedToken)) found++; else missing.push(t); } else { const re = new RegExp(`\\b${escapedRegex(normalizedToken)}\\b`, "i"); if (re.test(haystack)) found++; else missing.push(t); } } const score = found / tokens.length; return { score, detail: missing.length === 0 ? `all ${tokens.length} key tokens found` : `missing ${missing.length}/${tokens.length}: ${missing.slice(0, 6).join(", ")}${missing.length > 6 ? "…" : ""}`, }; } function escapedRegex(s: string): string { return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } /** METRIC 5 — Unicode preservation (UNICODE fixture only). * * Checks that each non-ASCII ground-truth token (José, Müller, Zürich, —, * smart quotes, Présent) appears verbatim in the extracted text. UTF-8 * round-trip is a silent failure mode: some PDFs replace unmappable chars * with `?`, and the document still looks fine visually. */ function unicodeScore(extracted: string): { score: number; detail: string } { const normalized = normalize(extracted); let found = 0; const missing: string[] = []; for (const tok of UNICODE_TOKENS) { if (normalized.includes(tok)) found++; else missing.push(tok); } const score = found / UNICODE_TOKENS.length; return { score, detail: missing.length === 0 ? `all ${UNICODE_TOKENS.length} unicode tokens preserved` : `missing: ${missing.join(", ")}`, }; } /** METRIC 6 — Cross-page bullet continuity (LONG fixture only). * * The classic Chromium PDF failure: a long bullet straddles a page break * and `pdftotext` reads the two halves as separate paragraphs. We check * that the long bullet — including its unique marker — survives extraction * as one continuous run, OR if fragmented, that the marker + a meaningful * chunk of the bullet's distinctive vocabulary are still all present. * * Scoring: * 1.0 = the full bullet text (sans marker) survives as one contiguous run * 0.5 = the marker survives but the bullet is fragmented; vocabulary is * mostly preserved * 0.0 = the marker itself is lost (the bullet was obliterated) */ function crossPageScore( extracted: string, _data: ResumeData, ): { score: number; detail: string } { const normalized = normalize(extracted); const markerPresent = normalized.includes(CROSS_PAGE_MARKER); // Distinctive vocabulary from the long bullet — used to detect "fragmented // but content survived" vs "lost entirely". Pulled from ats-fixtures.ts's // long bullet. Keep in sync with that fixture. const distinctiveVocab = [ "multi-quarter", "redesign", "ingress", "legacy", "proxies", "telemetry", "240ms", "78ms", "$340K", "developer velocity", ]; const vocabFound = distinctiveVocab.filter((v) => normalized.toLowerCase().includes(v.toLowerCase()), ).length; if (!markerPresent) { return { score: 0, detail: `CROSS_PAGE_MARKER lost; ${vocabFound}/${distinctiveVocab.length} vocab found`, }; } // Strip the marker and check if the bullet survived as one contiguous run. // The bullet body (after the marker) is ~50 words; check if a long // contiguous chunk is present. const afterMarker = normalized.slice(normalized.indexOf(CROSS_PAGE_MARKER)); // Remove the marker itself. const bulletBody = afterMarker.replace(CROSS_PAGE_MARKER, "").trim(); // Check if the first ~60 chars (a chunk longer than a single line) appear // as a contiguous run after the marker. const expectedStart = "Led a multi-quarter initiative to redesign"; const contiguousSurvived = bulletBody .slice(0, 100) .includes(expectedStart); if (contiguousSurvived) { return { score: 1, detail: `marker present, bullet body contiguous (${vocabFound}/${distinctiveVocab.length} vocab)`, }; } return { score: 0.5, detail: `marker present but bullet fragmented; ${vocabFound}/${distinctiveVocab.length} vocab survived`, }; } // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- /** Compute all applicable metrics for one (extracted-text, fixture) pair. * Returns the per-metric score; the caller wraps Chromium and WeasyPrint * results into a MetricResult. */ export interface PerMetricScore { id: string; description: string; score: number; detail: string; } export function scoreExtractedText( extracted: string, fixtureName: string, data: ResumeData, ): PerMetricScore[] { const scores: PerMetricScore[] = [ { id: "word-integrity", description: "Of ground-truth keywords, how many appear as standalone tokens", ...wordIntegrityScore(extracted, data), }, { id: "section-titles", description: "Of the 7 canonical section titles, how many are extractable", ...sectionTitleScore(extracted), }, { id: "contact-info", description: "Email, phone, and URLs matchable by standard ATS regexes", ...contactInfoScore(extracted, data), }, { id: "key-tokens", description: "Proper nouns, skills, and years survive as contiguous phrases", ...keyTokenScore(extracted, data), }, ]; if (fixtureName === "unicode") { scores.push({ id: "unicode-preservation", description: "Non-ASCII tokens (José, Müller, Zürich, em-dash, smart quotes) preserved verbatim", ...unicodeScore(extracted), }); } if (fixtureName === "long") { scores.push({ id: "cross-page-continuity", description: "Long bullet straddling a page break survives as one continuous run", ...crossPageScore(extracted, data), }); } return scores; } /** Combine per-engine scores into the MetricResult shape the report uses. */ export function combineScores( chromium: PerMetricScore[], weasyprint: PerMetricScore[], ): MetricResult[] { const wpById = new Map(weasyprint.map((s) => [s.id, s])); return chromium.map((c) => { const wp = wpById.get(c.id); return { id: c.id, description: c.description, chromium: c.score, weasyprint: wp?.score ?? 0, detail: wp?.detail === c.detail ? c.detail : `chromium: ${c.detail} | weasyprint: ${wp?.detail ?? "?"}`, }; }); }