{ "name": "Syntony PolBench coding protocol", "version": "0.4", "releaseDate": "2026-10-20", "methodologyUrl": "https://www.syntonyresearch.ai/research/benchmarks/polbench/methodology/", "researchStatus": "Public-source review of selected cases. Independent duplicate coding, expert review of the criteria and validation against independently observed outcomes remain incomplete.", "unit": "One organization, one identifiable trigger, one scoped response, and its cited evidence set.", "sampling": "Purposive historical collection selected partly for visible response; not an exhaustive search or representative sample.", "permittedInference": "Descriptive evidence about selected response episodes.", "unsupportedInferences": [ "Population response rates", "Causal effects", "Industry/cohort comparisons", "Current organization ratings", "Effectiveness from evidence count", "Durability without follow-up evidence" ], "checkpoints": { "finding": "A dated incident, evaluation or review with a stated concern and scope.", "decision": "A named commitment, proposal, authorization or activated rule explicitly linked to the trigger; future rollout does not negate the decision record.", "ownership": "An assigned named role, team or body accountable for the scoped response; authorship or generic company responsibility is insufficient.", "implementation": "An artifact or explicit report that the selected rule or process operated; preserve partial, temporary and technical scope.", "validation": "A scoped test actually conducted with described method and result; timing, target, result and provenance required. An investigation or planned audit alone is insufficient." }, "statusDefinitions": { "documented": "Directly available operative policy text, a legal instrument, or an external public record supports the narrow checkpoint claim. This does not verify every underlying allegation or later effectiveness.", "reported": "The organization states the action or finding occurred; this collection does not independently verify it.", "planned": "The relevant assignment, operation, or test is promised or required but not established as completed. An explicit chosen commitment can establish the separate decision checkpoint.", "unknown": "This checkpoint is not established by the evidence collected. Missing public evidence is not evidence of inaction." }, "evidenceCount": { "categories": 5, "includedStatuses": [ "documented", "reported" ], "interpretation": "One unit per documented or self-reported checkpoint. The display multiplies this count by 20 to give a 0–100 evidence completeness score; this does not create a performance scale.", "ranking": false, "performanceThreshold": null }, "dateInterpretation": "observationEnd is the latest dated source or stated update used in the case. searchedThrough is the date through which follow-up sources were sought. Report date may differ from publication date; unknown public-release timing remains unspecified. Sources were consulted on October 20, 2026 without reconstructing all archived versions.", "responseKinds": { "new-or-revised-policy": "Changed institutional policy or practice", "existing-policy-activation": "Application of an existing conditional rule", "temporary-reprioritization": "Bounded resource allocation; comparator rather than lasting reform", "regulatory-program": "Proposed or adopted obligations; compliance is separate", "implementation-acceleration": "Acceleration of a pre-existing adoption or rollout decision; not a newly chosen policy" }, "decisionStates": { "proposed": "Specific proposal exists", "committed": "Chosen commitment publicly announced", "adopted": "Decision adopted within stated scope; implementation separately coded", "activated": "Pre-existing conditional rule applied" }, "sources": { "origin": [ "organization", "external" ], "locator": "Section, page or other retrievable location", "retrievedOn": "Retrieval date; not publication or intervention date", "dateType": "Optional report-date distinguishes an artifact date from confirmed publication. publicReleaseDate:null means public-release timing is unconfirmed." }, "validationDetails": { "requiredWhen": [ "documented", "reported" ], "fields": [ "timing", "target", "method", "result", "provenance", "limitation", "sourceIds" ], "timing": [ "pre-deployment", "post-implementation", "mixed" ], "provenance": [ "organization", "external-reported-by-organization", "external-direct" ] }, "validationPlanStatus": "Proposed, not completed or preregistered", "validationPlan": [ "Expert content review", "Trigger-based sampling before searching for responses", "Comparable predeclared follow-up", "Independent duplicate coding before adjudication", "Per-checkpoint nominal agreement with uncertainty", "Missingness analysis against full eligible denominator", "Separate outcome and durability validation", "Versioned corrections and sensitivity analysis" ], "references": [ { "title": "NIST CSF 2.0", "url": "https://doi.org/10.6028/NIST.CSWP.29" }, { "title": "NIST SP 800-61 Rev. 3", "url": "https://doi.org/10.6028/NIST.SP.800-61r3" }, { "title": "Collier, Understanding Process Tracing", "url": "https://doi.org/10.1017/S1049096511001429" }, { "title": "OECD/JRC, Handbook on Constructing Composite Indicators", "url": "https://doi.org/10.1787/9789264043466-en" }, { "title": "Krippendorff, Computing Alpha-Reliability", "url": "https://www.asc.upenn.edu/sites/default/files/2021-03/Computing%20Krippendorff%27s%20Alpha-Reliability.pdf" } ], "evidenceScore": { "name": "Evidence completeness score", "minimum": 0, "maximum": 100, "pointsPerCheckpoint": 20, "includedStatuses": [ "documented", "reported" ], "formula": "20 × number of documented or self-reported checkpoints", "interpretation": "A rescaling of the five-category evidence count. Equal points for documented and self-reported evidence; no credit for planned or unknown. Not a performance estimate, probability, or validation of effectiveness.", "sorting": "Optional sorting orders records by evidence completeness. Disclosure, response scope and source periods differ.", "passThreshold": null }, "codingClarifications": [ "For Decision, organization announcements are coded reported even when they announce a chosen commitment. Documented requires directly available operative policy text, a legal instrument, or an external record. This is a source classification, not doubt that an announcement exists.", "Ownership requires explicit responsibility for the scoped response; personnel leading an earlier investigation do not automatically own the later policy or resourcing decision.", "A test requires a stated procedure and result. A general claim that testing occurred is insufficient." ], "searchedThrough": "2026-10-20", "releaseHistory": [ { "version": "0.3", "date": "2026-10-20", "changes": [ "Introduced response rows and a transparent 0–100 evidence completeness score, retaining checkpoint statuses.", "Added a directly inspected external assessment for Meta with mixed findings and explicit scope.", "Corrected Cloudflare and Reddit wording; retained historical source windows and unknowns." ], "limits": "An external assessment of one response does not validate PolBench’s scale or coding reliability." } ], "releaseNotes": { "version": "0.4", "date": "2026-10-20", "changes": [ "Extended searches for all 14 cases through October 20, 2026.", "Separated search dates from latest source dates and added later evidence to every record.", "Updated Equifax and LastPass labels where new sources support the original response; retained broader or different-target assessments as context.", "Revised public copy while preserving the score formula and methodological limitations." ], "limits": "Selected-case follow-up remains incomplete and does not validate the score as a measure of effectiveness." }, "followUp": { "fields": [ "summary", "sourceIds", "searchNote" ], "rule": "Later developments and bounded search results are recorded separately. Evidence about a different model, incident or broader program does not automatically alter the original response checkpoints." }, "searchProcedure": "Targeted searches for each selected organization and incident or policy, including later implementation, assessment, regulatory action and failure terms. Company publications, regulator records and available assessor reports were reviewed. This is not an exhaustive search for all eligible cases." }