From f59adcb7099e8784caebafee7c42b5d367564dd1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 28 Aug 2026 23:29:29 -0700 Subject: [PATCH 1/9] fix(differential,guards): narrow three 1.4.0 rules to the roles they exercise Each declared a role no diff it explains moves: `middle` on fix(cjk-comma-compound) and on both fix(#367) particle rules. classify() admits a rule when the diff is a SUBSET of its `fields`, so the excess is not inert -- it lets the rule keep claiming a diff that shrinks out of the declared role, which is what #410 found on fix(#424) and what #452 is about. Narrowed to the measured union of the diffs each rule explains, which cannot orphan a name: every name a rule explains contributed to that union and still matches. _CORPUS_CLAIMS re-measured, not adjusted. Gate 229/0. Refs #452 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 20 ++++- tools/differential/expected_since_1.4.0.toml | 82 ++++++++++++++++---- 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 860772b1..9bbfaacb 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1531,7 +1531,7 @@ def _claim(rule: dict) -> _Claim: "fix(#325) a credential run across a second comma reads as suffixes": _Claim(1, ('suffix', 'title'), "f025c5f70a4e"), "fix(#367) an inferred title no longer displaces a leading particle either": - _Claim(1, ('family', 'given', 'middle'), "d8ee9cd5da5f"), + _Claim(1, ('family', 'given'), "d8ee9cd5da5f"), "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(279, ('family', 'given'), "28a62b622a48"), "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost": @@ -1553,7 +1553,7 @@ def _claim(rule: dict) -> _Claim: "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name": _Claim(23, ('given', 'suffix'), "344de804e2c6"), "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": - _Claim(23, ('family', 'given', 'middle', 'suffix', 'title'), "344de804e2c6"), + _Claim(23, ('family', 'given', 'suffix', 'title'), "344de804e2c6"), "fix(cjk-glued-honorific-peel) glued honorific peels into suffix": _Claim(37, ('family', 'given', 'suffix'), "719c31233502"), "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": @@ -1565,7 +1565,7 @@ def _claim(rule: dict) -> _Claim: "fix(#424) an unlisted abbreviation is as transparent as a listed title to the leading particle": _Claim(1, ('family', 'given'), "ca7b37af6cf8"), "fix(#367) a title no longer displaces a leading particle out of the leading position": - _Claim(3, ('family', 'given', 'middle'), "724967a4a117"), + _Claim(3, ('family', 'given'), "724967a4a117"), "fix(#400) abd joins the word after it as one given name": _Claim(11, ('given', 'middle'), "1eaed91fc574"), "fix(#272/#308) nakaguro division and a glued hangul honorific in one name": @@ -2031,6 +2031,20 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: # they are in the same tier and both reach the name -- and a # later edit that moves either one silently hands it back. (".,", ("given",)): "fix(A2) content-free input names nobody, so every role empties", + # The one exception the cjk-comma-compound rule's `middle` + # argument turns on, added by #452's review. That comment says + # ten of the eleven names it explains have a single-token + # post-comma tail and this is the eleventh, escaping `middle` + # only because v1's fix_phd merges the split credential before + # parsing. Pinned because the count itself is not: _claim + # measures regex REACH (23 here), which does not move when a + # name inside it changes hands -- so if this name ever went to + # another rule, the argument would go false with nothing + # saying so. Shape measured against the 1.4.0 tag: v1 reads + # first '田中さん', suffix 'Ph. D.'; the tree reads family + # '田中', suffix 'さん, Ph. D.'. + ("田中さん, Ph. D.", ("family", "given", "suffix")): + "fix(cjk-comma-compound) comma routing compounds with the CJK order flip", # The jr rule's surplus, added by the #453 review. Its regex # reaches these three and does not explain them; `fields` is # what makes it ineligible -- none of the shapes below is a diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 85c6ca27..82f89c38 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -996,8 +996,34 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip # above, which names that shape and whose fields exclude `family` so # the two cannot compete. Until #372 they landed on fix(comma-family) # instead, on nothing but file order. +# +# `middle` was in that fields list too, on the strength of the +# mechanism paragraph above -- the CJK order flip CAN move a middle +# name for a three-token pre-comma run. None of the 11 names this +# rule actually explains has one: ten have a single-token post-comma +# tail, and the eleventh, '田中さん, Ph. D.', has two ('Ph.' and +# 'D.') but fix_phd merges them into one suffix unit before v1's +# comma split can turn the second token into `middle` the way an +# untreated two-token tail does at 1.4.0 ('Smith, John Middle' reads +# first 'John', middle 'Middle'). So no diff this rule claims has +# ever moved `middle`, for that reason rather than pre-comma token +# count. +# +# classify() admits by subset, so leaving `middle` out is not +# passive: a future diff that grows to include it alongside `title` +# or `suffix` -- the two roles this rule's comma-routing half +# actually claims -- now falls out of this rule instead of being +# silently absorbed, and should arrive UNEXPLAINED and be read once +# rather than get a widened rule pre-built for it (#452). That +# promise does not cover every shape, though: a diff that moves ONLY +# {given, middle, family} -- the order flip with no comma routing at +# all -- still classifies above, on fix(#271/#272/#298), whose bare +# CJK-codepoint regex sits ahead of this one and whose fields were +# never narrowed. `middle` alongside `title` or `suffix` is the +# combination #452 protects here; `middle` alone is +# fix(#271/#272/#298)'s to explain, as it always was. name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" -fields = ["given", "middle", "family", "title", "suffix"] +fields = ["given", "family", "title", "suffix"] [[change]] issue = "fix(cjk-glued-honorific-peel) glued honorific peels into suffix" @@ -1472,20 +1498,33 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # before. (`\.?` was inert into the bargain: `\S+` is greedy, so it # had already taken the period.) # -# `fields` is what carries the rest of the tightness -- and it is narrower -# than the change rather than a description of it. The change moves -# `title` as well: a word in BOTH vocabularies stops the transparency -# scan and stays a title piece instead of being chained onto the name, -# so 'Dr. St John Smith' goes from title 'Dr.', family 'St John Smith' -# to title 'Dr. St', given 'John', family 'Smith'. It moves `maiden` -# as well: un-chaining lets a marker standing behind the particle be -# seen at all, so 'Mr. Van Johnson nee Brown' goes from family -# 'Van Johnson nee Brown' with no maiden name to given 'Van', family -# 'Johnson', maiden 'Brown'. `suffix` and `nickname` are the two that -# genuinely never move. Naming three roles anyway is deliberate, for -# the reason the regex names only `van`: a corpus name that moves -# `title` or `maiden` should arrive UNEXPLAINED and be read once, -# rather than be absorbed here. +# `fields` is what carries the rest of the tightness -- and it is +# narrower than the change rather than a description of it. The wider +# mechanism moves more than `given`/`family`: un-chaining reveals +# whatever was hidden behind the particle, so 'Dr. St John Smith' +# moves `title` (title 'Dr.', family 'St John Smith' -> title +# 'Dr. St', given 'John', family 'Smith'), 'Mr. Van Johnson nee Brown' +# moves `maiden` (family 'Van Johnson nee Brown' with no maiden -> +# given 'Van', family 'Johnson', maiden 'Brown'), and a third token +# after the particle moves `middle` the same way ('Mr. Van Johnson +# Middle': v1 gives it all to `last`; the un-chained reading gives +# given 'Van', middle 'Johnson', last 'Middle'). `suffix` and +# `nickname` are the two roles that genuinely never move. +# +# `fields` is what keeps those wider shapes out, NOT the regex -- +# measured, `^(mr|dr|sir)\.?\s+van\b` has no end anchor and does +# reach 'Mr. Van Johnson nee Brown' and 'Mr. Van Johnson Middle'; +# only 'Dr. St John Smith' is outside it. The diff on the three names +# this rule actually claims (given in the next paragraph) is +# {given, family} alone, and `fields` names those two roles and no +# others, so `title`, `maiden` and `middle` are left out on purpose. +# classify() admits by subset, so leaving a role out is not passive: +# a diff that grows to include one of them falls out of THIS rule +# instead of being silently absorbed. Where it lands then is a +# separate question -- a `maiden` diff on the nee name above goes to +# fix(#274), which is its right home; a `middle` or `title` one is +# claimed by nothing and arrives unclassified, to be read once rather +# than have a widened rule pre-built for it (#452). # # This rule has to exist even though the run already exited 0 without # it. The diff was being classified by fix(suffix-routing), a @@ -1497,7 +1536,7 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # since deleted the catch-all, and no ledger has a fields-only rule # for anything to be claimed back from. name_regex = "(?i)^(mr|dr|sir)\\.?\\s+van\\b" -fields = ["given", "middle", "family"] +fields = ["given", "family"] [[change]] issue = "fix(#367) an inferred title no longer displaces a leading particle either" @@ -1508,8 +1547,17 @@ issue = "fix(#367) an inferred title no longer displaces a leading particle eith # member would now reach a word the vocabulary does not ship, and the # shape gets its literal. The reading is unchanged: 'Jr.' is still a # title by shape, and Van is still the leading name piece. +# +# `middle` was declared here too, the same excess the 'Mr./Dr./Sir.' +# rule above drops and for the same reason: this rule's one name, +# 'Jr. Van Johnson', diffs {given, family} alone. classify() admits +# by subset, so leaving `middle` out means a diff that grows to +# include it falls out of this rule instead of being silently +# absorbed. Measured, nothing else claims a `middle` diff on this +# rule's name either, so it arrives unclassified and is read once +# rather than have a widened rule pre-built for it (#452). name_regex = "(?i)^jr\\.\\s+van\\b" -fields = ["given", "middle", "family"] +fields = ["given", "family"] # Shapes that must never be explained. A [[change]] rule says "this # diff is intended, and here is what changed"; there was no rule From c504d086d120f63bc9d8397b8540e46a0edd8e84 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:02:07 -0700 Subject: [PATCH 2/9] fix(differential,guards): narrow five 2.0.0 rules to the roles they exercise Same defect as the previous commit, measured against this baseline: two fix(#296) credential rules declare `family` and `given`, two fix(#367) particle rules and fix(#412) declare `middle`, and no diff any of them explains moves those roles here. The fix(#296) pair keeps `family` and `given` in the 1.4.0 ledger on purpose -- that baseline produces bigger diffs and does move them, so the same rule is exactly exercised there and over-declared here. Each ledger is measured on its own run rather than copied. _CORPUS_CLAIMS re-measured. Gate 194/0. Refs #452 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 10 +-- tools/differential/expected_since_2.0.0.toml | 91 +++++++++++++++++--- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 9bbfaacb..992f0e6d 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1670,7 +1670,7 @@ def _claim(rule: dict) -> _Claim: "fix(#424) an unlisted abbreviation is as transparent as a listed title to the leading particle": _Claim(1, ('_ambiguities', 'family', 'given'), "ca7b37af6cf8"), "fix(#367) a title no longer displaces a leading particle out of the leading position": - _Claim(3, ('family', 'given', 'middle'), "724967a4a117"), + _Claim(3, ('family', 'given'), "724967a4a117"), "fix(#380) a trailing vd after a family comma is the tussenvoegsel, not a post-nominal": _Claim(2, ('family', 'suffix'), "ec0d45289dc1"), "fix(#399) a maiden marker bounds the particle chain that swallowed it": @@ -1686,7 +1686,7 @@ def _claim(rule: dict) -> _Claim: "fix(#411) the bound-given reserve stops counting words the maiden name takes": _Claim(1, ('given', 'maiden', 'middle'), "7515923c9613"), "fix(#412) a connective join no longer absorbs the maiden marker beside it": - _Claim(2, ('family', 'maiden', 'middle'), "51c0eb36b5c5"), + _Claim(2, ('family', 'maiden'), "51c0eb36b5c5"), "fix(#418) the connective carve-out counts the name the maiden clause leaves behind": _Claim(1, ('family', 'given', 'middle'), "7923e6d3c5a7"), "fix(#418) accepted: a suffix word inside the maiden name ends it, connective or not": @@ -1716,9 +1716,9 @@ def _claim(rule: dict) -> _Claim: "fix(#296) dr is not postnominal vocabulary, so a trailing Dr. is a name word": _Claim(9, ('family', 'middle', 'suffix'), "8d6e9a1b43c0"), "fix(#296) a credential-only comma string reads a name and its postnominal": - _Claim(2, ('family', 'given', 'suffix', 'title'), "3f983ff71dee"), + _Claim(2, ('suffix', 'title'), "3f983ff71dee"), "fix(#296) a lone post-comma credential is a suffix": - _Claim(18, ('family', 'given', 'suffix', 'title'), "1f79efa10444"), + _Claim(18, ('suffix', 'title'), "1f79efa10444"), "fix(#325) a split credential followed by another suffix after a one-word family comma reads as suffixes": _Claim(6, ('given', 'suffix', 'title'), "7911e0158337"), "fix(#325) a credential run across a second comma reads as suffixes": @@ -1732,7 +1732,7 @@ def _claim(rule: dict) -> _Claim: "fix(#296) an ambiguous acronym counts as a suffix only when written with its periods": _Claim(1, ('_ambiguities', 'family', 'suffix'), "e13b3c769de4"), "fix(#367) an inferred title no longer displaces a leading particle either": - _Claim(1, ('family', 'given', 'middle'), "d8ee9cd5da5f"), + _Claim(1, ('family', 'given'), "d8ee9cd5da5f"), "fix(#424) accepted: a particle of the suffix vocabulary opening the trailing run is a suffix piece": _Claim(1, ('_ambiguities', 'family', 'middle', 'suffix'), "a564b97f7162"), "fix(#424) an unlisted abbreviation is as transparent as a listed title to the leading particle, the P4 example": diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index c67f0366..b3af2e1d 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -520,8 +520,24 @@ issue = "fix(#296) a credential-only comma string reads a name and its postnomin # name; with 'phd' out of TITLES the comma structure reads it as a # one-word name plus a postnominal -- a reading of a string nobody # writes as a name, and the honest one for it. +# +# `given` and `family` are OUT here: by 2.0.0 the comma-family rule +# has already moved the pre-comma word from `first` to `last`, so +# both names diff {title, suffix} alone at this baseline -- checked +# against the released 2.0.0 and the tree directly, 'Jr., PhD' reads +# last 'Jr.' on both sides and only title 'PhD' -> suffix 'PhD' moves. +# The 1.4.0 copy of this same rule keeps `given`/`family`: there the +# pre-comma word is still v1's `first`, so the same names diff all +# four roles at that baseline and the wider declaration is exercised, +# not excess. classify() admits by subset, so leaving the two roles +# out here is not passive -- a future diff on either name that grows +# to move `given` or `family` falls out of this rule instead of being +# silently absorbed; measured against the narrowed ledger, no other +# rule claims a {given, family, title, suffix} diff on these two names +# either, so it would arrive unclassified rather than pre-claimed +# (#452). name_regex = "(?i)^[a-z]{2,3}\\.?,\\s*phd$" -fields = ["title", "given", "family", "suffix"] +fields = ["title", "suffix"] [[change]] issue = "fix(#296) a lone post-comma credential is a suffix" @@ -535,8 +551,21 @@ issue = "fix(#296) a lone post-comma credential is a suffix" # so the postnominal position is read first now: a post-comma run # that is nothing but suffix pieces is suffixes. A one-word name, a # comma, one short word: the reach is pinned by the guards. +# +# `given` and `family` are OUT here for the same reason as the +# credential-only rule above: the comma-family move already landed by +# 2.0.0, so 'Smith, Jr.' reads last 'Smith' on both the 2.0.0 release +# and the tree and diffs {title, suffix} alone -- checked directly. +# The 1.4.0 copy of this same rule keeps `given`/`family` because +# there the pre-comma word is still moving `first` -> `last`, the +# larger diff that baseline actually produces. classify() admits by +# subset, so a future diff here that grows to move `given` or `family` +# falls out of this rule instead of being silently absorbed; measured +# against the narrowed ledger, nothing else in this file claims a +# {given, family, title, suffix} diff on these names, so it would +# arrive unclassified (#452). name_regex = "(?i)^[a-z]+,\\s*[a-z]{2,6}\\.?$" -fields = ["title", "given", "family", "suffix"] +fields = ["title", "suffix"] [[change]] issue = "fix(#296) a glued honorific before a lone credential: the credential is the postnominal" @@ -755,11 +784,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # as well: un-chaining lets a marker standing behind the particle be # seen at all, so 'Mr. Van Johnson nee Brown' goes from family # 'Van Johnson nee Brown' with no maiden name to given 'Van', family -# 'Johnson', maiden 'Brown'. `suffix` and `nickname` are the two that -# genuinely never move. Naming three roles anyway is deliberate, for -# the reason the regex names only `van`: a corpus name that moves -# `title` or `maiden` should arrive UNEXPLAINED and be read once, -# rather than be absorbed here. +# 'Johnson', maiden 'Brown'. `suffix` and `nickname` are the two roles +# that genuinely never move. +# +# `fields` names the two this rule's diffs DO move and no others, so +# `title`, `maiden` and `middle` are all left out on purpose, for the +# reason the regex names only `van`. It is `fields` that keeps the +# wider shapes out rather than the regex: `^(mr|dr|sir)\.?\s+van\b` +# has no end anchor and does reach 'Mr. Van Johnson nee Brown'. A diff +# growing into one of the three left out falls out of this rule -- +# measured, {given, family, middle} on any of the three names it +# claims is claimed by nothing and arrives unclassified, to be read +# once rather than have a widened rule pre-built for it (#452). # # The 1.4 ledger carries the same rule, where it also has a second job: # there the diff was already being absorbed by that file's fields-only @@ -767,8 +803,21 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # ["given", "family", "suffix"] is a superset of this diff's fields. No # fields-only rule exists here, so this file's copy is the plain # classification. +# +# `middle` is OUT too, the same excess the paragraphs above already +# argue for `title` and `maiden`: the diff on the three names this rule +# actually claims here -- 'Mr. Van Nguyen', 'Dr. Van Johnson', 'Sir Van +# Johnson' -- is {given, family} alone at this baseline, checked +# directly against the released 2.0.0 and the tree (none of the three +# has a third name piece, so `middle` reads empty on both sides). +# classify() admits by subset, so leaving it out is not passive: a +# diff that grows to move `middle` falls out of this rule instead of +# being silently absorbed. Measured against the narrowed ledger, +# nothing else in this file claims a {given, family, middle} diff on +# any of the three names either, so it would arrive unclassified and +# be read once rather than have a widened rule pre-built for it (#452). name_regex = "(?i)^(mr|dr|sir)\\.?\\s+van\\b" -fields = ["given", "middle", "family"] +fields = ["given", "family"] [[change]] issue = "fix(#367) an inferred title no longer displaces a leading particle either" @@ -779,8 +828,19 @@ issue = "fix(#367) an inferred title no longer displaces a leading particle eith # member would now reach a word the vocabulary does not ship, and the # shape gets its literal. The reading is unchanged: 'Jr.' is still a # title by shape, and Van is still the leading name piece. +# +# `middle` was declared here too, the same excess the 'Mr./Dr./Sir' +# rule above drops and for the same reason: this rule's one name, +# 'Jr. Van Johnson', diffs {given, family} alone at this baseline, +# checked directly against the released 2.0.0 and the tree. classify() +# admits by subset, so leaving `middle` out means a diff that grows to +# include it falls out of this rule instead of being silently +# absorbed. Measured against the narrowed ledger, nothing else claims +# a {given, family, middle} diff on this rule's name either, so it +# arrives unclassified and is read once rather than have a widened +# rule pre-built for it (#452). name_regex = "(?i)^jr\\.\\s+van\\b" -fields = ["given", "middle", "family"] +fields = ["given", "family"] [[change]] issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" @@ -853,8 +913,19 @@ issue = "fix(#412) a connective join no longer absorbs the maiden marker beside # Jones'; the other name is left with one name word once the marker # takes the rest, so its diff gained `given` and it has a rule of its # own at the end of this file, as it does at 2.1.0. +# +# `middle` is OUT: checked directly against the released 2.0.0 and the +# tree, the one name this rule now explains diffs {family, maiden} +# alone -- `given` 'Jane' and `middle` (empty on both sides) are +# unmoved; only `last` shrinks from 'van der Berg née y Jones' to +# 'van der Berg' and `maiden` fills with 'y Jones'. classify() admits +# by subset, so declaring `middle` here was not inert: a diff that +# grows to move it falls out of this rule instead of being silently +# claimed. Measured against the narrowed ledger, nothing else in this +# file claims a {family, maiden, middle} diff on this name either, so +# it would arrive unclassified (#452). name_regex = "(?i)\\bn[eé]e\\s+(y|and)\\b" -fields = ["middle", "family", "maiden"] +fields = ["family", "maiden"] [[change]] issue = "fix(#418) the connective carve-out counts the name the maiden clause leaves behind" From 2f192899fd63fd77599e5a18f0db749187fee81b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:18:37 -0700 Subject: [PATCH 3/9] fix(differential,guards): narrow six 2.1.0 rules to the roles they exercise The last of the fourteen. Three fix(#296) credential rules declare `family` and/or `given`, two fix(#367) particle rules and fix(#412) declare `middle`, and no diff any of them explains moves those roles at this baseline. Measured against this ledger's own run rather than copied from the 2.0.0 edits. The fix(#296) rules keep the wider declaration in the 1.4.0 ledger on purpose: that baseline reads the pre-comma word as `first`, so all four roles move there and the declaration is earned. Every ledger now declares exactly the roles its rules exercise, which is what the check two commits from now asserts. _CORPUS_CLAIMS re-measured. Gate 102/0. Refs #452 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 12 +-- tools/differential/expected_since_2.1.0.toml | 106 +++++++++++++++++-- 2 files changed, 101 insertions(+), 17 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 992f0e6d..aa6210aa 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1774,7 +1774,7 @@ def _claim(rule: dict) -> _Claim: "fix(#424) an unlisted abbreviation is as transparent as a listed title to the leading particle": _Claim(1, ('_ambiguities', 'family', 'given'), "ca7b37af6cf8"), "fix(#367) a title no longer displaces a leading particle out of the leading position": - _Claim(3, ('family', 'given', 'middle'), "724967a4a117"), + _Claim(3, ('family', 'given'), "724967a4a117"), "fix(#380) a trailing vd after a family comma is the tussenvoegsel, not a post-nominal": _Claim(2, ('family', 'suffix'), "ec0d45289dc1"), "fix(#399) a maiden marker bounds the particle chain that swallowed it": @@ -1788,7 +1788,7 @@ def _claim(rule: dict) -> _Claim: "fix(#411) the bound-given reserve stops counting words the maiden name takes": _Claim(1, ('given', 'maiden', 'middle'), "7515923c9613"), "fix(#412) a connective join no longer absorbs the maiden marker beside it": - _Claim(2, ('family', 'maiden', 'middle'), "51c0eb36b5c5"), + _Claim(2, ('family', 'maiden'), "51c0eb36b5c5"), "fix(#418) the connective carve-out counts the name the maiden clause leaves behind": _Claim(1, ('family', 'given', 'middle'), "7923e6d3c5a7"), "fix(#418) accepted: a suffix word inside the maiden name ends it, connective or not": @@ -1818,15 +1818,15 @@ def _claim(rule: dict) -> _Claim: "fix(#296) dr is not postnominal vocabulary, so a trailing Dr. is a name word": _Claim(9, ('family', 'middle', 'suffix'), "8d6e9a1b43c0"), "fix(#296) a credential-only comma string reads a name and its postnominal": - _Claim(2, ('family', 'given', 'suffix', 'title'), "3f983ff71dee"), + _Claim(2, ('suffix', 'title'), "3f983ff71dee"), "fix(#296) a lone post-comma credential is a suffix": - _Claim(18, ('family', 'given', 'suffix', 'title'), "1f79efa10444"), + _Claim(18, ('suffix', 'title'), "1f79efa10444"), "fix(#325) a split credential followed by another suffix after a one-word family comma reads as suffixes": _Claim(6, ('given', 'suffix', 'title'), "7911e0158337"), "fix(#325) a credential run across a second comma reads as suffixes": _Claim(1, ('suffix', 'title'), "f025c5f70a4e"), "fix(#296) a glued honorific before a lone credential: the credential is the postnominal": - _Claim(1, ('family', 'suffix', 'title'), "01bf2bd3f895"), + _Claim(1, ('suffix', 'title'), "01bf2bd3f895"), "fix(#296) do is a name, so it no longer stops the leading-particle scan as a title": _Claim(1, ('family', 'given'), "faa2c70fc49e"), "fix(#296) dr is not postnominal vocabulary, so 'John Smith, Dr.' keeps its split and its title": @@ -1834,7 +1834,7 @@ def _claim(rule: dict) -> _Claim: "fix(#296) an ambiguous acronym counts as a suffix only when written with its periods": _Claim(1, ('_ambiguities', 'family', 'suffix'), "e13b3c769de4"), "fix(#367) an inferred title no longer displaces a leading particle either": - _Claim(1, ('family', 'given', 'middle'), "d8ee9cd5da5f"), + _Claim(1, ('family', 'given'), "d8ee9cd5da5f"), "fix(#424) accepted: a particle of the suffix vocabulary opening the trailing run is a suffix piece": _Claim(1, ('_ambiguities', 'family', 'middle', 'suffix'), "a564b97f7162"), "fix(#424) an unlisted abbreviation is as transparent as a listed title to the leading particle, the P4 example": diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index b9ee3023..d6d5c236 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -214,8 +214,22 @@ issue = "fix(#296) a credential-only comma string reads a name and its postnomin # name; with 'phd' out of TITLES the comma structure reads it as a # one-word name plus a postnominal -- a reading of a string nobody # writes as a name, and the honest one for it. +# +# `given` and `family` are OUT: by 2.1.0 the comma-family move has +# already landed, so both names diff {title, suffix} alone at this +# baseline -- checked directly, 'Jr., PhD' reads last 'Jr.' on both +# the 2.1.0 release and the tree and only title 'PhD' -> suffix 'PhD' +# moves. The 1.4.0 copy of this same rule keeps `given`/`family`: +# there the pre-comma word is still v1's `first`, so the same names +# diff all four roles at that baseline and the wider declaration is +# earned, not excess. classify() admits by subset, so leaving the two +# roles out here is not passive -- a diff on either name that grows to +# move `given` or `family` falls out of this rule instead of being +# silently absorbed; measured against the narrowed ledger, nothing +# else in this file claims a {given, family, title, suffix} diff on +# these names either, so it would arrive UNEXPLAINED (#452). name_regex = "(?i)^[a-z]{2,3}\\.?,\\s*phd$" -fields = ["title", "given", "family", "suffix"] +fields = ["title", "suffix"] [[change]] issue = "fix(#296) a lone post-comma credential is a suffix" @@ -229,16 +243,42 @@ issue = "fix(#296) a lone post-comma credential is a suffix" # so the postnominal position is read first now: a post-comma run # that is nothing but suffix pieces is suffixes. A one-word name, a # comma, one short word: the reach is pinned by the guards. +# +# `given` and `family` are OUT for the same reason as the +# credential-only rule above: the comma-family move already landed by +# 2.1.0, so 'Smith, Jr.' reads last 'Smith' on both the 2.1.0 release +# and the tree and diffs {title, suffix} alone -- checked directly. +# The 1.4.0 copy of this same rule keeps `given`/`family` because +# there the pre-comma word is still moving `first` -> `last`, the +# larger diff that baseline actually produces. classify() admits by +# subset, so a diff here that grows to move `given` or `family` falls +# out of this rule instead of being silently absorbed; measured +# against the narrowed ledger, nothing else in this file claims a +# {given, family, title, suffix} diff on these names, so it would +# arrive UNEXPLAINED (#452). name_regex = "(?i)^[a-z]+,\\s*[a-z]{2,6}\\.?$" -fields = ["title", "given", "family", "suffix"] +fields = ["title", "suffix"] [[change]] issue = "fix(#296) a glued honorific before a lone credential: the credential is the postnominal" # '田中さん, PhD': 'phd' left TITLES, so the lone post-comma credential # is the suffix it is, and the glued-honorific peel (#312) reads 田中 # and さん as before. title 'PhD' -> suffix 'さん, PhD'. +# +# `family` is OUT: at this baseline the glued-honorific peel (#312) +# already ships, so both the 2.1.0 release and the tree read family +# '田中' with さん already split into the postnominal position -- checked +# directly, only title 'PhD' -> suffix 'さん, PhD' moves. The 2.0.0 copy +# of this rule keeps `family` because #312 postdates that release: at +# 2.0.0 the whole 'さん' honorific is still riding on the family, so +# family '田中さん' -> '田中' is a real part of that baseline's diff. No +# 1.4.0 copy exists to compare against. classify() admits by subset, +# so a diff here that grows to move `family` falls out of this rule +# instead of being silently absorbed; measured against the narrowed +# ledger, nothing else in this file claims a {family, title, suffix} +# diff on this name either, so it would arrive UNEXPLAINED (#452). name_regex = "(?i)^田中さん,\\s*phd$" -fields = ["title", "family", "suffix"] +fields = ["title", "suffix"] [[change]] issue = "fix(#296) do is a name, so it no longer stops the leading-particle scan as a title" @@ -475,10 +515,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # seen at all, so 'Mr. Van Johnson nee Brown' goes from family # 'Van Johnson nee Brown' with no maiden name to given 'Van', family # 'Johnson', maiden 'Brown'. `suffix` and `nickname` are the two that -# genuinely never move. Naming three roles anyway is deliberate, for -# the reason the regex names only `van`: a corpus name that moves -# `title` or `maiden` should arrive UNEXPLAINED and be read once, -# rather than be absorbed here. +# genuinely never move. +# +# `fields` names the two this rule's diffs DO move and no others, so +# `title`, `maiden` and `middle` are all left out on purpose, for the +# reason the regex names only `van`. It is `fields` that keeps the +# wider shapes out rather than the regex: `^(mr|dr|sir)\.?\s+van\b` has +# no end anchor and does reach 'Mr. Van Johnson nee Brown' -- checked +# directly, that name diffs {given, family, maiden} at this baseline, +# outside `fields`, and classify() returns nothing for it here. A diff +# growing into `title` or `maiden` falls out of this rule the same way +# rather than being silently absorbed; it should arrive UNEXPLAINED +# and be read once (#452). # # The 1.4 ledger carries the same rule, where it also has a second job: # there the diff was already being absorbed by that file's fields-only @@ -486,8 +534,21 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # ["given", "family", "suffix"] is a superset of this diff's fields. No # fields-only rule exists here, so this file's copy is the plain # classification. +# +# `middle` is OUT too, the same excess the paragraphs above already +# argue for `title` and `maiden`: the diff on the three names this rule +# actually claims here -- 'Mr. Van Nguyen', 'Dr. Van Johnson', 'Sir Van +# Johnson' -- is {given, family} alone at this baseline, checked +# directly against the released 2.1.0 and the tree (none of the three +# has a third name piece, so `middle` reads empty on both sides). +# classify() admits by subset, so leaving it out is not passive: a +# diff that grows to move `middle` falls out of this rule instead of +# being silently absorbed. Measured against the narrowed ledger, +# nothing else in this file claims a {given, family, middle} diff on +# any of the three names either, so it would arrive UNEXPLAINED and be +# read once rather than have a widened rule pre-built for it (#452). name_regex = "(?i)^(mr|dr|sir)\\.?\\s+van\\b" -fields = ["given", "middle", "family"] +fields = ["given", "family"] [[change]] issue = "fix(#367) an inferred title no longer displaces a leading particle either" @@ -498,8 +559,20 @@ issue = "fix(#367) an inferred title no longer displaces a leading particle eith # member would now reach a word the vocabulary does not ship, and the # shape gets its literal. The reading is unchanged: 'Jr.' is still a # title by shape, and Van is still the leading name piece. +# +# `middle` was declared here too, the same excess the 'Mr./Dr./Sir' +# rule above drops and for the same reason: this rule's one name, +# 'Jr. Van Johnson', diffs {given, family} alone at this baseline, +# checked directly against the released 2.1.0 and the tree (no third +# name piece, so `middle` reads empty on both sides). classify() admits +# by subset, so leaving `middle` out means a diff that grows to include +# it falls out of this rule instead of being silently absorbed. +# Measured against the narrowed ledger, nothing else claims a +# {given, family, middle} diff on this rule's name either, so it +# arrives UNEXPLAINED and is read once rather than have a widened rule +# pre-built for it (#452). name_regex = "(?i)^jr\\.\\s+van\\b" -fields = ["given", "middle", "family"] +fields = ["given", "family"] [[change]] issue = "fix(#380) a trailing vd after a family comma is the tussenvoegsel, not a post-nominal" @@ -755,7 +828,7 @@ issue = "fix(#412) a connective join no longer absorbs the maiden marker beside # Since #445 (2026-08-27) this rule explains only the first of the # two: the marker in 'Jane née and Jones Smith' takes the rest of the # name and leaves one name word, so `given` empties into `family` and -# the diff outgrew these three fields. It has a rule of its own at +# the diff outgrew these fields. It has a rule of its own at # the end of this file. The names were split rather than the fields # widened, because 'Jane van der Berg née y Jones' keeps a given name # and adding `given` here would have pre-excused its loss. @@ -770,8 +843,19 @@ issue = "fix(#412) a connective join no longer absorbs the maiden marker beside # one arriving lands UNEXPLAINED and is read once, as a new name # should be. A marker followed by a name word ('Jane van der Berg née # Jones') is fix(#399)'s and does not move. +# +# `middle` is OUT: checked directly against the released 2.1.0 and the +# tree, the one name this rule now explains diffs {family, maiden} +# alone -- `given` 'Jane' and `middle` (empty on both sides) are +# unmoved; only `last` shrinks from 'van der Berg née y Jones' to +# 'van der Berg' and `maiden` fills with 'y Jones'. classify() admits +# by subset, so declaring `middle` here was not inert: a diff that +# grows to move it falls out of this rule instead of being silently +# claimed. Measured against the narrowed ledger, nothing else in this +# file claims a {family, maiden, middle} diff on this name either, so +# it would arrive UNEXPLAINED (#452). name_regex = "(?i)\\bn[eé]e\\s+(y|and)\\b" -fields = ["middle", "family", "maiden"] +fields = ["family", "maiden"] [[change]] issue = "fix(#418) the connective carve-out counts the name the maiden clause leaves behind" From e57022e884ed16f2f46ff256e1bc259e9b63d024 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:21:25 -0700 Subject: [PATCH 4/9] feat(differential): over_declared_rules, the check for a rule wider than its diffs Pure and unwired: the next commit calls it. classify() admits a rule when the diff is a SUBSET of its `fields`, so a rule keeps matching as the diff beneath it shrinks -- #410 narrowed 'Freiherr von Richthofen V' from three roles to two while the rule declaring all three kept claiming it, and no run said so. The statement is exact rather than heuristic: classify() already REQUIRES declared >= union(explained diffs), so the only possible error is the other direction and `declared == union` is the whole check. The union is also the repair. Dormant rules and rules with no `fields` are skipped, neither as an exemption -- one explains nothing by declaration and dormant_rules owns that finding, the other declares no roles to exceed. Refs #452 Co-Authored-By: Claude Opus 5 --- tests/v2/test_differential.py | 54 ++++++++++++++++++++++++++++++++ tools/differential/compare.py | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index f211193a..be3b5deb 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -1124,6 +1124,60 @@ def test_dormant_rules_sorts_before_diagnosing() -> None: assert report.undeclared[0].detail == "specific" +def test_over_declared_rules_flags_a_role_nothing_explains() -> None: + """The #452 shape: `fields` wider than every diff beneath it. + + classify() admits a rule when the diff is a SUBSET of `fields`, so + the excess is not inert -- it lets the rule keep claiming a name + whose diff shrank out of the declared role, which is what #410 + found on fix(#424) with no run saying so. + """ + rules = [{"issue": "fix(x) wide", "name_regex": "Smith", + "fields": ["given", "family", "suffix"]}] + found = compare.over_declared_rules( + rules, {"fix(x) wide": {"family", "suffix"}}) + assert len(found) == 1 + assert found[0].issue == "fix(x) wide" + assert found[0].unused == ("given",) + assert found[0].observed == ("family", "suffix") + + +def test_over_declared_rules_accepts_exactly_exercised_fields() -> None: + """Equality, not superset: `fields` may name every role its diffs + move and no more.""" + rules = [{"issue": "fix(x) exact", "name_regex": "Smith", + "fields": ["family", "suffix"]}] + assert compare.over_declared_rules( + rules, {"fix(x) exact": {"family", "suffix"}}) == () + + +def test_over_declared_rules_skips_a_dormant_rule() -> None: + """A dormant rule explains nothing by declaration, so there is no + union to compare against -- dormant_rules checks that claim in both + directions, and this must not duplicate or contradict it.""" + rules = [{"issue": "fix(x) idle", "name_regex": "Smith", + "fields": ["given", "family"], "dormant": "no corpus name"}] + assert compare.over_declared_rules(rules, {}) == () + + +def test_over_declared_rules_skips_a_rule_with_no_fields() -> None: + """A regex-only rule declares no roles, so it has nothing to + over-declare. (`fields` with no name_regex cannot exist since + #451.)""" + rules = [{"issue": "fix(x) regex only", "name_regex": "Smith"}] + assert compare.over_declared_rules( + rules, {"fix(x) regex only": {"family"}}) == () + + +def test_over_declared_rules_skips_a_rule_that_explained_nothing() -> None: + """Explaining nothing is dormancy's finding, not this one. Reporting + it here too would make one defect fail twice with two different + remedies.""" + rules = [{"issue": "fix(x) silent", "name_regex": "Smith", + "fields": ["given", "family"]}] + assert compare.over_declared_rules(rules, {}) == () + + def test_validate_rules_rejects_two_rules_sharing_an_issue() -> None: """The dormancy check identifies a rule by its `issue`, so a duplicate lets one rule hide behind the other -- it can explain diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 939f2b18..b6db9e41 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -781,6 +781,65 @@ def dormant_rules(rules: list[dict[str, object]], explained: set[str], return _Dormancy(tuple(undeclared), tuple(awake)) +class _OverDeclared(NamedTuple): + """One rule declaring a role nothing it explains moves. + + `unused` is the defect. `observed` is the repair -- the union of the + diffs the rule explained, which is what `fields` should say. + """ + issue: str + #: declared roles no explained diff moves, sorted + unused: tuple[str, ...] + #: the union of the diffs it explains, sorted; the correct `fields` + observed: tuple[str, ...] + + +def over_declared_rules( + rules: list[dict[str, object]], + roles_by_issue: dict[str, set[str]]) -> tuple[_OverDeclared, ...]: + """Rules whose declared `fields` exceed every diff they explain. + + classify() takes the first rule whose `fields` are a SUPERSET of the + observed diff, so a rule keeps matching when the diff beneath it + SHRINKS -- and shrinking is the common direction, most parser fixes + moving fewer roles rather than more. #410 narrowed + 'Freiherr von Richthofen V' from three roles to two while the + fix(#424) rule declaring all three kept claiming it, and no run + named the movement (decisions.md#H1). That rule was narrowed by + hand; this is what would have said so. + + The statement is exact rather than heuristic. classify() REQUIRES + `declared >= union(explained diffs)` -- declaring less would stop + the rule matching a name it explains -- so the only possible error + is the other direction, and `declared == union` is the whole check. + The union is also the repair, and narrowing to it cannot orphan a + name: every name the rule explains contributed to it. + + Two rules are skipped, neither as an exemption. A rule declaring + `dormant` explains nothing by declaration, so there is no union to + compare against, and dormant_rules already checks that claim in both + directions. A rule with no `fields` declares no roles and so has + nothing to over-declare; one with `fields` and no `name_regex` + cannot exist since #451. + + Pure, like dormant_rules: it needs only values main() already + derives, so it is testable without a corpus or a baseline worker. + """ + found: list[_OverDeclared] = [] + for rule in rules: + declared = rule.get("fields") + if "dormant" in rule or not isinstance(declared, list): + continue + moved = roles_by_issue.get(str(rule["issue"])) + if not moved: + continue # explains nothing -- dormant_rules owns it + unused = tuple(sorted(set(declared) - moved)) + if unused: + found.append(_OverDeclared( + str(rule["issue"]), unused, tuple(sorted(moved)))) + return tuple(found) + + def main() -> int: ap = argparse.ArgumentParser() # Every corpus by default: they have different blind spots (see From 6f2eb4875b3e0b3aa26b430754d7b852aaf363e7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:26:07 -0700 Subject: [PATCH 5/9] feat(differential): fail the gate on a rule wider than the diffs it explains Wires over_declared_rules into main(): a `roles_by_issue` accumulator beside `by_issue`, an OVER-DECLARED report beside the dormancy one, and a third clause on the return. Same exit code as an unexplained diff and a dormant rule, because all three mean the ledger has stopped describing what the code does. Silent on all three ledgers, because the previous three commits narrowed the fourteen rules it would have named -- measured, 0 over-declared where there were 14. Proved able to fail: widening one fix(#367) rule back by `middle` reports it. Closes #452 for the explained-reach half; the reach ceiling stays open on that issue. Co-Authored-By: Claude Opus 5 --- tools/differential/compare.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/differential/compare.py b/tools/differential/compare.py index b6db9e41..2dd21b17 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -932,6 +932,10 @@ def main() -> int: tell, old_rows = _run_worker(baseline, want_v2, corpus) print(f"baseline: nameparser {tell['__version__']} ({tell['__file__']})") by_issue: dict[str, list[str]] = {} + #: the union of the diffs each rule explained, for over_declared_rules. + #: Kept beside by_issue rather than inside it: the summary printout + #: and `changed` both read by_issue as a list of names. + roles_by_issue: dict[str, set[str]] = {} # BOTH surfaces' old/new are retained, not just the facade's. A diff # can exist on the v2 surface alone -- an _ambiguities-only change is # facade-identical by construction, and is the case _surfaces_for @@ -967,6 +971,7 @@ def main() -> int: (name, old["facade"], new, old.get("v2", {}), new_v2)) else: by_issue.setdefault(issue, []).append(name) + roles_by_issue.setdefault(issue, set()).update(diff) changed = [n for names in by_issue.values() for n in names] \ + [row[0] for row in unexplained] @@ -990,6 +995,17 @@ def main() -> int: f"the key") if dormancy.undeclared or dormancy.awake: print() + overwide = over_declared_rules(rules, roles_by_issue) + for wide in overwide: + print(f"OVER-DECLARED {wide.issue!r}\n " + f"declares {list(wide.unused)}, which no diff it explains " + f"moves; every one fits {list(wide.observed)}. Narrow " + f"`fields` to that. classify() matches by SUBSET, so the " + f"excess is not inert -- it lets this rule keep claiming a " + f"name whose diff shrinks out of the extra role, with " + f"nothing to say so (#452)") + if overwide: + print() if unexplained: print("Field names below are Role's, matching what a ledger " "`fields` rule must say.\n") @@ -1016,8 +1032,13 @@ def main() -> int: f" [v2 surface only]") # A rule explaining nothing is as much a broken contract as an # unexplained diff: both mean the ledger no longer describes what the - # code does. Same exit code, so neither can be the one nobody noticed. - return 1 if unexplained or dormancy.undeclared or dormancy.awake else 0 + # code does. A rule explaining LESS than it declares is the third + # way that happens -- it still matches, so nothing here looks + # broken, but the `fields` it names are no longer what the code + # moves. Same exit code for all three, so none of them is the one + # nobody noticed. + return 1 if unexplained or dormancy.undeclared or dormancy.awake \ + or overwide else 0 if __name__ == "__main__": From 0e0ea99905a9f030177d928c7af97725c33a5fbe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:50:31 -0700 Subject: [PATCH 6/9] docs(differential,decisions,mechanisms): record the declared-fields check The rule grammar gains the requirement; the fields-only arc section gains the decision, including why the statement is exact rather than heuristic (classify() already requires declared >= union, so the only possible error is the other direction) and why over-declaration is baseline-relative -- the fix(#296) pair is exactly exercised at 1.4.0 and over-declared at both 2.x baselines, which is why each ledger was measured on its own run rather than copied. mechanisms.md#LEDGER-RULE-SEPARATION said `fields` narrows by subset and stopped there, which is now half the contract: the exactness requirement is added beside it. The `[[never]]` passage says the requirement does not reach exclusions, which explain no diff, so the new paragraph above it cannot be read as governing them. Declined and recorded: `explains = N` per rule, on maintenance cost rather than merit, with the note that it is the design to revisit if a name-count shrink ever bites. Two figures corrected against measurement rather than pasted. The fix(#296) rule has TWO field lists across the three ledgers, not three -- both 2.x files narrowed to the same {suffix, title}. And #414's harvest moved 17 of the 30 _CORPUS_CLAIMS entries then present, not 20; the roster comment that owns the count says twenty and is wrong, left uncorrected by a prose-only bundle and recorded as such. Refs #452 Co-Authored-By: Claude Opus 5 --- docs/design/decisions.md | 4 +++ docs/design/mechanisms.md | 2 +- tools/differential/README.md | 54 ++++++++++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 3d05deb0..d5c0b8a4 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -633,6 +633,9 @@ Decisions that landed: - 2026-08-27 #451 — a ledger rule's character class must be a copy of the range the IMPLEMENTATION acts on, and is pinned to it. The emoji rule's first draft spanned `U+1F300-U+1FAFF` while `_tokenize.py`'s `_EMOJI_RANGES` divides on two narrower blocks: it claimed 1072 codepoints where the parser leaves the token whole, so its own prose ("an emoji inside a token divides it") was false for every one of them and a `{given, family}` diff with another cause would have classified as intended. `test_the_emoji_boundary_rule_copies_the_dividing_ranges` holds it, and fails on the draft it replaced. - 2026-08-27 #451 — `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads. Verified against all three: the ban removes the only shape that could occupy its second tier, so file order settles every tie there is and the function reorders nothing. Kept because it is what makes the ban safe to state for any reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture. Deleting it would move the guarantee from the code into a convention, which is the trade #451 was filed to undo. Recorded as a decision rather than as a consequence precisely so a later dead-code sweep finds the reasoning before the `return`. - 2026-08-27 #451 — past-tense counts stay in a ledger comment; live ones do not. "It claimed 25 until #372" is a fixed fact. "Four corpus names, and the prose fits all four" was written when the rule explained four and still read that way at fourteen, because nothing recomputes it. Where a count is owned by another comment, point at the owner rather than restating it — two copies means one gets updated and the other does not. +- 2026-08-28 #452 — a rule's declared `fields` must EQUAL the union of the diffs it explains, checked by `compare.py` at the end of every run and failing it like an unexplained diff. The statement is exact rather than heuristic, which is what makes it cheap: `classify()` already requires `declared >= union` for the rule to match the names it matches, so the only possible error is the other direction, and the union is simultaneously the check and the repair. Narrowing to it cannot orphan a name, since every name a rule explains contributed to it. Measured before landing: 3 of 67 explaining rules over-declared at 1.4.0, 5 of 58 at 2.0.0, 6 of 51 at 2.1.0 — all fourteen narrowed first, so the check was silent the day it arrived. +- 2026-08-28 #452 — over-declaration is BASELINE-RELATIVE, and each ledger is measured on its own run rather than copied. `fix(#296) a lone post-comma credential is a suffix` declared `{family, given, suffix, title}` in all three ledgers: exactly exercised at 1.4.0, where v1 reads the pre-comma word as `first` and all four roles move, and over-declared at both 2.x baselines, where the same behaviour moves only `{suffix, title}` — which is all it declares in those two files now. A reader comparing the three sees one rule with TWO different field lists and should read that as correct rather than as drift. Two and not three, measured: the 2.x pair narrowed to the same set, so the split is 1.4.0 against both 2.x ledgers, not one list per file. +- 2026-08-28 #452 — NO escape hatch, decided rather than deferred. `dormant` already covers the explains-nothing case in both directions, a rule with no `fields` has nothing to over-declare, and the ledger's own doctrine — "a rule that pre-claims shapes it has never seen is the #372 failure mode" — makes strictness the existing principle. Accepted cost, stated so it is not rediscovered as a surprise: the first rule that genuinely needs a wider declaration has to argue for a key the way `dormant` was argued for in #373, rather than reaching for one that already exists. Found rather than decided, and worth as much: @@ -645,6 +648,7 @@ Declined: - One replacement rule with a `(jr|v|mp|x|m\.?a\.?|mc)` tail (2026-08-27) — measured, not argued: `m\.?a\.?` matches the corpus fragment `'M.A.'`, which `_normalize` leaves as `'m.a'` and is not a `SUFFIX_ACRONYMS` entry, so the member guard rejects it — the same finding the `ambiguous-surname-acronym` rule already records. `x` is in no vocabulary set at all, reaching `suffix` through the trailing-roman-numeral fork, so it could not be a member either. - A literal alternation of all 14 absorbed names (2026-08-27), on #413's literal-anchored precedent — kept as the fallback if a reading turned out contested, and not needed: every family had an existing rule in another ledger, a decision to cite, or an open issue. - Extending #451 to the trailing-numeral ambiguity (2026-08-27) — `'Mohamad X'` loses its family to `_ROMAN`, and by C-i's "under uncertainty, default to AMBIGUOUS" a trailing `X` IS borne as an ordinary name in that position (Malcolm X). Numerals reach `suffix` through a pattern rather than a vocabulary set, so there is no ambiguous SUBSET for `x` to join — but the fork does not go unreported: measured, `parse('Mohamad X')` and `parse('John V')` both carry `AmbiguityKind.SUFFIX_OR_NAME`, which `_assign` raises from the fork itself because "V/X/I are ordinary middle initials, so taking it as a suffix is a call, not a fact". The contrast worth keeping is with the acronym half: `parse('Aishwarya Rai')` loses its family and reports NOTHING, because `rai` is unambiguous vocabulary. So the numeral fork already flags its own uncertainty and the vocabulary path does not — which is an argument about `SUFFIX_ACRONYMS_AMBIGUOUS`, not about `_ROMAN`, and it belongs to #342 rather than here. `'Malcolm X'` is in no corpus (`'Mohamad X'` is, and is what this bundle classified); no open issue on the numeral half. +- `explains = N` per rule (2026-08-28, #452's own first sketch) — catches a rule explaining fewer NAMES even while its fields stay exercised, which the field check does not. Declined on maintenance cost rather than merit: 179 numbers across the three ledgers, every one of them liable to move whenever a corpus does rather than only when a rule does. Measured on the precedent — #414's harvest moved 17 of the 30 `_CORPUS_CLAIMS` entries that existed then and added 14 more, parsing the roster at `7a10689^` and `7a10689`. (That roster's own comment says "Twenty of these moved"; measured three ways — that commit, the whole of PR #415, and the backtick-harvest commit after it — the figure is 17 every time. Left uncorrected here because this bundle is prose-only; the comment is the owner of that count and is where the fix belongs.) The field check needs no number and cannot go stale. This is the design to revisit if a name-count shrink ever bites. - Extending it to `'Donald mc'` (2026-08-27) — `mc` is `SUFFIX_ACRONYMS` AND `PARTICLES`, the collision C-ii decided for `vd`, and C-ii reaches this shape only through P6's attachment, which is scoped to the comma form. The authority is rules.md#S2's Accepted — "an unambiguous suffix is consumed even when that leaves no family name at all" — and NOT P6's scope note, which was this entry's first citation and does not support it. P6 promises the comma-less shapes "keep their positional reading", and for the words that are both particle and suffix vocabulary it does not hold: measured, `'Jong de'` and `'Smith do'` keep theirs (family 'de', family 'do') while `'Donald mc'` and `'Smith vd'` lose the family entirely to `suffix`. So `'Donald mc'` is out of #451's scope on S2's statement, and P6's own Accepted clause is falsified by two of the three words it sweeps in — its only example, `"Jong Anke de"`, is `de`-shaped, which is why no test catches it. Recorded here rather than fixed; it is a rules.md defect, not a ledger one. ### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index cde269af..8743b49c 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -73,7 +73,7 @@ Problem shape. A guard needs to know what the answer WAS, so it can detect the a ## LEDGER-RULE-SEPARATION — file order decides, fields narrow by subset -Problem shape. Two differential-ledger rules claim overlapping names. Contract statement. Every ledger rule must carry a `name_regex` — since #451 `validate_rules` REJECTS a rule with `fields` and no `name_regex`, and one with neither was already rejected — so every rule sits in ONE tier, the sort is stable, and FILE ORDER decides every contest: the narrower rule must be written first. `fields` narrows a rule by subset; it does not separate rules by sorting. The ban ends the SHAPE and not the property it enabled: a required `name_regex` bounds nothing by itself, since the only width check is the sentinel probe — measured, `[a-z]` validates and reaches 941 of 1090 corpus names. What changed is that such a rule now carries a `_CORPUS_CLAIMS` reach and digest, so its breadth is visible once at recording time rather than never (#452). The two-tier sort in `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads (measured on all three): it is the defence for a reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture — and its docstring in tools/differential/compare.py says so. How it works. Detail is owned by tools/differential/README.md. The file-order clause is measured, not theoretical: in the 1.4 ledger the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair should be separated by a predicate instead of by order is +Problem shape. Two differential-ledger rules claim overlapping names. Contract statement. Every ledger rule must carry a `name_regex` — since #451 `validate_rules` REJECTS a rule with `fields` and no `name_regex`, and one with neither was already rejected — so every rule sits in ONE tier, the sort is stable, and FILE ORDER decides every contest: the narrower rule must be written first. `fields` narrows a rule by subset; it does not separate rules by sorting. Narrowing by subset is not the whole contract: since #452 a rule's `fields` must EQUAL the union of the diffs it explains, and `compare.py` reports OVER-DECLARED and exits non-zero otherwise — a declared role no diff moves is not inert, it lets the rule keep claiming a name whose diff SHRINKS into the excess (decisions.md#differential-ledger). The ban ends the SHAPE and not the property it enabled: a required `name_regex` bounds nothing by itself, since the only width check is the sentinel probe — measured, `[a-z]` validates and reaches 941 of 1090 corpus names. What changed is that such a rule now carries a `_CORPUS_CLAIMS` reach and digest, so its breadth is visible once at recording time rather than never (#452). The two-tier sort in `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads (measured on all three): it is the defence for a reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture — and its docstring in tools/differential/compare.py says so. How it works. Detail is owned by tools/differential/README.md. The file-order clause is measured, not theoretical: in the 1.4 ledger the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair should be separated by a predicate instead of by order is [#382](https://github.com/derek73/python-nameparser/issues/382). The old #271/#272 slug taboo is RETIRED (#333): the canonical-rule selector that keyed on those substrings is deliberately deleted — rule authors are free to use them in compound slugs — and the surviving rosters select on their own explicit keys (_HONORIFIC_SOURCES and _LATIN_ALTERNATION_SOURCES by named issue strings, _SPAN_BEARING_RULES by exact leading fix(...) tag). Lives in. tools/differential/compare.py, the expected_since_*.toml ledgers. Reach for it when. A ledger rule's behavior seems to depend on where it sits in the file — it does, and the reorder mutation is the test (run twice in #375; it fails _CROSS_RULE_WINNERS). History: #372 (closed) measured the then-existing fields-only rule owning 1639 of 5257 name×field pairs as filed (2026-08-10); #375/#376 then cut its classifier-of-record share sharply, and the residual pair ownership was read as the last-resort tier working as designed rather than a defect — until #451 retired the shape outright (decisions.md#differential-ledger). #372's two proposed mechanical checks were DECLINED with measurements (see decisions.md#differential-ledger), not left open. diff --git a/tools/differential/README.md b/tools/differential/README.md index 6f77579d..2e084c91 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -301,11 +301,12 @@ Each `[[change]]` entry needs `issue` (a short label, ideally an issue number or `fix()` matching a `tests/v2/cases.py` classification) and `name_regex` (searched against the raw input string). It may narrow further with `fields` (the diffing rule matches -only if the observed diff fields are a subset of this list). Keep both -as tight as the actual diff allows -- a loose rule can mask a real -regression. `name_regex` is REQUIRED since #451: `validate_rules` -rejects a rule carrying `fields` and no `name_regex`, as it already -rejected one carrying neither. +only if the observed diff fields are a subset of this list), which +since #452 must also name EXACTLY the roles that rule's own diffs move +-- see below. Keep both as tight as the actual diff allows -- a loose +rule can mask a real regression. `name_regex` is REQUIRED since #451: +`validate_rules` rejects a rule carrying `fields` and no `name_regex`, +as it already rejected one carrying neither. **That closes the SHAPE, not the property.** A required `name_regex` is not a bound on how much a rule reaches: the only width check is the @@ -317,8 +318,44 @@ that such a rule now has a `_CORPUS_CLAIMS` reach and digest to record, so its breadth is visible ONCE, to whoever reviews that number, instead of being invisible forever. That roster is by its own docstring "inert for a brand-new rule", so the review is the check. A reach ceiling is -the mechanism that would bound this; it is proposed on -[#452](https://github.com/derek73/python-nameparser/issues/452). +the mechanism that would bound this; it is still proposed on +[#452](https://github.com/derek73/python-nameparser/issues/452), whose +other half -- the declared-fields check below -- has landed. + +**`fields` must be EXACT, not merely a bound.** A rule's declaration +must equal the union of the diffs it actually explains, and +`compare.py` recomputes that union at the end of every run: a rule +declaring a role none of its diffs moves is reported `OVER-DECLARED` +and exits the run non-zero, like an unexplained diff or a dormant rule +that explained one. The excess is not inert. `classify()` admits a +rule when the observed diff is a SUBSET of `fields`, so a role nothing +moves is a standing claim on every future diff that shrinks into it -- +which is how `fix(#424)` kept explaining 'Freiherr von Richthofen V' +after #410 narrowed that diff from three roles to two, with no run +ever naming it (see decisions.md#H1). The check is exact rather than +heuristic, and cheap for the same reason: `classify()` already +requires `declared >= union` for the rule to match the names it +matches, so the only possible error is the other direction, and the +union is simultaneously the check and the repair. Narrowing a rule to +it cannot orphan a name, since every name the rule explains +contributed to it. + +Width is BASELINE-RELATIVE, so measure each ledger on its own run +rather than copying an edit across the three. `fix(#296) a lone +post-comma credential is a suffix` declares all four of `{family, +given, suffix, title}` at 1.4.0, where v1 reads the pre-comma word as +`first` and every one of those roles moves, and only `{suffix, title}` +at both 2.x baselines, where the same behaviour moves nothing else. +One rule with two different field lists across the three files is +correct there, not drift. + +Two shapes are skipped, neither as an exemption: a rule declaring +`dormant` explains nothing by declaration, and the dormancy check +below owns that finding in both directions; a rule with no `fields` +declares no roles to exceed. There is no third way out, by decision +rather than by omission -- a rule that genuinely needs a wider +declaration must argue for a key of its own, the way `dormant` was +argued for in #373. **File order decides.** That is not a detail -- every rule in every ledger carries a `name_regex`, so they all sit in one tier, the sort @@ -380,7 +417,8 @@ can safely delete -- plus `name_regex` and `examples`. The examples are required, not decoration: a protected shape need not appear in any corpus, so the entry has to carry its own test data. `fields` is optional and narrows WHICH READING is protected, by the same subset -test the rules use. +test the rules use -- but only that test. An exclusion explains no +diff, so the exactness requirement above is not asked of it. That last key earns its keep on the ASCII pairs. Parens mark nicknames, maiden names, suffixes and credentials alike, and no regex tells them From 626e2a8ff683a13769c2f451d8fb6b68d3136f0f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:53:46 -0700 Subject: [PATCH 7/9] test(differential,guards): kill a vacuous test and a stale roster count Two things the #452 doc review found, both the failure mode this issue is about, one level up. test_over_declared_rules_skips_a_dormant_rule could not fail. It passed roles_by_issue={}, which the `if not moved` guard catches whether or not the `dormant` clause exists -- mutation-tested, the test is green with that clause deleted. A test that cannot fail, inside the check just landed to catch rules that cannot be seen to drift. It now passes a dormant rule that DID explain a diff, which is the only separating input and also the only interesting one: that state is what dormant_rules reports as NO LONGER DORMANT, one defect with one remedy (remove the `dormant` key), and reporting it here as well would demand a second, contradictory one. Verified against a mutant built outside the repo: with the clause deleted the rule is flagged, so the test now fails on its removal. _CORPUS_CLAIMS' own comment said "Twenty of these moved at once when corpus_rules.jsonl landed (#414)". Measured by parsing the roster out of 7a10689^ and 7a10689 with ast and diffing per-entry: 30 entries before, 44 after, 17 changed and 14 added. Three ways -- that commit, the whole of PR #415, the backtick-harvest commit after it -- all give 17. The comment carries the figure and the recipe now, and the decisions.md entry that cites it agrees rather than noting a contradiction it left standing. 5347 passed; gates 229/0, 194/0, 102/0, no OVER-DECLARED. Refs #452 Co-Authored-By: Claude Opus 5 --- docs/design/decisions.md | 2 +- tests/v2/test_differential.py | 21 +++++++++++++++++---- tests/v2/test_ledger_guards.py | 9 +++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index d5c0b8a4..421d4623 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -648,7 +648,7 @@ Declined: - One replacement rule with a `(jr|v|mp|x|m\.?a\.?|mc)` tail (2026-08-27) — measured, not argued: `m\.?a\.?` matches the corpus fragment `'M.A.'`, which `_normalize` leaves as `'m.a'` and is not a `SUFFIX_ACRONYMS` entry, so the member guard rejects it — the same finding the `ambiguous-surname-acronym` rule already records. `x` is in no vocabulary set at all, reaching `suffix` through the trailing-roman-numeral fork, so it could not be a member either. - A literal alternation of all 14 absorbed names (2026-08-27), on #413's literal-anchored precedent — kept as the fallback if a reading turned out contested, and not needed: every family had an existing rule in another ledger, a decision to cite, or an open issue. - Extending #451 to the trailing-numeral ambiguity (2026-08-27) — `'Mohamad X'` loses its family to `_ROMAN`, and by C-i's "under uncertainty, default to AMBIGUOUS" a trailing `X` IS borne as an ordinary name in that position (Malcolm X). Numerals reach `suffix` through a pattern rather than a vocabulary set, so there is no ambiguous SUBSET for `x` to join — but the fork does not go unreported: measured, `parse('Mohamad X')` and `parse('John V')` both carry `AmbiguityKind.SUFFIX_OR_NAME`, which `_assign` raises from the fork itself because "V/X/I are ordinary middle initials, so taking it as a suffix is a call, not a fact". The contrast worth keeping is with the acronym half: `parse('Aishwarya Rai')` loses its family and reports NOTHING, because `rai` is unambiguous vocabulary. So the numeral fork already flags its own uncertainty and the vocabulary path does not — which is an argument about `SUFFIX_ACRONYMS_AMBIGUOUS`, not about `_ROMAN`, and it belongs to #342 rather than here. `'Malcolm X'` is in no corpus (`'Mohamad X'` is, and is what this bundle classified); no open issue on the numeral half. -- `explains = N` per rule (2026-08-28, #452's own first sketch) — catches a rule explaining fewer NAMES even while its fields stay exercised, which the field check does not. Declined on maintenance cost rather than merit: 179 numbers across the three ledgers, every one of them liable to move whenever a corpus does rather than only when a rule does. Measured on the precedent — #414's harvest moved 17 of the 30 `_CORPUS_CLAIMS` entries that existed then and added 14 more, parsing the roster at `7a10689^` and `7a10689`. (That roster's own comment says "Twenty of these moved"; measured three ways — that commit, the whole of PR #415, and the backtick-harvest commit after it — the figure is 17 every time. Left uncorrected here because this bundle is prose-only; the comment is the owner of that count and is where the fix belongs.) The field check needs no number and cannot go stale. This is the design to revisit if a name-count shrink ever bites. +- `explains = N` per rule (2026-08-28, #452's own first sketch) — catches a rule explaining fewer NAMES even while its fields stay exercised, which the field check does not. Declined on maintenance cost rather than merit: 179 numbers across the three ledgers, every one of them liable to move whenever a corpus does rather than only when a rule does. Measured on the precedent — #414's harvest moved 17 of the 30 `_CORPUS_CLAIMS` entries that existed then and added 14 more, parsing the roster at `7a10689^` and `7a10689`. (That roster's own comment said "Twenty of these moved" and was wrong; measured three ways — that commit, the whole of PR #415, and the backtick-harvest commit after it — the figure is 17 every time, and the comment now says so with the recipe.) The field check needs no number and cannot go stale. This is the design to revisit if a name-count shrink ever bites. - Extending it to `'Donald mc'` (2026-08-27) — `mc` is `SUFFIX_ACRONYMS` AND `PARTICLES`, the collision C-ii decided for `vd`, and C-ii reaches this shape only through P6's attachment, which is scoped to the comma form. The authority is rules.md#S2's Accepted — "an unambiguous suffix is consumed even when that leaves no family name at all" — and NOT P6's scope note, which was this entry's first citation and does not support it. P6 promises the comma-less shapes "keep their positional reading", and for the words that are both particle and suffix vocabulary it does not hold: measured, `'Jong de'` and `'Smith do'` keep theirs (family 'de', family 'do') while `'Donald mc'` and `'Smith vd'` lose the family entirely to `suffix`. So `'Donald mc'` is out of #451's scope on S2's statement, and P6's own Accepted clause is falsified by two of the three words it sweeps in — its only example, `"Jong Anke de"`, is `de`-shaped, which is why no test catches it. Recorded here rather than fixed; it is a rules.md defect, not a ledger one. ### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index be3b5deb..446566ad 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -1152,12 +1152,25 @@ def test_over_declared_rules_accepts_exactly_exercised_fields() -> None: def test_over_declared_rules_skips_a_dormant_rule() -> None: - """A dormant rule explains nothing by declaration, so there is no - union to compare against -- dormant_rules checks that claim in both - directions, and this must not duplicate or contradict it.""" + """A `dormant` rule is this check's business never, even when it + HAS explained something. + + The input matters, and the obvious one is vacuous: passing an empty + `roles_by_issue` is caught by the `if not moved` guard whether or + not the dormant clause exists, so the test would pass on its + deletion -- mutation-tested, and the reason this row is written the + way it is (#452 review). The separating input is a dormant rule + that DID explain a diff, which is also the only interesting one: it + is exactly the state dormant_rules reports as NO LONGER DORMANT. + That is one defect with one remedy -- remove the `dormant` key -- + and reporting it here as well would demand a second, contradictory + one: narrow `fields` on a rule whose real problem is that its + dormancy claim went false. + """ rules = [{"issue": "fix(x) idle", "name_regex": "Smith", "fields": ["given", "family"], "dormant": "no corpus name"}] - assert compare.over_declared_rules(rules, {}) == () + assert compare.over_declared_rules( + rules, {"fix(x) idle": {"family"}}) == () def test_over_declared_rules_skips_a_rule_with_no_fields() -> None: diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index aa6210aa..b7bd31db 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1474,8 +1474,13 @@ def _claim(rule: dict) -> _Claim: #: That is the intended cost: a corpus name added under an existing #: rule is a real change in what that rule explains, and it should be #: read once rather than absorbed silently. -#: Twenty of these moved at once when corpus_rules.jsonl landed -#: (#414), which is a lot of re-recording to review. The jumps are all +#: Seventeen of these moved at once when corpus_rules.jsonl landed +#: (#414) and fourteen more were added, which is a lot of re-recording +#: to review. The figure read "Twenty" until #452 measured it: parse +#: this roster out of `7a10689^` and `7a10689` with ast and diff the +#: per-entry dumps -- 30 entries before, 44 after, 17 changed. The same +#: three ways (that commit, the whole of PR #415, the backtick-harvest +#: commit after it) all give 17. The jumps are all #: one cause -- 113 names arriving -- and the ones worth naming are #: the broad rules: the fields-only catch-alls grew to the whole #: corpus (751 -> 864), the comma rules 215 -> 236, fix(#274) 4 -> 11 From 13e432a63f84dd9c2761a7da52e59d4cf66455a9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 00:54:47 -0700 Subject: [PATCH 8/9] docs(decisions): three #451 claims the #452 review falsified All three are in the fields-only arc section, all three are mine, and all three were found by the design-docs review of the #452 docs commit. Measured, not argued. The reusable-lesson bullet named the wrong cause. It said the 2.x "should arrive UNEXPLAINED" idiom is true there BECAUSE those ledgers have no fields-only rule. A fields-only rule is the extreme way to break that promise, not the only way: measured while fixing #452, a `maiden` diff on fix(#367)'s own probe goes to fix(#274) in the 1.4.0 ledger, which has no fields-only rule at all. The check that generalizes is "does anything above me claim this shape", driven through classify(). This bullet is the one a future arc copies, so the wrong cause would have propagated. The emoji bullet asserted equality where the guard checks a subset, and said two blocks where _EMOJI_RANGES has four. The rule copies the two ASTRAL blocks and leaves the BMP pair out because no corpus name reaches them through its token anchor; the guard asserts only that the class reaches nothing the tokenizer ignores. Read as equality it would tell the next author to pre-claim the BMP half -- the #372 shape this same section leans on. "FORCED rather than chosen ... none of them taste" overstated by one. _LatinCopy.vocabulary is a hand-supplied frozenset, so declaring SUFFIX_WORDS | SUFFIX_ACRONYMS would let (jr\.?|v|mp|mc) pass the member checks -- measured. Two of the three splits are forced; the third is a judgement that an alternation should name one wordlist a reader can go and check, and it is recorded as one now. Refs #451, #452 Co-Authored-By: Claude Opus 5 --- docs/design/decisions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 421d4623..8f667e11 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -629,8 +629,8 @@ Decisions that landed: - 2026-08-27 #451 — `validate_rules` REJECTS a rule carrying `fields` and no `name_regex`. Such a rule claims every name whose diff fits those fields, and `_CORPUS_CLAIMS` records a regexless rule's reach as the WHOLE corpus — already at its maximum, so arrivals never move it and `test_every_rule_claims_the_recorded_share_of_the_corpus` passes whether it explains 4 names or 40. The one rule with the shape drifted from the four names its prose described to 25 before #372 cut it back, and then to SEVENTEEN after — decisions.md#H1 records that 14 -> 17 step and #410 taking three back — standing at fourteen across six unrelated behaviour families when this bundle opened, every guard green throughout, absorbing two open v2.2 bugs (#342, #397) as intentional 2.0 changes while AGENTS.md names the 1.4 summary as the release log's source for Behavior Changes. Free to state only because the fourteen were given rules first; a role-count floor was the earlier proposal and stays declined (#372/#373, vacuous). - 2026-08-27 #451 — a rule whose declared `fields` EXCEED every diff it explains is a defect in its own right, not untidiness. Measured on the catch-all: it declared `{given, family, suffix}`, and by the time this bundle's first three commits had given its `{given, family}` names rules of their own, nothing it still explained moved `given`. Note WHEN the excess appeared, because it is the whole mechanism — the declaration was earned when the rule was written and became excess as its explained set SHRANK under it, which is exactly `fix(#424)` at decisions.md#H1, where #410 narrowed a diff from three roles to two and the rule kept matching by superset with no run naming it. A rule does not have to widen to become over-declared; the diffs beneath it only have to leave. The two together are why #452 was filed — nothing counts what a rule actually explains, only whether it explains anything. - 2026-08-27 #451 — a ledger rule may classify a reading NOBODY WANTS, provided its `issue` says so. `fix(#342)` and `fix(#397)` carry NOT WANTED in the issue text, the cause-not-consent sentence decisions.md#P5 established for the #369 regression, and a delete-when-fixed instruction the dormancy check enforces once the fix lands. -- 2026-08-27 #451 — one vocabulary per alternation, FORCED rather than chosen. `_LatinCopy` carries one `vocabulary` and `test_latin_alternations_mean_something_the_vocabulary_ships` asserts exactly one roster key per alternation, so `jr` (SUFFIX_WORDS) and `mc`/`mp` (SUFFIX_ACRONYMS) cannot share one. That constraint forces ONE of the three splits. The other two are forced separately and are recorded below under the declined single-rule tail: `m\.?a\.?` cannot be an alternation member at all, and the numeral alternation copies no wordlist. Four rules, three independent reasons, none of them taste. -- 2026-08-27 #451 — a ledger rule's character class must be a copy of the range the IMPLEMENTATION acts on, and is pinned to it. The emoji rule's first draft spanned `U+1F300-U+1FAFF` while `_tokenize.py`'s `_EMOJI_RANGES` divides on two narrower blocks: it claimed 1072 codepoints where the parser leaves the token whole, so its own prose ("an emoji inside a token divides it") was false for every one of them and a `{given, family}` diff with another cause would have classified as intended. `test_the_emoji_boundary_rule_copies_the_dividing_ranges` holds it, and fails on the draft it replaced. +- 2026-08-27 #451 — one vocabulary per alternation, FORCED rather than chosen. `_LatinCopy` carries one `vocabulary` and `test_latin_alternations_mean_something_the_vocabulary_ships` asserts exactly one roster key per alternation, so `jr` (SUFFIX_WORDS) and `mc`/`mp` (SUFFIX_ACRONYMS) cannot share one. That constraint forces one of the three splits, and forces it only given the roster as written: `_LatinCopy.vocabulary` is a hand-supplied frozenset, so declaring `SUFFIX_WORDS | SUFFIX_ACRONYMS` would let `(jr\.?|v|mp|mc)` pass the member checks — measured. Declining to write a union vocabulary is a judgement, that a rule's alternation should name one wordlist a reader can go and check, and it is recorded here as one rather than as a constraint. The other two are forced separately and are recorded below under the declined single-rule tail: `m\.?a\.?` cannot be an alternation member at all, and the numeral alternation copies no wordlist. Four rules, three independent reasons, none of them taste. +- 2026-08-27 #451 — a ledger rule's character class must not reach a codepoint the IMPLEMENTATION does not act on, and is pinned to that. A SUBSET, not a copy: `test_the_emoji_boundary_rule_copies_the_dividing_ranges` asserts `claimed - divides == set()` and nothing in the other direction, deliberately — `_EMOJI_RANGES` has FOUR blocks and the rule copies the two astral ones, leaving the BMP pair out because no corpus name reaches them through its token anchor. Reading it as equality would instruct the next author to pre-claim the BMP half, which is the #372 shape this same section leans on. The emoji rule's first draft spanned `U+1F300-U+1FAFF` while `_tokenize.py`'s `_EMOJI_RANGES` divides on two narrower blocks: it claimed 1072 codepoints where the parser leaves the token whole, so its own prose ("an emoji inside a token divides it") was false for every one of them and a `{given, family}` diff with another cause would have classified as intended. `test_the_emoji_boundary_rule_copies_the_dividing_ranges` holds it, and fails on the draft it replaced. - 2026-08-27 #451 — `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads. Verified against all three: the ban removes the only shape that could occupy its second tier, so file order settles every tie there is and the function reorders nothing. Kept because it is what makes the ban safe to state for any reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture. Deleting it would move the guarantee from the code into a convention, which is the trade #451 was filed to undo. Recorded as a decision rather than as a consequence precisely so a later dead-code sweep finds the reasoning before the `return`. - 2026-08-27 #451 — past-tense counts stay in a ledger comment; live ones do not. "It claimed 25 until #372" is a fixed fact. "Four corpus names, and the prose fits all four" was written when the rule explained four and still read that way at fourteen, because nothing recomputes it. Where a count is owned by another comment, point at the owner rather than restating it — two copies means one gets updated and the other does not. - 2026-08-28 #452 — a rule's declared `fields` must EQUAL the union of the diffs it explains, checked by `compare.py` at the end of every run and failing it like an unexplained diff. The statement is exact rather than heuristic, which is what makes it cheap: `classify()` already requires `declared >= union` for the rule to match the names it matches, so the only possible error is the other direction, and the union is simultaneously the check and the repair. Narrowing to it cannot orphan a name, since every name a rule explains contributed to it. Measured before landing: 3 of 67 explaining rules over-declared at 1.4.0, 5 of 58 at 2.0.0, 6 of 51 at 2.1.0 — all fourteen narrowed first, so the check was silent the day it arrived. @@ -639,7 +639,7 @@ Decisions that landed: Found rather than decided, and worth as much: -- **A fields-only rule silently falsifies OTHER comments.** The 2.x ledgers' idiom "such a name should arrive UNEXPLAINED and be read once" is true there because they have no fields-only rule. In the 1.4.0 ledger it was false for anything of the catch-all's shape, and nothing said so. Measured while the rule still lived, by driving `compare.classify()` directly over the seven `_MUST_NOT_MATCH` probes of the two rules #451 ported, at diff shape `{given, family}`: ZERO arrived unexplained. Four went to the catch-all ('Mcintyre Smith Jr.', 'Los Santos', 'John Sir de Mesnil', 'Sir Smith'), two to `fix(comma-precomma-family)` ('McDonald, Ronald', 'Smith, Sir de Vaux'), and one ('Ste Marie') to the neighbouring `fix(#360) ste` rule that exists to claim it. Driven through `classify()` rather than read off a gate run, deliberately: six of the seven are in no corpus, so a run would never reach them — which is also why the promise they carried could go false unnoticed. The blast radius of the shape was never just its own heading. +- **A fields-only rule silently falsifies OTHER comments.** The 2.x ledgers' idiom "such a name should arrive UNEXPLAINED and be read once" holds there because no rule above the one making the promise claims that diff shape. A fields-only rule is the extreme way to break it, not the only way: measured while fixing #452, a `maiden` diff on `fix(#367)`'s own probe goes to `fix(#274)` in the 1.4.0 ledger with no fields-only rule anywhere. The reusable check is "does anything above me claim this shape", which is `classify()` driven directly — NOT "is there a catch-all". In the 1.4.0 ledger it was false for anything of the catch-all's shape, and nothing said so. Measured while the rule still lived, by driving `compare.classify()` directly over the seven `_MUST_NOT_MATCH` probes of the two rules #451 ported, at diff shape `{given, family}`: ZERO arrived unexplained. Four went to the catch-all ('Mcintyre Smith Jr.', 'Los Santos', 'John Sir de Mesnil', 'Sir Smith'), two to `fix(comma-precomma-family)` ('McDonald, Ronald', 'Smith, Sir de Vaux'), and one ('Ste Marie') to the neighbouring `fix(#360) ste` rule that exists to claim it. Driven through `classify()` rather than read off a gate run, deliberately: six of the seven are in no corpus, so a run would never reach them — which is also why the promise they carried could go false unnoticed. The blast radius of the shape was never just its own heading. - **Copy-paste is how a ledger comment goes false without being edited.** All three ledgers carried the same `fix(#379)` comment; four of its sentences were true only in the 1.4.0 file they were written for, including one citing `fix(suffix-routing)`, which exists in no 2.x ledger, and one saying a name "falls to the fields-only catch-all" twenty lines after the same file says no fields-only rule exists there. Corrected in `8237b54`. A cross-ledger claim needs re-deriving per file, not copying. Declined: From ee6aaa6d6bc7ff471ad8242abaaaf0f7bdbf49ec Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 29 Aug 2026 01:16:07 -0700 Subject: [PATCH 9/9] fix(differential,guards,docs): the PR review findings, measured Four reviewers on #455. Two converged on the same Critical, and five of the rest are corrections of mine that were half-applied or written without being driven through classify(). CRITICAL -- main()'s wiring had no composition test. Measured on mutants built outside the repo: deleting the roles_by_issue accumulator, deleting the report loop, or dropping `or overwide` from the return each left the whole suite green, and with the accumulator gone the check reports nothing forever while looking wired up. The file already made this call for the sibling check -- _run_main's own docstring says "every unit test proves a helper WORKS while none proves main() calls it, and in a gate the composition is the part that can go silently permissive" -- and the dormancy check has three such tests. This adds two, verified to kill all three mutants, one of them pinning exit 0 so the check cannot pass by being unconditional. CRITICAL -- four prose claims falsified by classify(), all mine: - "a maiden diff on the nee name goes to fix(#274)" is true of {family, maiden} and FALSE of {given, family, maiden}, the shape the rule would actually grow into, because fix(#274)'s own fields stop at maiden/middle/family. Same claim in decisions.md's reusable-lesson bullet, which is the one a future arc copies. Both now name the shape rather than the role, which is the bullet's own stated method. - "`middle` alongside `title` or `suffix` is the combination #452 protects here" -- measured over the 23 names the regex reaches, {middle, suffix} is absorbed on four of them. The comment now carries the per-shape breakdown. - the emoji bullet said FOUR blocks at the front and "two narrower blocks" at the back: my own correction, half-applied. - a cross-reference pointed at the wrong paragraph AND named the wrong trio, including the next rule's only name. HIGH -- _CROSS_RULE_WINNERS covered one of three ledgers and its coverage assertion was `<=`, so a ledger with no rows read as "no contest" rather than "nobody looked". Both sibling rosters use equality. The narrowings are what made that expensive: shrinking a rule's fields hands every shape it no longer admits to whatever claims it next, and measured, that moved shapes in both 2.x files. Assertion is now `==`, both ledgers have sections, and the two handovers are pinned -- with the issue strings MEASURED after a guessed one was caught by the guard itself. Also: the OVER-DECLARED report now names the ledger, because this rule's correct fields differ per baseline and a message without the file sends the reader to edit a rule that is not the broken one; the report says so when --corpus made the union partial, where the printed repair could orphan a name; the docstring records that the union is a per-RULE bound, so a diff shape no single name produced is still claimable; `if not moved` became `is None`; the roster comment's flat "FORCED" is walked back to match decisions.md; a 2.0.0 paragraph that misdescribed the one above it is fixed; and AGENTS.md's release checklist now says a skipped baseline loses the over-declaration audit too. 5349 passed; gates 229/0, 194/0, 102/0, no OVER-DECLARED. Refs #452 Co-Authored-By: Claude Opus 5 --- AGENTS.md | 5 +- docs/design/decisions.md | 4 +- tests/v2/test_differential.py | 46 ++++++++++++++ tests/v2/test_ledger_guards.py | 66 +++++++++++++++++--- tools/differential/compare.py | 55 +++++++++++++--- tools/differential/expected_since_1.4.0.toml | 52 ++++++++++----- tools/differential/expected_since_2.0.0.toml | 6 +- 7 files changed, 196 insertions(+), 38 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8060e880..2be01ec0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,7 +99,10 @@ uv run sphinx-build -b html docs dist/docs # 1. Run the differential gate at EVERY baseline that has a ledger with rules, # and confirm each exits 0. The dynamic check only runs for the ledger # matching the baseline being run, so a ledger left out here gets no -# dynamic dormancy check at all -- see tools/differential/README.md: +# dynamic dormancy check AND no over-declaration check (#452) at all +# -- and neither is hypothetical: #452 found 3, 5 and 6 stale +# declarations sitting in the three files. See +# tools/differential/README.md: # uv run python tools/differential/compare.py --baseline 1.4.0 # the v1 compat contract # uv run python tools/differential/compare.py # the previous minor # uv run python tools/differential/compare.py --baseline 2.0.0 # 2.0.0's ledger has rules too diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 8f667e11..6aae4df4 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -630,7 +630,7 @@ Decisions that landed: - 2026-08-27 #451 — a rule whose declared `fields` EXCEED every diff it explains is a defect in its own right, not untidiness. Measured on the catch-all: it declared `{given, family, suffix}`, and by the time this bundle's first three commits had given its `{given, family}` names rules of their own, nothing it still explained moved `given`. Note WHEN the excess appeared, because it is the whole mechanism — the declaration was earned when the rule was written and became excess as its explained set SHRANK under it, which is exactly `fix(#424)` at decisions.md#H1, where #410 narrowed a diff from three roles to two and the rule kept matching by superset with no run naming it. A rule does not have to widen to become over-declared; the diffs beneath it only have to leave. The two together are why #452 was filed — nothing counts what a rule actually explains, only whether it explains anything. - 2026-08-27 #451 — a ledger rule may classify a reading NOBODY WANTS, provided its `issue` says so. `fix(#342)` and `fix(#397)` carry NOT WANTED in the issue text, the cause-not-consent sentence decisions.md#P5 established for the #369 regression, and a delete-when-fixed instruction the dormancy check enforces once the fix lands. - 2026-08-27 #451 — one vocabulary per alternation, FORCED rather than chosen. `_LatinCopy` carries one `vocabulary` and `test_latin_alternations_mean_something_the_vocabulary_ships` asserts exactly one roster key per alternation, so `jr` (SUFFIX_WORDS) and `mc`/`mp` (SUFFIX_ACRONYMS) cannot share one. That constraint forces one of the three splits, and forces it only given the roster as written: `_LatinCopy.vocabulary` is a hand-supplied frozenset, so declaring `SUFFIX_WORDS | SUFFIX_ACRONYMS` would let `(jr\.?|v|mp|mc)` pass the member checks — measured. Declining to write a union vocabulary is a judgement, that a rule's alternation should name one wordlist a reader can go and check, and it is recorded here as one rather than as a constraint. The other two are forced separately and are recorded below under the declined single-rule tail: `m\.?a\.?` cannot be an alternation member at all, and the numeral alternation copies no wordlist. Four rules, three independent reasons, none of them taste. -- 2026-08-27 #451 — a ledger rule's character class must not reach a codepoint the IMPLEMENTATION does not act on, and is pinned to that. A SUBSET, not a copy: `test_the_emoji_boundary_rule_copies_the_dividing_ranges` asserts `claimed - divides == set()` and nothing in the other direction, deliberately — `_EMOJI_RANGES` has FOUR blocks and the rule copies the two astral ones, leaving the BMP pair out because no corpus name reaches them through its token anchor. Reading it as equality would instruct the next author to pre-claim the BMP half, which is the #372 shape this same section leans on. The emoji rule's first draft spanned `U+1F300-U+1FAFF` while `_tokenize.py`'s `_EMOJI_RANGES` divides on two narrower blocks: it claimed 1072 codepoints where the parser leaves the token whole, so its own prose ("an emoji inside a token divides it") was false for every one of them and a `{given, family}` diff with another cause would have classified as intended. `test_the_emoji_boundary_rule_copies_the_dividing_ranges` holds it, and fails on the draft it replaced. +- 2026-08-27 #451 — a ledger rule's character class must not reach a codepoint the IMPLEMENTATION does not act on, and is pinned to that. A SUBSET, not a copy: `test_the_emoji_boundary_rule_copies_the_dividing_ranges` asserts `claimed - divides == set()` and nothing in the other direction, deliberately — `_EMOJI_RANGES` has FOUR blocks and the rule copies the two astral ones, leaving the BMP pair out because no corpus name reaches them through its token anchor. Reading it as equality would instruct the next author to pre-claim the BMP half, which is the #372 shape this same section leans on. The emoji rule's first draft spanned `U+1F300-U+1FAFF` while the two ASTRAL blocks of `_tokenize.py`'s `_EMOJI_RANGES` are narrower than that span: it claimed 1072 codepoints where the parser leaves the token whole, so its own prose ("an emoji inside a token divides it") was false for every one of them and a `{given, family}` diff with another cause would have classified as intended. `test_the_emoji_boundary_rule_copies_the_dividing_ranges` holds it, and fails on the draft it replaced. - 2026-08-27 #451 — `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads. Verified against all three: the ban removes the only shape that could occupy its second tier, so file order settles every tie there is and the function reorders nothing. Kept because it is what makes the ban safe to state for any reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture. Deleting it would move the guarantee from the code into a convention, which is the trade #451 was filed to undo. Recorded as a decision rather than as a consequence precisely so a later dead-code sweep finds the reasoning before the `return`. - 2026-08-27 #451 — past-tense counts stay in a ledger comment; live ones do not. "It claimed 25 until #372" is a fixed fact. "Four corpus names, and the prose fits all four" was written when the rule explained four and still read that way at fourteen, because nothing recomputes it. Where a count is owned by another comment, point at the owner rather than restating it — two copies means one gets updated and the other does not. - 2026-08-28 #452 — a rule's declared `fields` must EQUAL the union of the diffs it explains, checked by `compare.py` at the end of every run and failing it like an unexplained diff. The statement is exact rather than heuristic, which is what makes it cheap: `classify()` already requires `declared >= union` for the rule to match the names it matches, so the only possible error is the other direction, and the union is simultaneously the check and the repair. Narrowing to it cannot orphan a name, since every name a rule explains contributed to it. Measured before landing: 3 of 67 explaining rules over-declared at 1.4.0, 5 of 58 at 2.0.0, 6 of 51 at 2.1.0 — all fourteen narrowed first, so the check was silent the day it arrived. @@ -639,7 +639,7 @@ Decisions that landed: Found rather than decided, and worth as much: -- **A fields-only rule silently falsifies OTHER comments.** The 2.x ledgers' idiom "such a name should arrive UNEXPLAINED and be read once" holds there because no rule above the one making the promise claims that diff shape. A fields-only rule is the extreme way to break it, not the only way: measured while fixing #452, a `maiden` diff on `fix(#367)`'s own probe goes to `fix(#274)` in the 1.4.0 ledger with no fields-only rule anywhere. The reusable check is "does anything above me claim this shape", which is `classify()` driven directly — NOT "is there a catch-all". In the 1.4.0 ledger it was false for anything of the catch-all's shape, and nothing said so. Measured while the rule still lived, by driving `compare.classify()` directly over the seven `_MUST_NOT_MATCH` probes of the two rules #451 ported, at diff shape `{given, family}`: ZERO arrived unexplained. Four went to the catch-all ('Mcintyre Smith Jr.', 'Los Santos', 'John Sir de Mesnil', 'Sir Smith'), two to `fix(comma-precomma-family)` ('McDonald, Ronald', 'Smith, Sir de Vaux'), and one ('Ste Marie') to the neighbouring `fix(#360) ste` rule that exists to claim it. Driven through `classify()` rather than read off a gate run, deliberately: six of the seven are in no corpus, so a run would never reach them — which is also why the promise they carried could go false unnoticed. The blast radius of the shape was never just its own heading. +- **A fields-only rule silently falsifies OTHER comments.** The 2.x ledgers' idiom "such a name should arrive UNEXPLAINED and be read once" holds there because no rule above the one making the promise claims that diff shape. A fields-only rule is the extreme way to break it, not the only way: measured while fixing #452, a `{family, maiden}` diff on `fix(#367)`'s own probe goes to `fix(#274)` in the 1.4.0 ledger, which has no fields-only rule anywhere. Note the shape, not the role: `{given, family, maiden}` on the same probe is claimed by nothing, because `fix(#274)`'s `fields` stop at maiden/middle/family. That is the point — the question is only ever answerable per shape. The reusable check is "does anything above me claim this shape", which is `classify()` driven directly — NOT "is there a catch-all". In the 1.4.0 ledger it was false for anything of the catch-all's shape, and nothing said so. Measured while the rule still lived, by driving `compare.classify()` directly over the seven `_MUST_NOT_MATCH` probes of the two rules #451 ported, at diff shape `{given, family}`: ZERO arrived unexplained. Four went to the catch-all ('Mcintyre Smith Jr.', 'Los Santos', 'John Sir de Mesnil', 'Sir Smith'), two to `fix(comma-precomma-family)` ('McDonald, Ronald', 'Smith, Sir de Vaux'), and one ('Ste Marie') to the neighbouring `fix(#360) ste` rule that exists to claim it. Driven through `classify()` rather than read off a gate run, deliberately: six of the seven are in no corpus, so a run would never reach them — which is also why the promise they carried could go false unnoticed. The blast radius of the shape was never just its own heading. - **Copy-paste is how a ledger comment goes false without being edited.** All three ledgers carried the same `fix(#379)` comment; four of its sentences were true only in the 1.4.0 file they were written for, including one citing `fix(suffix-routing)`, which exists in no 2.x ledger, and one saying a name "falls to the fields-only catch-all" twenty lines after the same file says no fields-only rule exists there. Corrected in `8237b54`. A cross-ledger claim needs re-deriving per file, not copying. Declined: diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index 446566ad..cc2fa9f3 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -585,6 +585,52 @@ def test_main_exits_1_and_names_a_rule_that_explained_nothing( assert "unexplained: 0" in out +def test_main_exits_1_and_names_an_over_declared_rule( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The #452 gate. `wide` explains the only diff here and declares + two roles it never moves -- the run must say so and fail, even + though every diff IS explained and no rule is idle. + + Nothing else pins this, which is the same gap _run_main's own + docstring names for the dormancy check: five unit tests prove + over_declared_rules WORKS and none proved main() calls it. + Measured on mutants built outside the repo -- deleting the + `roles_by_issue` accumulator, deleting this report loop, or + dropping `or overwide` from main's return each leaves the whole + suite green (#455 review). + """ + code, out = _run_main( + tmp_path, monkeypatch, + '[[change]]\nissue = "wide"\nname_regex = "Smith"\n' + 'fields = ["family", "given", "suffix"]\n', _DIFFERS) + assert code == 1 + # the ledger is named before the issue: this rule's correct + # `fields` differ per baseline, so the file is part of the finding + assert "OVER-DECLARED" in out and "'wide'" in out + assert out.count(".toml: 'wide'") == 1 + assert "['given', 'suffix']" in out # the roles nothing moves + assert "['family']" in out # the repair + # the diff itself was explained; this failure is only about the rule + assert "unexplained: 0" in out + + +def test_main_accepts_a_rule_declaring_exactly_what_it_moves( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The other direction, so the #452 check cannot pass by being + unconditional. `exact` declares the one role the one diff moves, + and the run is silent and exits 0. + + Its partner above would still pass if over_declared_rules reported + every rule; this is what makes that impossible. + """ + code, out = _run_main( + tmp_path, monkeypatch, + '[[change]]\nissue = "exact"\nname_regex = "Smith"\n' + 'fields = ["family"]\n', _DIFFERS) + assert code == 0 + assert "OVER-DECLARED" not in out + + def test_main_only_feeds_diffing_names_to_the_dormancy_check( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """If a non-diffing name reached `diffing`, a rule matching only diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index b7bd31db..7ebe2af3 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1039,12 +1039,19 @@ class _LatinCopy(NamedTuple): # partial for fix(#379)'s reason, SUFFIX_ACRONYMS running to # hundreds of entries that nobody writes after a bare name word. # - # This is the roster entry that FORCED #451's split into four rules - # rather than one: the sibling rule's `jr` is a SUFFIX_WORDS entry, - # _LatinCopy carries exactly one `vocabulary`, and the test below - # asserts exactly one roster key per alternation -- so an - # alternation spanning both vocabularies could not be pinned - # against either, and had to become two rules. + # This is the roster entry behind one of #451's three splits, and + # it forces that split only GIVEN THE ROSTER AS WRITTEN: the + # sibling rule's `jr` is a SUFFIX_WORDS entry, _LatinCopy carries + # exactly one `vocabulary`, and the test below asserts exactly one + # roster key per alternation -- so an alternation spanning both + # vocabularies cannot be pinned against either AS DECLARED HERE. + # `vocabulary` is a hand-supplied frozenset, though: declaring + # SUFFIX_WORDS | SUFFIX_ACRONYMS would let `(jr\.?|v|mp|mc)` pass + # the member checks, measured. Declining to write a union is a + # judgement -- an alternation should name one wordlist a reader can + # go and check -- and decisions.md records it as one rather than as + # a constraint. An earlier version of this comment said FORCED + # flatly and was walked back (#452 review). "fix(suffix-routing) a two-token name ending in a credential acronym keeps it in `suffix`": _LatinCopy(vocabulary=SUFFIX_ACRONYMS, covers=frozenset({"mc", "mp"})), @@ -2069,6 +2076,39 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: ("김민준씨 Jr.", ("family", "given", "suffix")): "fix(cjk-glued-honorific-peel) glued honorific peels into suffix", }, + # The two 2.x ledgers had NO section here until #452, and the + # coverage assertion below was `<=`, so their absence read as "no + # contest to pin" rather than "nobody looked". The #452 narrowings + # are what made that expensive: shrinking a rule's `fields` hands + # every shape it no longer admits to whatever claims it next, and + # measured across the fourteen, that moved shapes in both files. + # Neither _CORPUS_CLAIMS nor the gate's totals can see a handover -- + # reach is regex-only and the total is per-corpus -- so these rows + # are the only thing that would. + "expected_since_2.0.0.toml": { + # fix(#296) lost `family` and `given`; {family} on this name is + # one of the shapes it stopped admitting, and fix(#379) takes it. + # The right home -- a tussenvoegsel attaching to the family is + # exactly what that rule is about -- which is not the point: the + # point is that a later edit sends it somewhere else in silence. + ("Nguyen, Van", ("family",)): + "fix(#379) a tussenvoegsel after a family comma attaches to the family", + # fix(#412) lost `middle`; this shape went to fix(#445), which + # sits BEHIND it in file order, so the handover was decided by + # the narrowing rather than by position. + ("Jane née and Jones Smith", ("family", "maiden", "middle")): + "fix(#445) the lone name word beside a marker a connective join no longer absorbs", + }, + "expected_since_2.1.0.toml": { + # The same two handovers, measured against this baseline's own + # run rather than copied from the 2.0.0 rows -- the ledgers + # differ, and #452's own lesson is that a claim true in one file + # is not thereby true in its siblings. + ("Nguyen, Van", ("family",)): + "fix(#379) a tussenvoegsel after a family comma attaches to the family", + ("Jane née and Jones Smith", ("family", "maiden", "middle")): + "fix(#445) the lone name word beside a marker a connective join no longer absorbs", + }, } @@ -2107,9 +2147,17 @@ def test_the_recorded_rule_still_wins_each_contested_name() -> None: f"describe is #372, and it stays green everywhere else") checked += 1 assert checked, "no contested name was checked, so this pin is vacuous" - assert set(_CROSS_RULE_WINNERS) <= {led.name for led in _LEDGERS}, ( - f"_CROSS_RULE_WINNERS names ledgers that do not exist: " - f"{sorted(set(_CROSS_RULE_WINNERS) - {L.name for L in _LEDGERS})}") + assert set(_CROSS_RULE_WINNERS) == {led.name for led in _LEDGERS}, ( + f"_CROSS_RULE_WINNERS must name every ledger on disk, with an " + f"explicit empty mapping for one that genuinely has no contest. " + f"Missing: {sorted({L.name for L in _LEDGERS} - set(_CROSS_RULE_WINNERS))}; " + f"unknown: {sorted(set(_CROSS_RULE_WINNERS) - {L.name for L in _LEDGERS})}. " + f"This was `<=` until #452, which made a ledger with no rows " + f"indistinguishable from one needing none -- and the #452 " + f"narrowings moved shapes between rules in the two 2.x ledgers " + f"that had no section at all. Both sibling rosters " + f"(_CORPUS_CLAIMS, _SPAN_BEARING_RULES) use equality; this one " + f"was the odd one out.") class _Excluded(NamedTuple): diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 2dd21b17..15a9d8fc 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -815,12 +815,32 @@ def over_declared_rules( The union is also the repair, and narrowing to it cannot orphan a name: every name the rule explains contributed to it. - Two rules are skipped, neither as an exemption. A rule declaring - `dormant` explains nothing by declaration, so there is no union to - compare against, and dormant_rules already checks that claim in both - directions. A rule with no `fields` declares no roles and so has - nothing to over-declare; one with `fields` and no `name_regex` - cannot exist since #451. + Three rules are skipped, none as an exemption. A rule declaring + `dormant` is dormant_rules' finding in BOTH directions -- including + when it has explained a diff, which that check reports as NO LONGER + DORMANT: one defect with one remedy (remove the key), where + reporting it here too would demand a second, contradictory one. + (Do not say "a dormant rule has no union to compare against" -- it + can have one, and the test that pins this skip uses exactly that + input, because the empty-union input cannot discriminate.) A rule + with no `fields` declares no roles and so has nothing to + over-declare; one with `fields` and no `name_regex` cannot exist + since #451. And a rule that explained nothing is dormant_rules' + too, which is the third `continue` below. + + What this does NOT bound is a diff shape no single name produced. + A rule declaring {family, suffix} where one name moves `family` + and another moves `suffix` passes -- the union is both -- while + still standing ready to claim a name diffing the two together. The + union is a per-RULE bound, not a per-name one, and narrowing + further would orphan one of the two. The #452 hazard survives in + that reduced form by choice. + + One caveat for a partial run: under `--corpus` the union is + computed over the names actually compared, so a subset run can + report a rule that is correctly declared for the full gate, with a + `observed` repair that would orphan a name on the next full run. + The report says so. Pure, like dormant_rules: it needs only values main() already derives, so it is testable without a corpus or a baseline worker. @@ -831,8 +851,13 @@ def over_declared_rules( if "dormant" in rule or not isinstance(declared, list): continue moved = roles_by_issue.get(str(rule["issue"])) - if not moved: - continue # explains nothing -- dormant_rules owns it + if moved is None: + # explains nothing -- dormant_rules owns that finding. + # `is None` rather than falsy: an empty union cannot reach + # this dict today (main() skips a name with no diff), and if + # that ever changes an empty one should be REPORTED rather + # than silently skipped. + continue unused = tuple(sorted(set(declared) - moved)) if unused: found.append(_OverDeclared( @@ -997,13 +1022,23 @@ def main() -> int: print() overwide = over_declared_rules(rules, roles_by_issue) for wide in overwide: - print(f"OVER-DECLARED {wide.issue!r}\n " + # The ledger is NAMED, unlike the two blocks above, because this + # rule's correct `fields` differ per baseline -- fix(#296) is + # exactly exercised at 1.4.0 and over-declared at both 2.x -- so + # a message without the file sends the reader to edit a rule + # that is not the broken one, which is validate_rules' own + # stated reason for carrying `ledger`. + print(f"OVER-DECLARED {ledger.name}: {wide.issue!r}\n " f"declares {list(wide.unused)}, which no diff it explains " f"moves; every one fits {list(wide.observed)}. Narrow " f"`fields` to that. classify() matches by SUBSET, so the " f"excess is not inert -- it lets this rule keep claiming a " f"name whose diff shrinks out of the extra role, with " - f"nothing to say so (#452)") + f"nothing to say so (#452)." + + (" NOTE: this run used --corpus, so the union above is " + "over a SUBSET and the repair may orphan a name the " + "full gate compares -- confirm before narrowing." + if args.corpus else "")) if overwide: print() if unexplained: diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 82f89c38..427a7dbe 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1015,13 +1015,30 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip # actually claims -- now falls out of this rule instead of being # silently absorbed, and should arrive UNEXPLAINED and be read once # rather than get a widened rule pre-built for it (#452). That -# promise does not cover every shape, though: a diff that moves ONLY -# {given, middle, family} -- the order flip with no comma routing at -# all -- still classifies above, on fix(#271/#272/#298), whose bare -# CJK-codepoint regex sits ahead of this one and whose fields were -# never narrowed. `middle` alongside `title` or `suffix` is the -# combination #452 protects here; `middle` alone is -# fix(#271/#272/#298)'s to explain, as it always was. +# promise does not cover every shape, and the exceptions are per +# SHAPE and per NAME rather than per role -- measured with classify() +# over the 23 names this regex reaches, not reasoned from which role +# grew: +# +# {given, middle, family} all 23 -> fix(#271/#272/#298), the order +# flip with no comma routing, whose +# bare CJK regex sits ahead of this +# rule and whose fields were never +# narrowed +# {middle, suffix} 4 of 23 -> '田中さん, Dr.' to fix(#296), and +# '田中さん, 様.', '김민준, 씨', +# '김민준, 씨.' to +# fix(cjk-honorific-suffix) +# {middle, title} none -> unclassified +# {middle, suffix, title} none -> unclassified +# +# So what #452 buys here is narrower than "middle alongside title or +# suffix": it is the shapes in the last two rows, plus {middle, suffix} +# on the nineteen names the honorific rules do not reach. An earlier +# draft of this paragraph claimed the whole `title`-or-`suffix` +# combination and was falsified by driving classify() over the reach -- +# the method decisions.md#differential-ledger names as the only one +# that answers this question. name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "family", "title", "suffix"] @@ -1514,17 +1531,24 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # `fields` is what keeps those wider shapes out, NOT the regex -- # measured, `^(mr|dr|sir)\.?\s+van\b` has no end anchor and does # reach 'Mr. Van Johnson nee Brown' and 'Mr. Van Johnson Middle'; -# only 'Dr. St John Smith' is outside it. The diff on the three names -# this rule actually claims (given in the next paragraph) is -# {given, family} alone, and `fields` names those two roles and no +# only 'Dr. St John Smith' is outside it. The three names this rule +# actually claims are 'Mr. Van Nguyen', 'Dr. Van Johnson' and +# 'Sir Van Johnson' -- NOT the trio two paragraphs up, which includes +# 'Jr. Van Johnson', the next rule's one name -- and each diffs +# {given, family} alone. `fields` names those two roles and no # others, so `title`, `maiden` and `middle` are left out on purpose. # classify() admits by subset, so leaving a role out is not passive: # a diff that grows to include one of them falls out of THIS rule # instead of being silently absorbed. Where it lands then is a -# separate question -- a `maiden` diff on the nee name above goes to -# fix(#274), which is its right home; a `middle` or `title` one is -# claimed by nothing and arrives unclassified, to be read once rather -# than have a widened rule pre-built for it (#452). +# separate question, and it depends on the SHAPE rather than the role. +# Measured on the nee name above: {family, maiden} goes to fix(#274), +# which is its right home, but {given, family, maiden} -- the shape +# this rule would actually grow into, since it already moves `given` +# -- is claimed by nothing, because fix(#274)'s own `fields` stop at +# maiden/middle/family and its comment says why. A `middle` or `title` +# growth is likewise claimed by nothing. So the honest statement is +# per shape, not per role: drive classify() rather than reasoning from +# which role moved (#452). # # This rule has to exist even though the run already exited 0 without # it. The diff was being classified by fix(suffix-routing), a diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index b3af2e1d..3094c535 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -804,8 +804,10 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # fields-only rule exists here, so this file's copy is the plain # classification. # -# `middle` is OUT too, the same excess the paragraphs above already -# argue for `title` and `maiden`: the diff on the three names this rule +# The paragraph above already argues `middle` out alongside `title` +# and `maiden`; what follows is the measurement behind it, kept +# because the names are worth naming: the diff on the three names this +# rule # actually claims here -- 'Mr. Van Nguyen', 'Dr. Van Johnson', 'Sir Van # Johnson' -- is {given, family} alone at this baseline, checked # directly against the released 2.0.0 and the tree (none of the three