diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index bcfbeda..31fb249 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,8 @@ Notable changes to the `phileas-redact` package, most recent first. ## Unreleased +* The `phoneNumber` filter now finds a number written with a bare `1` country code and an area code beginning with 1, such as `1-112-997-5633`, `1 112 997 5633`, or `1-109-299-5128 x512`, when a configured `region` uses country code 1 (such as `US`, the default). libphonenumber read the `1` as the country code and the area code's `1` as a national prefix, leaving too few digits, so these were missed while `+1 112-997-5633` was found. A number found before keeps its span and confidence, and a number with no separators, such as `11129975633`, is still not found. Same as the Java library (philterd/phileas#458). + * The `ssn` filter no longer detects the `NN-NNNNNNN` form. That shape is an Employer Identification Number, not a Social Security Number, and is already covered by the `ein` filter. A policy that enables only `ssn` therefore no longer detects values such as `12-3456789`; enable `ein` to detect them. SSN forms (`NNN-NN-NNNN`, `NNN NN NNNN`, and nine digits with no hyphen) are unchanged. ## Version 1.1.0 diff --git a/docs/filters.md b/docs/filters.md index dea75dc..e463ea2 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -147,6 +147,8 @@ Only the hyphenated form is matched. A bare nine-digit run is left to the `ssn` Detects phone numbers with [libphonenumber](https://github.com/daviddrysdale/python-phonenumbers). A `+`-prefixed international number is detected whatever its country (`+44 20 7946 0958`); a number without one is read as US (`(555) 867-5309`, `555-867-5309`, `555.867.5309`). +A number is detected when libphonenumber considers it possible, which is mostly a matter of its length. When a configured region uses country code 1 (such as `US`), a bare `1` country code followed by a separator is also read as `+1`, so `1-112-997-5633`, `1 112 997 5633` and `1.112.997.5633` are found like `+1 112-997-5633`. A number with no separators, such as `11129975633`, is not found. + Span confidence reflects how the number is written: `0.95` for plain NANP formatting, `0.75` for longer forms, `0.60` otherwise. A `confidence` condition sees those values. ```python diff --git a/phileas/filters/phone_number_filter.py b/phileas/filters/phone_number_filter.py index 742d9cd..7e8974f 100644 --- a/phileas/filters/phone_number_filter.py +++ b/phileas/filters/phone_number_filter.py @@ -24,7 +24,7 @@ import re from typing import List -from phonenumbers import Leniency, PhoneNumberMatcher, is_valid_number +from phonenumbers import Leniency, PhoneNumberMatcher, country_code_for_region, is_valid_number from phileas.models.span import Span from .base import BaseFilter, FilterType @@ -35,6 +35,12 @@ # A number written the plain NANP way. Matching it is what earns full confidence. _NANP_FORMAT = re.compile(r"(\+\d{1,2}\s)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}") +# A bare "1" country code followed by a separator and a phone-formatted number, such +# as "1-112-997-5633". Without a "+", libphonenumber strips the "1" as the country +# code and then an area code's leading "1" as a national prefix, leaving too few +# digits for a possible number, so the match is dropped. +_BARE_COUNTRY_CODE = re.compile(r"(? float: if _NANP_FORMAT.fullmatch(raw): @@ -77,22 +83,46 @@ class PhoneNumberFilter(BaseFilter): def __init__(self, config=None): super().__init__(FilterType.PHONE_NUMBER, config) self.regions = _regions(self.config) + # The first configured region that uses country code 1, or None when none does. + self.north_american_region = next( + (r for r in self.regions if country_code_for_region(r) == 1), None + ) def detect(self, text: str, context: str = "default") -> List[Span]: matches: list = [] for region in self.regions: matches.extend(PhoneNumberMatcher(text, region, leniency=Leniency.POSSIBLE)) - return [ - Span( - character_start=match.start, - character_end=match.end, - filter_type=self.filter_type, - context=context, - confidence=_confidence(match.raw_string), - text=match.raw_string, - replacement="", - ignored=False, + # Also scan, once, a copy in which a bare "1" country code becomes "+1", so + # "1-112-997-5633" is found like "+1 112-997-5633". "1-" and "+1" have the same + # length, so offsets in the copy are offsets in the text. A match from the copy + # is kept only where the text itself had none, so a number already found, such + # as "112-997-5633" in "Item 1 112-997-5633", keeps its span and confidence. + if self.north_american_region: + rewritten = _BARE_COUNTRY_CODE.sub("+1", text) + if rewritten != text: + found = list(matches) + for match in PhoneNumberMatcher( + rewritten, self.north_american_region, leniency=Leniency.POSSIBLE + ): + if not any(match.start < m.end and m.start < match.end for m in found): + matches.append(match) + + spans = [] + for match in _dedupe(matches): + # Taken from the text rather than raw_string, which is the rewritten copy + # for a match found there. + raw = text[match.start:match.end] + spans.append( + Span( + character_start=match.start, + character_end=match.end, + filter_type=self.filter_type, + context=context, + confidence=_confidence(raw), + text=raw, + replacement="", + ignored=False, + ) ) - for match in _dedupe(matches) - ] + return spans diff --git a/tests/test_phone_number_detection.py b/tests/test_phone_number_detection.py index 5381bf0..78319f0 100644 --- a/tests/test_phone_number_detection.py +++ b/tests/test_phone_number_detection.py @@ -354,3 +354,46 @@ def test_repeated_regions_are_scanned_once(self): def test_order_is_preserved(self): assert PhoneNumberFilter({"region": ["GB", "US", "GB"]}).regions == ["GB", "US"] + + +class TestBareCountryCode: + """A bare "1" country code before an area code beginning with 1 (issue #107).""" + + @pytest.mark.parametrize( + "number", + ["1-112-997-5633", "1 112 997 5633", "1.112.997.5633", "1-109-299-5128 x512"], + ) + def test_whole_number_detected(self, number): + assert _texts(f"call {number} now") == [number] + + def test_offsets_and_text_come_from_the_input(self): + text = "call 1-112-997-5633 now" + (span,) = _spans(text) + assert text[span.character_start:span.character_end] == span.text == "1-112-997-5633" + + def test_not_rewritten_without_a_country_code_1_region(self): + assert _texts("call 1-112-997-5633 now", {"region": "GB"}) == [] + + @pytest.mark.parametrize( + "text,expected", + [ + ("Item 1 112-997-5633", "112-997-5633"), + ("call 1 (112) 997-5633", "(112) 997-5633"), + ], + ) + def test_number_found_without_the_rewrite_keeps_its_span(self, text, expected): + (span,) = _spans(text) + (alone,) = _spans(expected) + assert span.text == expected + assert span.confidence == alone.confidence + + @pytest.mark.parametrize( + "text,found", + [ + ("call +1 112-997-5633 now", ["+1 112-997-5633"]), + ("call 1-212-997-5633 now", ["1-212-997-5633"]), + ("call 11129975633 now", []), + ], + ) + def test_pinned_libphonenumber_behaviour(self, text, found): + assert _texts(text) == found