Find phone numbers with a bare 1 country code and an area code beginning with 1

RELEASE_NOTES.md

@@ -4,6 +4,8 @@ Notable changes to the `phileas-redact` package, most recent first.
 
 ## Unreleased
 
+* The `phoneNumber` filter now finds a number written with a bare `1` country code and an area code beginning with 1, such as `1-112-997-5633`, `1 112 997 5633`, or `1-109-299-5128 x512`, when a configured `region` uses country code 1 (such as `US`, the default). libphonenumber read the `1` as the country code and the area code's `1` as a national prefix, leaving too few digits, so these were missed while `+1 112-997-5633` was found. A number found before keeps its span and confidence, and a number with no separators, such as `11129975633`, is still not found. Same as the Java library (philterd/phileas#458).
+
 * The `ssn` filter no longer detects the `NN-NNNNNNN` form. That shape is an Employer Identification Number, not a Social Security Number, and is already covered by the `ein` filter. A policy that enables only `ssn` therefore no longer detects values such as `12-3456789`; enable `ein` to detect them. SSN forms (`NNN-NN-NNNN`, `NNN NN NNNN`, and nine digits with no hyphen) are unchanged.
 
 ## Version 1.1.0

docs/filters.md

@@ -147,6 +147,8 @@ Only the hyphenated form is matched. A bare nine-digit run is left to the `ssn`
 
 Detects phone numbers with [libphonenumber](https://github.com/daviddrysdale/python-phonenumbers). A `+`-prefixed international number is detected whatever its country (`+44 20 7946 0958`); a number without one is read as US (`(555) 867-5309`, `555-867-5309`, `555.867.5309`).
 
+A number is detected when libphonenumber considers it possible, which is mostly a matter of its length. When a configured region uses country code 1 (such as `US`), a bare `1` country code followed by a separator is also read as `+1`, so `1-112-997-5633`, `1 112 997 5633` and `1.112.997.5633` are found like `+1 112-997-5633`. A number with no separators, such as `11129975633`, is not found.
+
 Span confidence reflects how the number is written: `0.95` for plain NANP formatting, `0.75` for longer forms, `0.60` otherwise. A `confidence` condition sees those values.
 
 ```python

phileas/filters/phone_number_filter.py

@@ -24,7 +24,7 @@
 import re
 from typing import List
 
-from phonenumbers import Leniency, PhoneNumberMatcher, is_valid_number
+from phonenumbers import Leniency, PhoneNumberMatcher, country_code_for_region, is_valid_number
 
 from phileas.models.span import Span
 from .base import BaseFilter, FilterType
@@ -35,6 +35,12 @@
 # A number written the plain NANP way. Matching it is what earns full confidence.
 _NANP_FORMAT = re.compile(r"(\+\d{1,2}\s)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}")
 
+# A bare "1" country code followed by a separator and a phone-formatted number, such
+# as "1-112-997-5633". Without a "+", libphonenumber strips the "1" as the country
+# code and then an area code's leading "1" as a national prefix, leaving too few
+# digits for a possible number, so the match is dropped.
+_BARE_COUNTRY_CODE = re.compile(r"(?<![\d+])1[-. ](?=\(?\d{3}\)?[-. ]\d{3}[-. ]\d{4})")
+
 
 def _confidence(raw: str) -> float:
     if _NANP_FORMAT.fullmatch(raw):
@@ -77,22 +83,46 @@ class PhoneNumberFilter(BaseFilter):
     def __init__(self, config=None):
         super().__init__(FilterType.PHONE_NUMBER, config)
         self.regions = _regions(self.config)
+        # The first configured region that uses country code 1, or None when none does.
+        self.north_american_region = next(
+            (r for r in self.regions if country_code_for_region(r) == 1), None
+        )
 
     def detect(self, text: str, context: str = "default") -> List[Span]:
         matches: list = []
         for region in self.regions:
             matches.extend(PhoneNumberMatcher(text, region, leniency=Leniency.POSSIBLE))
 
-        return [
-            Span(
-                character_start=match.start,
-                character_end=match.end,
-                filter_type=self.filter_type,
-                context=context,
-                confidence=_confidence(match.raw_string),
-                text=match.raw_string,
-                replacement="",
-                ignored=False,
+        # Also scan, once, a copy in which a bare "1" country code becomes "+1", so
+        # "1-112-997-5633" is found like "+1 112-997-5633". "1-" and "+1" have the same
+        # length, so offsets in the copy are offsets in the text. A match from the copy
+        # is kept only where the text itself had none, so a number already found, such
+        # as "112-997-5633" in "Item 1 112-997-5633", keeps its span and confidence.
+        if self.north_american_region:
+            rewritten = _BARE_COUNTRY_CODE.sub("+1", text)
+            if rewritten != text:
+                found = list(matches)
+                for match in PhoneNumberMatcher(
+                    rewritten, self.north_american_region, leniency=Leniency.POSSIBLE
+                ):
+                    if not any(match.start < m.end and m.start < match.end for m in found):
+                        matches.append(match)
+
+        spans = []
+        for match in _dedupe(matches):
+            # Taken from the text rather than raw_string, which is the rewritten copy
+            # for a match found there.
+            raw = text[match.start:match.end]
+            spans.append(
+                Span(
+                    character_start=match.start,
+                    character_end=match.end,
+                    filter_type=self.filter_type,
+                    context=context,
+                    confidence=_confidence(raw),
+                    text=raw,
+                    replacement="",
+                    ignored=False,
+                )
             )
-            for match in _dedupe(matches)
-        ]
+        return spans

tests/test_phone_number_detection.py

@@ -354,3 +354,46 @@ def test_repeated_regions_are_scanned_once(self):
 
     def test_order_is_preserved(self):
         assert PhoneNumberFilter({"region": ["GB", "US", "GB"]}).regions == ["GB", "US"]
+
+
+class TestBareCountryCode:
+    """A bare "1" country code before an area code beginning with 1 (issue #107)."""
+
+    @pytest.mark.parametrize(
+        "number",
+        ["1-112-997-5633", "1 112 997 5633", "1.112.997.5633", "1-109-299-5128 x512"],
+    )
+    def test_whole_number_detected(self, number):
+        assert _texts(f"call {number} now") == [number]
+
+    def test_offsets_and_text_come_from_the_input(self):
+        text = "call 1-112-997-5633 now"
+        (span,) = _spans(text)
+        assert text[span.character_start:span.character_end] == span.text == "1-112-997-5633"
+
+    def test_not_rewritten_without_a_country_code_1_region(self):
+        assert _texts("call 1-112-997-5633 now", {"region": "GB"}) == []
+
+    @pytest.mark.parametrize(
+        "text,expected",
+        [
+            ("Item 1 112-997-5633", "112-997-5633"),
+            ("call 1 (112) 997-5633", "(112) 997-5633"),
+        ],
+    )
+    def test_number_found_without_the_rewrite_keeps_its_span(self, text, expected):
+        (span,) = _spans(text)
+        (alone,) = _spans(expected)
+        assert span.text == expected
+        assert span.confidence == alone.confidence
+
+    @pytest.mark.parametrize(
+        "text,found",
+        [
+            ("call +1 112-997-5633 now", ["+1 112-997-5633"]),
+            ("call 1-212-997-5633 now", ["1-212-997-5633"]),
+            ("call 11129975633 now", []),
+        ],
+    )
+    def test_pinned_libphonenumber_behaviour(self, text, found):
+        assert _texts(text) == found