From 23e4cdf0e2787463bfb7c69210af591b6e3a0014 Mon Sep 17 00:00:00 2001
From: jwilson232 <joe.wilson@opendataservices.coop>
Date: Thu, 3 Sep 2026 15:20:38 +0100
Subject: [PATCH] lib360dataquality: bump libcove for unique id check

bumps libcove to 0.32.1 to avoid validator brute force comparison that
was causing issues in the pipeline when a large amount of grants had
missing identifiers.

this requires json schema to be upgraded to 4.18 as well as a patch to
force jsonschema to use our validator due to https://github.com/python-jsonschema/jsonschema/issues/994
---
 360-ds                                     |  1 -
 cove/cove_360/tests/tests.py               | 28 +++++++++++++++++++++-
 lib360dataquality/cove/threesixtygiving.py |  5 ++++
 requirements_cove.txt                      |  4 ++--
 requirements_cove_dev.txt                  |  4 ++--
 requirements_cove_dokku.txt                |  4 ++--
 setup.py                                   |  2 +-
 7 files changed, 39 insertions(+), 9 deletions(-)
 delete mode 160000 360-ds

diff --git a/360-ds b/360-ds
deleted file mode 160000
index bd34fe8..0000000
--- a/360-ds
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit bd34fe8ff6b2aed37e70ed185d1f402683bf3cce
diff --git a/cove/cove_360/tests/tests.py b/cove/cove_360/tests/tests.py
index a171525..b5fd869 100644
--- a/cove/cove_360/tests/tests.py
+++ b/cove/cove_360/tests/tests.py
@@ -9,7 +9,7 @@
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import UploadedFile
 
-from lib360dataquality.cove.threesixtygiving import get_grants_aggregates, run_extra_checks, extend_numbers, spreadsheet_style_errors_table, TEST_CLASSES, group_validation_errors
+from lib360dataquality.cove.threesixtygiving import get_grants_aggregates, run_extra_checks, extend_numbers, spreadsheet_style_errors_table, validator, TEST_CLASSES, group_validation_errors
 from lib360dataquality.additional_test import TestCategories, TestImportance
 
 # Source is cove_360/fixtures/fundingproviders-grants_fixed_2_grants.json
@@ -990,6 +990,32 @@ def test_extend_numbers():
     assert list(extend_numbers([4, 5, 7, 2001])) == [3, 4, 5, 6, 7, 8, 2000, 2001, 2002]
 
 
+def test_unique_ids():
+    schema = {
+        "type": "object",
+        "properties": {
+            "grants": {
+                "type": "array",
+                "items": {"type": "object"},
+                "uniqueItems": True,
+            }
+        },
+    }
+    v = validator(schema)
+
+    errors = list(v.iter_errors({"grants": [{"id": "1"}, {"id": "2"}, {"id": "1"}]}))
+    assert [(e.message, e.error_id) for e in errors] == [("Non-unique id values", "uniqueItems_with_id")]
+
+    errors = list(v.iter_errors({"grants": [{"title": "a"}, {"title": "b"}]}))
+    assert errors == []
+
+    errors = list(v.iter_errors({"grants": [{"title": "a"}, {"title": "a"}]}))
+    assert [(e.message, e.error_id) for e in errors] == [("Array has non-unique elements", "uniqueItems_no_ids")]
+
+    errors = list(v.iter_errors({"grants": [{"id": "1"}, {"id": "2"}]}))
+    assert errors == []
+
+
 def ex(value):
     ''' Shorthand for value metadata with type `example`.'''
     return {'type': 'example', 'value': value}
diff --git a/lib360dataquality/cove/threesixtygiving.py b/lib360dataquality/cove/threesixtygiving.py
index 1768dab..e009568 100644
--- a/lib360dataquality/cove/threesixtygiving.py
+++ b/lib360dataquality/cove/threesixtygiving.py
@@ -9,6 +9,7 @@
 from decimal import Decimal
 import logging
 
+import jsonschema
 import libcove.lib.common as libcove_common
 import libcove.lib.tools as tools
 import openpyxl
@@ -172,6 +173,10 @@ def oneOf_draft4(validator, oneOf, instance, schema):
 
 validator.VALIDATORS["oneOf"] = oneOf_draft4
 
+# Force jsonschema to use our validator
+# https://github.com/python-jsonschema/jsonschema/issues/994
+jsonschema.validators._META_SCHEMAS["https://json-schema.org/draft/2020-12/schema"] = validator
+
 
 @tools.ignore_errors
 def get_grants_aggregates(json_data):
diff --git a/requirements_cove.txt b/requirements_cove.txt
index 9dfefb1..67d2e1f 100644
--- a/requirements_cove.txt
+++ b/requirements_cove.txt
@@ -67,12 +67,12 @@ jsonref==1.1.0
     # via
     #   flattentool
     #   libcove
-jsonschema==3.2.0
+jsonschema==4.18
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
     #   libcove
-libcove==0.31.0
+libcove==0.32.1
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
diff --git a/requirements_cove_dev.txt b/requirements_cove_dev.txt
index f4417bf..f56b1cf 100644
--- a/requirements_cove_dev.txt
+++ b/requirements_cove_dev.txt
@@ -123,12 +123,12 @@ jsonref==1.1.0
     # via
     #   flattentool
     #   libcove
-jsonschema==3.2.0
+jsonschema==4.18
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
     #   libcove
-libcove==0.31.0
+libcove==0.32.1
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
diff --git a/requirements_cove_dokku.txt b/requirements_cove_dokku.txt
index 3d7cfec..4938d66 100644
--- a/requirements_cove_dokku.txt
+++ b/requirements_cove_dokku.txt
@@ -90,12 +90,12 @@ jsonref==1.1.0
     #   -r requirements_cove.txt
     #   flattentool
     #   libcove
-jsonschema==3.2.0
+jsonschema==4.18
     # via
     #   -r requirements_cove.txt
     #   lib360dataquality
     #   libcove
-libcove==0.31.0
+libcove==0.32.1
     # via
     #   -r requirements_cove.txt
     #   lib360dataquality
diff --git a/setup.py b/setup.py
index c2181ad..0ef3cb1 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@
         'python-dateutil',
         'rangedict',
         'ijson',
-        'jsonschema<4',
+        'jsonschema>=4.18',
         'json-merge-patch',
     ],
     extras_require={