From f8d747aa5b92a734deeb75af5017cdc7d47ad8fc Mon Sep 17 00:00:00 2001 From: psaesha Date: Wed, 21 Oct 2026 23:13:29 +0530 Subject: [PATCH] feat(metadata): add source identifier dataset metadata item CivicDataLab's scrapers fill "Source identifier" on every dataset they upload (dataspace-scrapers//) and read it back to recognise their own datasets, so a dataset is never created twice and their upload records can be rebuilt from DataSpace. The data migration creates the item, or switches it back on if an admin switched it off; reversing it deletes nothing. Tests: tests/test_source_identifier_migration.py Co-Authored-By: Claude Opus 5.5 --- .../0054_add_source_identifier_metadata.py | 48 ++++++++++++ tests/test_source_identifier_migration.py | 78 +++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 api/migrations/0054_add_source_identifier_metadata.py create mode 100644 tests/test_source_identifier_migration.py diff --git a/api/migrations/0054_add_source_identifier_metadata.py b/api/migrations/0054_add_source_identifier_metadata.py new file mode 100644 index 00000000..f39e0d8c --- /dev/null +++ b/api/migrations/0054_add_source_identifier_metadata.py @@ -0,0 +1,48 @@ +"""Add the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers (DataSpace-Scrapers) fill it on every dataset they upload with +dataspace-scrapers//, and read it back to recognise their own +datasets, so a dataset is never created twice and their upload records can be rebuilt +from DataSpace. People never fill it in. Platform imports (Kaggle, Hugging Face, GitHub) +also fill it, with the platform's own ID, because the label matches the import's +"source identifier" field; only values starting with dataspace-scrapers/ are a scraper's. + +Creates the item if it is missing and switches it back on if an admin switched it off. +Reversing it deletes nothing, since datasets may already carry values for it. +""" + +from django.db import migrations + +LABEL = "Source identifier" + + +def add_source_identifier(apps, schema_editor): + Metadata = apps.get_model("api", "Metadata") + item, created = Metadata.objects.get_or_create( + label=LABEL, + model="DATASET", + defaults={ + "urn": "dataspace-scrapers:source-identifier", + "data_type": "STRING", + "type": "OPTIONAL", + "data_standard": "NA", + "options": None, + "validator": [], + "enabled": True, + "filterable": False, + }, + ) + if not created and not item.enabled: + item.enabled = True + item.save(update_fields=["enabled"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_collaborative_summary"), + ] + + operations = [ + migrations.RunPython(add_source_identifier, migrations.RunPython.noop), + ] diff --git a/tests/test_source_identifier_migration.py b/tests/test_source_identifier_migration.py new file mode 100644 index 00000000..1f9d6e94 --- /dev/null +++ b/tests/test_source_identifier_migration.py @@ -0,0 +1,78 @@ +"""Tests for the migration that adds the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers look the item up by its exact label, on datasets, and +only while it is switched on, so these check that the migration leaves exactly +one such item behind, switched on, whatever state it finds. + +The test settings skip migrations, so the migration's function is called +directly against the current models. +""" + +import importlib + +import pytest +from django.apps import apps + +from api.models import Metadata + +migration = importlib.import_module("api.migrations.0054_add_source_identifier_metadata") + + +def source_identifier_items(): + return Metadata.objects.filter(label="Source identifier", model="DATASET") + + +@pytest.mark.django_db +def test_creates_the_item_switched_on_when_missing(): + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "dataspace-scrapers:source-identifier" + assert item.data_type == "STRING" + assert item.type == "OPTIONAL" + assert item.filterable is False + + +@pytest.mark.django_db +def test_switches_an_existing_item_back_on_without_adding_another(): + Metadata.objects.create( + label="Source identifier", + model="DATASET", + urn="admin-made:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "admin-made:source-identifier" + + +@pytest.mark.django_db +def test_running_twice_leaves_one_item(): + migration.add_source_identifier(apps, None) + migration.add_source_identifier(apps, None) + + assert source_identifier_items().count() == 1 + + +@pytest.mark.django_db +def test_leaves_a_same_named_item_on_another_model_alone(): + resource_item = Metadata.objects.create( + label="Source identifier", + model="RESOURCE", + urn="resource:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + resource_item.refresh_from_db() + assert resource_item.enabled is False + assert source_identifier_items().get().urn == "dataspace-scrapers:source-identifier"