feat(metadata): add source identifier dataset metadata item

api/migrations/0054_add_source_identifier_metadata.py

@@ -0,0 +1,48 @@
+"""Add the "Source identifier" dataset metadata item.
+
+CivicDataLab's scrapers (DataSpace-Scrapers) fill it on every dataset they upload with
+dataspace-scrapers/<scraper_id>/<dataset_key>, and read it back to recognise their own
+datasets, so a dataset is never created twice and their upload records can be rebuilt
+from DataSpace. People never fill it in. Platform imports (Kaggle, Hugging Face, GitHub)
+also fill it, with the platform's own ID, because the label matches the import's
+"source identifier" field; only values starting with dataspace-scrapers/ are a scraper's.
+
+Creates the item if it is missing and switches it back on if an admin switched it off.
+Reversing it deletes nothing, since datasets may already carry values for it.
+"""
+
+from django.db import migrations
+
+LABEL = "Source identifier"
+
+
+def add_source_identifier(apps, schema_editor):
+    Metadata = apps.get_model("api", "Metadata")
+    item, created = Metadata.objects.get_or_create(
+        label=LABEL,
+        model="DATASET",
+        defaults={
+            "urn": "dataspace-scrapers:source-identifier",
+            "data_type": "STRING",
+            "type": "OPTIONAL",
+            "data_standard": "NA",
+            "options": None,
+            "validator": [],
+            "enabled": True,
+            "filterable": False,
+        },
+    )
+    if not created and not item.enabled:
+        item.enabled = True
+        item.save(update_fields=["enabled"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0053_alter_collaborative_summary"),
+    ]
+
+    operations = [
+        migrations.RunPython(add_source_identifier, migrations.RunPython.noop),
+    ]

tests/test_source_identifier_migration.py

@@ -0,0 +1,78 @@
+"""Tests for the migration that adds the "Source identifier" dataset metadata item.
+
+CivicDataLab's scrapers look the item up by its exact label, on datasets, and
+only while it is switched on, so these check that the migration leaves exactly
+one such item behind, switched on, whatever state it finds.
+
+The test settings skip migrations, so the migration's function is called
+directly against the current models.
+"""
+
+import importlib
+
+import pytest
+from django.apps import apps
+
+from api.models import Metadata
+
+migration = importlib.import_module("api.migrations.0054_add_source_identifier_metadata")
+
+
+def source_identifier_items():
+    return Metadata.objects.filter(label="Source identifier", model="DATASET")
+
+
+@pytest.mark.django_db
+def test_creates_the_item_switched_on_when_missing():
+    migration.add_source_identifier(apps, None)
+
+    item = source_identifier_items().get()
+    assert item.enabled is True
+    assert item.urn == "dataspace-scrapers:source-identifier"
+    assert item.data_type == "STRING"
+    assert item.type == "OPTIONAL"
+    assert item.filterable is False
+
+
+@pytest.mark.django_db
+def test_switches_an_existing_item_back_on_without_adding_another():
+    Metadata.objects.create(
+        label="Source identifier",
+        model="DATASET",
+        urn="admin-made:source-identifier",
+        data_type="STRING",
+        type="OPTIONAL",
+        enabled=False,
+    )
+
+    migration.add_source_identifier(apps, None)
+
+    item = source_identifier_items().get()
+    assert item.enabled is True
+    assert item.urn == "admin-made:source-identifier"
+
+
+@pytest.mark.django_db
+def test_running_twice_leaves_one_item():
+    migration.add_source_identifier(apps, None)
+    migration.add_source_identifier(apps, None)
+
+    assert source_identifier_items().count() == 1
+
+
+@pytest.mark.django_db
+def test_leaves_a_same_named_item_on_another_model_alone():
+    resource_item = Metadata.objects.create(
+        label="Source identifier",
+        model="RESOURCE",
+        urn="resource:source-identifier",
+        data_type="STRING",
+        type="OPTIONAL",
+        enabled=False,
+    )
+
+    migration.add_source_identifier(apps, None)
+
+    resource_item.refresh_from_db()
+    assert resource_item.enabled is False
+    assert source_identifier_items().get().urn == "dataspace-scrapers:source-identifier"