diff --git a/api/migrations/0054_add_source_identifier_metadata.py b/api/migrations/0054_add_source_identifier_metadata.py new file mode 100644 index 00000000..f39e0d8c --- /dev/null +++ b/api/migrations/0054_add_source_identifier_metadata.py @@ -0,0 +1,48 @@ +"""Add the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers (DataSpace-Scrapers) fill it on every dataset they upload with +dataspace-scrapers//, and read it back to recognise their own +datasets, so a dataset is never created twice and their upload records can be rebuilt +from DataSpace. People never fill it in. Platform imports (Kaggle, Hugging Face, GitHub) +also fill it, with the platform's own ID, because the label matches the import's +"source identifier" field; only values starting with dataspace-scrapers/ are a scraper's. + +Creates the item if it is missing and switches it back on if an admin switched it off. +Reversing it deletes nothing, since datasets may already carry values for it. +""" + +from django.db import migrations + +LABEL = "Source identifier" + + +def add_source_identifier(apps, schema_editor): + Metadata = apps.get_model("api", "Metadata") + item, created = Metadata.objects.get_or_create( + label=LABEL, + model="DATASET", + defaults={ + "urn": "dataspace-scrapers:source-identifier", + "data_type": "STRING", + "type": "OPTIONAL", + "data_standard": "NA", + "options": None, + "validator": [], + "enabled": True, + "filterable": False, + }, + ) + if not created and not item.enabled: + item.enabled = True + item.save(update_fields=["enabled"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_collaborative_summary"), + ] + + operations = [ + migrations.RunPython(add_source_identifier, migrations.RunPython.noop), + ] diff --git a/tests/test_source_identifier_migration.py b/tests/test_source_identifier_migration.py new file mode 100644 index 00000000..1f9d6e94 --- /dev/null +++ b/tests/test_source_identifier_migration.py @@ -0,0 +1,78 @@ +"""Tests for the migration that adds the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers look the item up by its exact label, on datasets, and +only while it is switched on, so these check that the migration leaves exactly +one such item behind, switched on, whatever state it finds. + +The test settings skip migrations, so the migration's function is called +directly against the current models. +""" + +import importlib + +import pytest +from django.apps import apps + +from api.models import Metadata + +migration = importlib.import_module("api.migrations.0054_add_source_identifier_metadata") + + +def source_identifier_items(): + return Metadata.objects.filter(label="Source identifier", model="DATASET") + + +@pytest.mark.django_db +def test_creates_the_item_switched_on_when_missing(): + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "dataspace-scrapers:source-identifier" + assert item.data_type == "STRING" + assert item.type == "OPTIONAL" + assert item.filterable is False + + +@pytest.mark.django_db +def test_switches_an_existing_item_back_on_without_adding_another(): + Metadata.objects.create( + label="Source identifier", + model="DATASET", + urn="admin-made:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "admin-made:source-identifier" + + +@pytest.mark.django_db +def test_running_twice_leaves_one_item(): + migration.add_source_identifier(apps, None) + migration.add_source_identifier(apps, None) + + assert source_identifier_items().count() == 1 + + +@pytest.mark.django_db +def test_leaves_a_same_named_item_on_another_model_alone(): + resource_item = Metadata.objects.create( + label="Source identifier", + model="RESOURCE", + urn="resource:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + resource_item.refresh_from_db() + assert resource_item.enabled is False + assert source_identifier_items().get().urn == "dataspace-scrapers:source-identifier"