feat(dwca): add visual editor

.github/workflows/update-gbif-catalogs.yml

@@ -0,0 +1,77 @@
+name: Update GBIF catalogs
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 1 * *'
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-gbif-catalogs
+  cancel-in-progress: true
+
+jobs:
+  update:
+    name: Generate GBIF catalogs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: specifyweb/frontend/js_src/package.json
+          cache: npm
+          cache-dependency-path: specifyweb/frontend/js_src/package-lock.json
+
+      - name: Install frontend dependencies
+        working-directory: specifyweb/frontend/js_src
+        run: npm ci
+
+      - name: Generate GBIF catalogs
+        working-directory: specifyweb/frontend/js_src
+        run: npm run dwca:catalog
+
+      - name: Verify GBIF catalog examples
+        working-directory: specifyweb/frontend/js_src
+        run: |
+          node --input-type=module -e '
+            import cores from "./lib/components/DwcaDefinition/data/gbifCores.json" with { type: "json" };
+            import extensions from "./lib/components/DwcaDefinition/data/gbifExtensions.json" with { type: "json" };
+            const fields = [...cores, ...extensions].flatMap(({ fields }) => fields);
+            if (!fields.some(({ examples }) => examples !== undefined && examples !== "")) {
+              throw new Error("GBIF catalogs do not contain any field examples");
+            }
+          '
+
+      - name: Check for catalog changes
+        id: changes
+        run: |
+          if git diff --quiet -- \
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifCores.json \
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifExtensions.json; then
+            echo 'changed=false' >> "$GITHUB_OUTPUT"
+          else
+            echo 'changed=true' >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open update pull request if catalogs changed
+        if: steps.changes.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: 'chore(dwca): update GBIF catalogs'
+          title: 'Update GBIF catalogs'
+          body: |
+            Updates the generated GBIF core and extension catalogs from the GBIF registry.
+
+            This pull request was created automatically by the monthly catalog refresh.
+          branch: automation/update-gbif-catalogs
+          delete-branch: true
+          add-paths: |
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifCores.json
+            specifyweb/frontend/js_src/lib/components/DwcaDefinition/data/gbifExtensions.json

config/backstop/dataobj_formatters.xml

@@ -549,13 +549,6 @@
 			</fields>
 		</switch>
 	</format>
-	<format name="AgentIdentifier" title="Agent Identifier" class="edu.ku.brc.specify.datamodel.AgentIdentifier" default="true">
-		<switch single="true">
-			<fields>
-				<field sep=" ">identifier</field>
-			</fields>
-		</switch>
-	</format>
 	<format name="TectonicUnit" title="Tectonic Unit" class="edu.ku.brc.specify.datamodel.TectonicUnit" default="true">
 		<switch single="true">
 			<fields>
@@ -584,6 +577,40 @@
 			</fields>
 		</switch>
 	</format>
+	<format name="AgentIdentifier" title="Agent Identifier" class="edu.ku.brc.specify.datamodel.AgentIdentifier" default="true">
+		<switch single="false" field="identifierType">
+			<fields>
+				<field>identifier</field>
+			</fields>
+			<fields value="ORCID">
+				<field sep="https://orcid.org/">identifier</field>
+			</fields>
+			<fields value="Wikidata">
+				<field sep="https://www.wikidata.org/entity/">identifier</field>
+			</fields>
+		</switch>
+	</format>
+	<format name="CollectorID" title="CollectorID" class="edu.ku.brc.specify.datamodel.Collector" default="">
+		<switch single="true">
+			<fields>
+				<field>agent.identifiers</field>
+			</fields>
+		</switch>
+	</format>
+	<format name="Determiner" title="Determiner" class="edu.ku.brc.specify.datamodel.Determiner" default="true">
+		<switch single="true">
+			<fields>
+				<field sep=" ">agent</field>
+			</fields>
+		</switch>
+	</format>
+	<format name="DeterminerID" title="DeterminerID" class="edu.ku.brc.specify.datamodel.Determiner" default="">
+		<switch single="true">
+			<fields>
+				<field>agent.identifiers</field>
+			</fields>
+		</switch>
+	</format>
 	<aggregators>
 		<aggregator name="AccessionAgent" title="AccessionAgent" class="edu.ku.brc.specify.datamodel.AccessionAgent" default="true" separator="; " ending="" format="AccessionAgent"/>
 		<aggregator name="AccessionAttachment" title="AccessionAttachment" class="edu.ku.brc.specify.datamodel.AccessionAttachment" default="true" separator="; " ending="" format="identity" orderfieldname="ordinal"/>
@@ -629,12 +656,15 @@
 		<aggregator name="PcrPerson" title="PcrPerson" class="edu.ku.brc.specify.datamodel.PcrPerson" default="true" separator="; " ending="" format="PcrPerson" orderfieldname="orderNumber"/>
 		<aggregator name="TaxonAttachment" title="TaxonAttachment" class="edu.ku.brc.specify.datamodel.TaxonAttachment" default="true" separator="; " ending="" format="identity" orderfieldname="ordinal"/>
 		<aggregator name="CollectionObjectGroupJoin" title="Collection Object Group Join" class="edu.ku.brc.specify.datamodel.CollectionObjectGroupJoin" default="true" separator="; " ending="" format=""/>
-		<aggregator name="AgentIdentifier" title="Agent Identifier" class="edu.ku.brc.specify.datamodel.AgentIdentifier" default="true" separator="; " ending="" format=""/>
 		<aggregator name="Collection" title="Collection" class="edu.ku.brc.specify.datamodel.Collection" default="true" separator=" | " ending="" format=""/>
 		<aggregator name="Discipline" title="Discipline" class="edu.ku.brc.specify.datamodel.Discipline" default="true" separator=" | " ending="" format=""/>
 		<aggregator name="Division" title="Division" class="edu.ku.brc.specify.datamodel.Division" default="true" separator=" | " ending="" format=""/>
 		<aggregator name="Taxon" title="Taxon" class="edu.ku.brc.specify.datamodel.Taxon" default="true" separator="; " ending="" format=""/>
 		<aggregator name="Geography" title="Geography" class="edu.ku.brc.specify.datamodel.Geography" default="true" separator="; " ending="" format=""/>
 		<aggregator name="Storage" title="Storage" class="edu.ku.brc.specify.datamodel.Storage" default="true" separator="; " ending="" format=""/>
+		<aggregator name="AgentIdentifier" title="Agent Identifier" class="edu.ku.brc.specify.datamodel.AgentIdentifier" default="true" separator=" | " ending="" format=""/>
+		<aggregator name="Collector Identifiers" title="Collector Identifiers" class="edu.ku.brc.specify.datamodel.Collector" default="" separator=" | " ending="" format="CollectorID" orderfieldname="orderNumber"/>
+		<aggregator name="Determiner" title="Determiner" class="edu.ku.brc.specify.datamodel.Determiner" default="true" separator="; " ending="" format=""/>
+		<aggregator name="Determiner Identifiers" title="Determiner Identifiers" class="edu.ku.brc.specify.datamodel.Determiner" default="" separator=" | " ending="" format="DeterminerID" orderfieldname="orderNumber"/>
 	</aggregators>
 </formatters>

config/common/common.views.xml

@@ -5579,7 +5579,7 @@
 						<!-- Shown if the Identifier Type is Wikidata -->
 						<rows condition="identifierType=Wikidata">
 							<row>
-								<cell type="field" id="Wikidata" name="this" uitype="plugin" initialize="name=WebLinkButton;weblink=Wikidata;icon=https://upload.wikimedia.org/wikipedia/commons/thumb/f/ff/Wikidata-logo.svg/2560px-Wikidata-logo.svg.png"/>
+								<cell type="field" id="Wikidata" name="this" uitype="plugin" initialize="name=WebLinkButton;weblink=Wikidata;icon=https://upload.wikimedia.org/wikipedia/commons/b/be/Wikidata-logo.png"/>
 							</row>
 						</rows>
 					</cell>

config/common/weblinks.xml

@@ -21,8 +21,8 @@
 	</weblinkdef>
 	<weblinkdef>
 		<name>ORCID</name>
-		<tableName>Agent</tableName>
-		<baseURLStr>orcid.com/&lt;this&gt;</baseURLStr>
+		<tableName>AgentIdentifier</tableName>
+		<baseURLStr>https://orcid.org/&lt;this&gt;</baseURLStr>
 		<args>
 			<weblinkdefarg>
 				<name>this</name>
@@ -39,8 +39,8 @@
 	</weblinkdef>
 	<weblinkdef>
 		<name>Wikidata</name>
-		<tableName>Agent</tableName>
-		<baseURLStr>wikidata.com/&lt;this&gt;</baseURLStr>
+		<tableName>AgentIdentifier</tableName>
+		<baseURLStr>https://www.wikidata.org/entity/&lt;this&gt;</baseURLStr>
 		<args>
 			<weblinkdefarg>
 				<name>this</name>

specifyweb/backend/accounts/views.py

@@ -209,7 +209,7 @@ def oic_callback(request: http.HttpRequest) -> http.HttpResponse:
             'id': str(ext_user['sub']),
             'provider': provider,
             'provider_title': provider_info['title'],
-            'name': ext_user.get('name', ext_user.get('email', None)),
+            'name': ext_user.get('name') or ext_user.get('email') or '',
             'idtoken': ext_user,
         }
         request.session['external_user'] = external_user

specifyweb/backend/context/app_resource.py

@@ -323,27 +323,48 @@ def get_app_resource_dirs_for_level(collection, user, level):
     return Spappresourcedir.objects.filter(**filters)
 
 def merge_formatters(custom_xml, default_xml):
-    """Merge custom and default XML formatters.
-    This function takes two XML strings, one for custom formatters and one for default formatters,
-    and merges them by adding any formatters from the default XML that are not already present in
-    the custom XML. It returns the merged XML as a string.
-    """
+    """Keep custom definitions while supplying required backstop ID formats."""
 
     custom_root = ElementTree.fromstring(custom_xml)
     default_root = ElementTree.fromstring(default_xml)
 
-    # Create a set of existing classes in the custom XML to avoid duplicates
-    custom_classes = set()
-    for format_elem in custom_root.findall('format'):
-        class_attr = format_elem.get('class')
-        if class_attr:
-            custom_classes.add(class_attr)
-
-    # Add formatters from default that don't exist in custom
-    for format_elem in default_root.findall('format'):
-        class_attr = format_elem.get('class')
-        if class_attr and class_attr not in custom_classes:
-            custom_root.append(format_elem)
+    # Custom resources override a table's ordinary formats. The named ID
+    # formats are also needed by the shipped DwC-A templates for that table.
+    required_formats = {'AgentIdentifier', 'CollectorID', 'DeterminerID'}
+    required_aggregators = {'AgentIdentifier', 'Collector Identifiers', 'Determiner Identifiers'}
+    custom_formats = custom_root.findall('format')
+    custom_classes = {element.get('class') for element in custom_formats}
+    custom_keys = {
+        (element.get('class'), element.get('name')) for element in custom_formats
+    }
+    for element in default_root.findall('format'):
+        class_name = element.get('class')
+        key = (class_name, element.get('name'))
+        if class_name and key not in custom_keys and (
+            class_name not in custom_classes or element.get('name') in required_formats
+        ):
+            custom_root.append(element)
+            custom_keys.add(key)
+
+    default_aggregators = default_root.find('aggregators')
+    if default_aggregators is not None:
+        custom_aggregators = custom_root.find('aggregators')
+        existing = (
+            {
+                (element.get('class'), element.get('name'))
+                for element in custom_aggregators.findall('aggregator')
+            }
+            if custom_aggregators is not None else set()
+        )
+        for element in default_aggregators.findall('aggregator'):
+            if element.get('name') not in required_aggregators or (
+                element.get('class'), element.get('name')
+            ) in existing:
+                continue
+            if custom_aggregators is None:
+                custom_aggregators = ElementTree.SubElement(custom_root, 'aggregators')
+            custom_aggregators.append(element)
+            existing.add((element.get('class'), element.get('name')))
 
     # Convert the merged XML tree back to a string
     return ElementTree.tostring(custom_root, encoding='utf-8').decode('utf-8')

specifyweb/backend/context/tests/test_app_resource.py

@@ -1,16 +1,120 @@
-from django.test import Client
-
-from specifyweb.specify.tests.test_api import ApiTests
+from pathlib import Path
 from unittest.mock import Mock, patch
+from xml.etree import ElementTree
 
-from specifyweb.backend.context.app_resource import get_data_view_queries_resource
+from django.conf import settings
+from django.test import Client, SimpleTestCase
+
+from specifyweb.backend.context.app_resource import (
+    get_app_resource,
+    get_data_view_queries_resource,
+    merge_formatters,
+)
+from specifyweb.specify.tests.test_api import ApiTests
 
 
 FOUND_RESOURCE = ('<resource><text>"Value"</text></resource>', "text/xml", 4)
 
 
+class TestMergeFormatters(SimpleTestCase):
+    def test_inherits_named_formats_and_aggregators_without_replacing_custom_ones(self):
+        custom = '''<formatters>
+          <format class="Collector" name="Collector" default="true" title="Custom"/>
+          <aggregators>
+            <aggregator class="Collector" name="Collector" default="true" title="Custom"/>
+          </aggregators>
+        </formatters>'''
+        defaults = '''<formatters>
+          <format class="Collector" name="Collector" default="true" title="Default"/>
+          <format class="Collector" name="CollectorID" default="false"/>
+          <format class="Collector" name="Other" default="false"/>
+          <aggregators>
+            <aggregator class="Collector" name="Collector" default="true" title="Default"/>
+            <aggregator class="Collector" name="Collector Identifiers" default="false" format="CollectorID"/>
+            <aggregator class="Collector" name="Other" default="false"/>
+          </aggregators>
+        </formatters>'''
+
+        merged = ElementTree.fromstring(merge_formatters(custom, defaults))
+        self.assertEqual(
+            [(node.get('name'), node.get('title')) for node in merged.findall('format')],
+            [('Collector', 'Custom'), ('CollectorID', None)],
+        )
+        self.assertEqual(
+            [
+                (node.get('name'), node.get('title'))
+                for node in merged.findall('aggregators/aggregator')
+            ],
+            [('Collector', 'Custom'), ('Collector Identifiers', None)],
+        )
+        self.assertEqual(
+            [node.get('name') for node in merged.findall('format[@default="true"]')],
+            ['Collector'],
+        )
+        without_aggregators = '''<formatters>
+          <format class="Collector" name="Collector" default="true"/>
+        </formatters>'''
+        inherited = ElementTree.fromstring(
+            merge_formatters(without_aggregators, defaults)
+        )
+        self.assertIsNotNone(
+            inherited.find('aggregators/aggregator[@name="Collector Identifiers"]')
+        )
+
+    def test_backstop_id_formats_survive_custom_formatters(self):
+        custom = '''<formatters>
+          <format class="edu.ku.brc.specify.datamodel.Collector" name="Collector" default="true"/>
+          <format class="edu.ku.brc.specify.datamodel.Determiner" name="Determiner" default="true"/>
+          <format class="edu.ku.brc.specify.datamodel.AgentIdentifier" name="CustomIdentifier" default="true"/>
+          <aggregators>
+            <aggregator class="edu.ku.brc.specify.datamodel.Collector" name="Collector" default="true"/>
+            <aggregator class="edu.ku.brc.specify.datamodel.Determiner" name="Determiner" default="true"/>
+          </aggregators>
+        </formatters>'''
+        backstop = (
+            Path(settings.SPECIFY_CONFIG_DIR) / 'backstop' / 'dataobj_formatters.xml'
+        ).read_text()
+        merged = ElementTree.fromstring(merge_formatters(custom, backstop))
+
+        for name in ('AgentIdentifier', 'CollectorID', 'DeterminerID'):
+            self.assertIsNotNone(merged.find(f'format[@name="{name}"]'))
+        for name in ('AgentIdentifier', 'Collector Identifiers', 'Determiner Identifiers'):
+            self.assertIsNotNone(merged.find(f'aggregators/aggregator[@name="{name}"]'))
+
+
 class TestAppResource(ApiTests):
 
+    @patch("specifyweb.backend.context.app_resource.load_resource_at_level")
+    @patch("specifyweb.backend.context.app_resource.get_app_resource_from_db")
+    def test_custom_formatters_inherit_backstop_aggregations(
+        self, get_from_db: Mock, load_from_filesystem: Mock
+    ):
+        custom = '''<formatters>
+          <format class="edu.ku.brc.specify.datamodel.Collector" name="Collector" default="true"/>
+          <aggregators>
+            <aggregator class="edu.ku.brc.specify.datamodel.Collector" name="Collector" default="true"/>
+          </aggregators>
+        </formatters>'''
+        backstop = (
+            Path(settings.SPECIFY_CONFIG_DIR) / 'backstop' / 'dataobj_formatters.xml'
+        ).read_text()
+        get_from_db.side_effect = lambda _collection, _user, level, _name: (
+            (custom, 'text/xml', 1) if level == 'Collection' else None
+        )
+        load_from_filesystem.side_effect = lambda _collection, _user, level, _name: (
+            (backstop, 'text/xml', 2) if level == 'Backstop' else None
+        )
+
+        result = get_app_resource(self.collection, self.specifyuser, 'DataObjFormatters')
+        self.assertIsNotNone(result)
+        merged = ElementTree.fromstring(result[0])
+        self.assertIsNotNone(
+            merged.find('aggregators/aggregator[@name="Collector Identifiers"]')
+        )
+        self.assertIsNotNone(
+            merged.find('aggregators/aggregator[@name="Determiner Identifiers"]')
+        )
+
     @patch("specifyweb.backend.context.app_resource.load_resource_at_level")
     @patch("specifyweb.backend.context.app_resource.get_app_resource_from_db")
     def test_data_view_queries_inherit_by_table(