Keep UNESCO sites' points through the refresh

scripts/refresh.ts

@@ -12,6 +12,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { gunzipSync, gzipSync } from "node:zlib";
 import { REFERENCE_DIR } from "../src/reference/common";
+import type { SourceSite } from "../src/reference/heritage";
 import {
   admin1Codes,
   citiesTsv,
@@ -104,7 +105,7 @@ await step("UNESCO World Heritage List", async () => {
     lock.unesco = { ...lock.unesco!, modified };
     return "not modified";
   }
-  const saved = (JSON.parse(gunzipped("whc-sites.json.gz")) as unknown[]).length;
+  const saved = JSON.parse(gunzipped("whc-sites.json.gz")) as SourceSite[];
   const known = JSON.parse(readFileSync(join(REFERENCE_DIR, "heritage-ids.json"), "utf8")) as Record<string, string>;
   const { sites, ids } = unescoSites(res.body.toString("utf8"), saved, known);
   const changed = save("whc-sites.json.gz", JSON.stringify(sites) + "\n");

src/reference/heritage.ts

@@ -25,10 +25,36 @@ export interface SourceSite {
   id_no: number | string;
   name_en: string;
   category: string;
-  iso_codes?: string;
-  coordinates?: { lat?: number; lon?: number };
+  /** "FR,ES" in the saved copy; UNESCO's export gives ["FR", "ES"]. */
+  iso_codes?: string | string[];
+  /** { lat, lon } in the saved copy; see sitePoint. */
+  coordinates?: unknown;
 }
 
+/**
+ * A site's point as the build reads it, { lat, lon }, from either shape UNESCO has used:
+ * { lat, lon }, or a GeoJSON Point or MultiPoint ([lon, lat] each). A site of several
+ * parts takes the part nearest their middle, so its point is always one of its own.
+ */
+export function sitePoint(c: unknown): { lat: number; lon: number } | undefined {
+  const o = c as { lat?: unknown; lon?: unknown; geometry?: { coordinates?: unknown } } | null;
+  const valid = (lat: unknown, lon: unknown): lat is number =>
+    typeof lat === "number" && typeof lon === "number" && Math.abs(lat) <= 90 && Math.abs(lon) <= 180;
+  if (valid(o?.lat, o?.lon)) return { lat: o!.lat as number, lon: o!.lon as number };
+  const g = o?.geometry?.coordinates;
+  if (!Array.isArray(g)) return undefined;
+  const points = (Array.isArray(g[0]) ? g : [g])
+    .map((p) => (Array.isArray(p) ? { lon: p[0] as unknown, lat: p[1] as unknown } : {}))
+    .filter((p): p is { lat: number; lon: number } => valid(p.lat, p.lon));
+  if (points.length === 0) return undefined;
+  const lat = points.reduce((n, p) => n + p.lat, 0) / points.length;
+  const lon = points.reduce((n, p) => n + p.lon, 0) / points.length;
+  const k = Math.cos((lat * Math.PI) / 180);
+  const d = (p: { lat: number; lon: number }) => (p.lat - lat) ** 2 + ((p.lon - lon) * k) ** 2;
+  return points.reduce((best, p) => (d(p) < d(best) ? p : best));
+}
+
+
 export interface HeritageSite {
   id: string;
   name: string;
@@ -78,16 +104,15 @@ function entries(s: SourceSite): { key: string; candidate: string; site: Omit<He
   if (!name || !category) throw new Error(`heritage: site ${s.id_no} has a bad name or category`);
   const isos = [
     ...new Set(
-      String(s.iso_codes ?? "")
+      (Array.isArray(s.iso_codes) ? s.iso_codes.join(",") : String(s.iso_codes ?? ""))
         .split(",")
         .map((c) => c.trim().toUpperCase())
         .filter((c) => /^[A-Z]{2}$/.test(c)),
     ),
   ];
-  const c = s.coordinates;
-  const has = typeof c?.lat === "number" && typeof c?.lon === "number" && Number.isFinite(c.lat) && Number.isFinite(c.lon);
-  const lat = has ? Number(c!.lat!.toFixed(4)) : 0;
-  const lon = has ? Number(c!.lon!.toFixed(4)) : 0;
+  const at = sitePoint(s.coordinates);
+  const lat = at ? Number(at.lat.toFixed(4)) : 0;
+  const lon = at ? Number(at.lon.toFixed(4)) : 0;
   const slug = slugify(name);
   return isos.map((iso) => ({
     key: `${s.id_no}-${iso}`,

src/reference/refresh.ts

@@ -4,7 +4,7 @@
  * Every function here throws on anything unexpected, so a bad download keeps the old copy.
  */
 import type { SourceCountry } from "./countries";
-import { heritageIds, type SourceSite } from "./heritage";
+import { heritageIds, sitePoint, type SourceSite } from "./heritage";
 
 const UA = "offware-data-refresh (+https://github.com/offware-apps/data)";
 
@@ -95,13 +95,20 @@ export function countriesJson(text: string, saved: number): SourceCountry[] {
   });
 }
 
-/** The UNESCO list, and every site's id: those it had, and new ones for new sites. */
-export function unescoSites(text: string, saved: number, ids: Record<string, string>): { sites: SourceSite[]; ids: Record<string, string> } {
+/** The UNESCO list in the saved shape, and every site's id: those it had, and new ones for new sites. */
+export function unescoSites(text: string, saved: SourceSite[], ids: Record<string, string>): { sites: SourceSite[]; ids: Record<string, string> } {
   const list = JSON.parse(text) as SourceSite[];
-  if (!Array.isArray(list) || !enough(list.length, saved)) throw new Error("heritage: unexpected file");
+  if (!Array.isArray(list) || !enough(list.length, saved.length)) throw new Error("heritage: unexpected file");
   const sites = list
-    .map((s) => ({ id_no: String(s.id_no), name_en: s.name_en, iso_codes: s.iso_codes, category: s.category, coordinates: s.coordinates }))
+    .map((s) => {
+      const at = sitePoint(s.coordinates);
+      const iso = Array.isArray(s.iso_codes) ? s.iso_codes.join(",") : s.iso_codes;
+      return { id_no: String(s.id_no), name_en: s.name_en, iso_codes: iso, category: s.category, ...(at ? { coordinates: at } : {}) };
+    })
     .sort((a, b) => Number(a.id_no) - Number(b.id_no));
   if (sites.some((s) => !/^\d+$/.test(s.id_no))) throw new Error("heritage: a site without a number");
+  // A change of shape that loses the points must not pass for an update.
+  const located = (l: SourceSite[]) => l.filter((s) => sitePoint(s.coordinates)).length;
+  if (!enough(located(sites), located(saved))) throw new Error(`heritage: ${located(sites)} sites with a point, ${located(saved)} saved`);
   return { sites, ids: heritageIds(sites, ids) };
 }

tests/refresh.test.ts

@@ -56,8 +56,31 @@ describe("source refresh", () => {
   it("keeps a UNESCO site's id when it is renamed, and never gives a new site an old id", () => {
     const site = (id_no: string, name_en: string, iso_codes = "fr"): SourceSite => ({ id_no, name_en, iso_codes, category: "Cultural", coordinates: { lat: 1, lon: 2 } });
     const known = { "1-FR": "whs-old-town", "9-FR": "whs-harbour" };
-    const { sites, ids } = unescoSites(JSON.stringify([site("7", "Harbour"), site("1", "Old Town and its Walls")]), 2, known);
+    const { sites, ids } = unescoSites(JSON.stringify([site("7", "Harbour"), site("1", "Old Town and its Walls")]), [site("1", "Old Town"), site("9", "Harbour")], known);
     expect(sites.map((s) => s.id_no)).toEqual(["1", "7"]);
     expect(ids).toEqual({ "1-FR": "whs-old-town", "7-FR": "whs-harbour-7", "9-FR": "whs-harbour" });
   });
+
+  it("reads UNESCO's GeoJSON points and code lists into the saved shape", () => {
+    const feature = (points: number[][]) => ({ type: "Feature", geometry: { type: "MultiPoint", coordinates: points }, properties: {} });
+    const raw = [
+      { id_no: "1", name_en: "One", iso_codes: ["FR"], category: "Cultural", coordinates: feature([[2.35, 48.85]]) },
+      // Three parts: the one nearest their middle is kept.
+      { id_no: "2", name_en: "Two", iso_codes: ["FR", "ES"], category: "Natural", coordinates: feature([[0, 40], [10, 50], [5, 46]]) },
+      { id_no: "3", name_en: "Three", iso_codes: ["IT"], category: "Mixed", coordinates: null },
+    ];
+    const saved: SourceSite[] = [{ id_no: "1", name_en: "One", iso_codes: "FR", category: "Cultural", coordinates: { lat: 48.85, lon: 2.35 } }];
+    const { sites } = unescoSites(JSON.stringify(raw), saved, {});
+    expect(sites).toEqual([
+      { id_no: "1", name_en: "One", iso_codes: "FR", category: "Cultural", coordinates: { lon: 2.35, lat: 48.85 } },
+      { id_no: "2", name_en: "Two", iso_codes: "FR,ES", category: "Natural", coordinates: { lon: 5, lat: 46 } },
+      { id_no: "3", name_en: "Three", iso_codes: "IT", category: "Mixed" },
+    ]);
+  });
+
+  it("keeps the saved UNESCO copy when a new shape loses the points", () => {
+    const saved: SourceSite[] = [1, 2, 3].map((n) => ({ id_no: String(n), name_en: `Site ${n}`, iso_codes: "FR", category: "Cultural", coordinates: { lat: 1, lon: 2 } }));
+    const unknownShape = saved.map((s) => ({ ...s, coordinates: { x: 2, y: 1 } }));
+    expect(() => unescoSites(JSON.stringify(unknownShape), saved, {})).toThrow(/with a point/);
+  });
 });