diff --git a/data/reference/whc-sites.json.gz b/data/reference/whc-sites.json.gz index 32efdaa..8310a21 100644 Binary files a/data/reference/whc-sites.json.gz and b/data/reference/whc-sites.json.gz differ diff --git a/scripts/refresh.ts b/scripts/refresh.ts index e85ad8e..44fb117 100644 --- a/scripts/refresh.ts +++ b/scripts/refresh.ts @@ -12,6 +12,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { gunzipSync, gzipSync } from "node:zlib"; import { REFERENCE_DIR } from "../src/reference/common"; +import type { SourceSite } from "../src/reference/heritage"; import { admin1Codes, citiesTsv, @@ -104,7 +105,7 @@ await step("UNESCO World Heritage List", async () => { lock.unesco = { ...lock.unesco!, modified }; return "not modified"; } - const saved = (JSON.parse(gunzipped("whc-sites.json.gz")) as unknown[]).length; + const saved = JSON.parse(gunzipped("whc-sites.json.gz")) as SourceSite[]; const known = JSON.parse(readFileSync(join(REFERENCE_DIR, "heritage-ids.json"), "utf8")) as Record; const { sites, ids } = unescoSites(res.body.toString("utf8"), saved, known); const changed = save("whc-sites.json.gz", JSON.stringify(sites) + "\n"); diff --git a/src/reference/heritage.ts b/src/reference/heritage.ts index ad495a8..4681caf 100644 --- a/src/reference/heritage.ts +++ b/src/reference/heritage.ts @@ -25,10 +25,36 @@ export interface SourceSite { id_no: number | string; name_en: string; category: string; - iso_codes?: string; - coordinates?: { lat?: number; lon?: number }; + /** "FR,ES" in the saved copy; UNESCO's export gives ["FR", "ES"]. */ + iso_codes?: string | string[]; + /** { lat, lon } in the saved copy; see sitePoint. */ + coordinates?: unknown; } +/** + * A site's point as the build reads it, { lat, lon }, from either shape UNESCO has used: + * { lat, lon }, or a GeoJSON Point or MultiPoint ([lon, lat] each). A site of several + * parts takes the part nearest their middle, so its point is always one of its own. + */ +export function sitePoint(c: unknown): { lat: number; lon: number } | undefined { + const o = c as { lat?: unknown; lon?: unknown; geometry?: { coordinates?: unknown } } | null; + const valid = (lat: unknown, lon: unknown): lat is number => + typeof lat === "number" && typeof lon === "number" && Math.abs(lat) <= 90 && Math.abs(lon) <= 180; + if (valid(o?.lat, o?.lon)) return { lat: o!.lat as number, lon: o!.lon as number }; + const g = o?.geometry?.coordinates; + if (!Array.isArray(g)) return undefined; + const points = (Array.isArray(g[0]) ? g : [g]) + .map((p) => (Array.isArray(p) ? { lon: p[0] as unknown, lat: p[1] as unknown } : {})) + .filter((p): p is { lat: number; lon: number } => valid(p.lat, p.lon)); + if (points.length === 0) return undefined; + const lat = points.reduce((n, p) => n + p.lat, 0) / points.length; + const lon = points.reduce((n, p) => n + p.lon, 0) / points.length; + const k = Math.cos((lat * Math.PI) / 180); + const d = (p: { lat: number; lon: number }) => (p.lat - lat) ** 2 + ((p.lon - lon) * k) ** 2; + return points.reduce((best, p) => (d(p) < d(best) ? p : best)); +} + + export interface HeritageSite { id: string; name: string; @@ -78,16 +104,15 @@ function entries(s: SourceSite): { key: string; candidate: string; site: Omit c.trim().toUpperCase()) .filter((c) => /^[A-Z]{2}$/.test(c)), ), ]; - const c = s.coordinates; - const has = typeof c?.lat === "number" && typeof c?.lon === "number" && Number.isFinite(c.lat) && Number.isFinite(c.lon); - const lat = has ? Number(c!.lat!.toFixed(4)) : 0; - const lon = has ? Number(c!.lon!.toFixed(4)) : 0; + const at = sitePoint(s.coordinates); + const lat = at ? Number(at.lat.toFixed(4)) : 0; + const lon = at ? Number(at.lon.toFixed(4)) : 0; const slug = slugify(name); return isos.map((iso) => ({ key: `${s.id_no}-${iso}`, diff --git a/src/reference/refresh.ts b/src/reference/refresh.ts index cfe42ce..5216b73 100644 --- a/src/reference/refresh.ts +++ b/src/reference/refresh.ts @@ -4,7 +4,7 @@ * Every function here throws on anything unexpected, so a bad download keeps the old copy. */ import type { SourceCountry } from "./countries"; -import { heritageIds, type SourceSite } from "./heritage"; +import { heritageIds, sitePoint, type SourceSite } from "./heritage"; const UA = "offware-data-refresh (+https://github.com/offware-apps/data)"; @@ -95,13 +95,20 @@ export function countriesJson(text: string, saved: number): SourceCountry[] { }); } -/** The UNESCO list, and every site's id: those it had, and new ones for new sites. */ -export function unescoSites(text: string, saved: number, ids: Record): { sites: SourceSite[]; ids: Record } { +/** The UNESCO list in the saved shape, and every site's id: those it had, and new ones for new sites. */ +export function unescoSites(text: string, saved: SourceSite[], ids: Record): { sites: SourceSite[]; ids: Record } { const list = JSON.parse(text) as SourceSite[]; - if (!Array.isArray(list) || !enough(list.length, saved)) throw new Error("heritage: unexpected file"); + if (!Array.isArray(list) || !enough(list.length, saved.length)) throw new Error("heritage: unexpected file"); const sites = list - .map((s) => ({ id_no: String(s.id_no), name_en: s.name_en, iso_codes: s.iso_codes, category: s.category, coordinates: s.coordinates })) + .map((s) => { + const at = sitePoint(s.coordinates); + const iso = Array.isArray(s.iso_codes) ? s.iso_codes.join(",") : s.iso_codes; + return { id_no: String(s.id_no), name_en: s.name_en, iso_codes: iso, category: s.category, ...(at ? { coordinates: at } : {}) }; + }) .sort((a, b) => Number(a.id_no) - Number(b.id_no)); if (sites.some((s) => !/^\d+$/.test(s.id_no))) throw new Error("heritage: a site without a number"); + // A change of shape that loses the points must not pass for an update. + const located = (l: SourceSite[]) => l.filter((s) => sitePoint(s.coordinates)).length; + if (!enough(located(sites), located(saved))) throw new Error(`heritage: ${located(sites)} sites with a point, ${located(saved)} saved`); return { sites, ids: heritageIds(sites, ids) }; } diff --git a/tests/refresh.test.ts b/tests/refresh.test.ts index 1e0dc3b..e212ce2 100644 --- a/tests/refresh.test.ts +++ b/tests/refresh.test.ts @@ -56,8 +56,31 @@ describe("source refresh", () => { it("keeps a UNESCO site's id when it is renamed, and never gives a new site an old id", () => { const site = (id_no: string, name_en: string, iso_codes = "fr"): SourceSite => ({ id_no, name_en, iso_codes, category: "Cultural", coordinates: { lat: 1, lon: 2 } }); const known = { "1-FR": "whs-old-town", "9-FR": "whs-harbour" }; - const { sites, ids } = unescoSites(JSON.stringify([site("7", "Harbour"), site("1", "Old Town and its Walls")]), 2, known); + const { sites, ids } = unescoSites(JSON.stringify([site("7", "Harbour"), site("1", "Old Town and its Walls")]), [site("1", "Old Town"), site("9", "Harbour")], known); expect(sites.map((s) => s.id_no)).toEqual(["1", "7"]); expect(ids).toEqual({ "1-FR": "whs-old-town", "7-FR": "whs-harbour-7", "9-FR": "whs-harbour" }); }); + + it("reads UNESCO's GeoJSON points and code lists into the saved shape", () => { + const feature = (points: number[][]) => ({ type: "Feature", geometry: { type: "MultiPoint", coordinates: points }, properties: {} }); + const raw = [ + { id_no: "1", name_en: "One", iso_codes: ["FR"], category: "Cultural", coordinates: feature([[2.35, 48.85]]) }, + // Three parts: the one nearest their middle is kept. + { id_no: "2", name_en: "Two", iso_codes: ["FR", "ES"], category: "Natural", coordinates: feature([[0, 40], [10, 50], [5, 46]]) }, + { id_no: "3", name_en: "Three", iso_codes: ["IT"], category: "Mixed", coordinates: null }, + ]; + const saved: SourceSite[] = [{ id_no: "1", name_en: "One", iso_codes: "FR", category: "Cultural", coordinates: { lat: 48.85, lon: 2.35 } }]; + const { sites } = unescoSites(JSON.stringify(raw), saved, {}); + expect(sites).toEqual([ + { id_no: "1", name_en: "One", iso_codes: "FR", category: "Cultural", coordinates: { lon: 2.35, lat: 48.85 } }, + { id_no: "2", name_en: "Two", iso_codes: "FR,ES", category: "Natural", coordinates: { lon: 5, lat: 46 } }, + { id_no: "3", name_en: "Three", iso_codes: "IT", category: "Mixed" }, + ]); + }); + + it("keeps the saved UNESCO copy when a new shape loses the points", () => { + const saved: SourceSite[] = [1, 2, 3].map((n) => ({ id_no: String(n), name_en: `Site ${n}`, iso_codes: "FR", category: "Cultural", coordinates: { lat: 1, lon: 2 } })); + const unknownShape = saved.map((s) => ({ ...s, coordinates: { x: 2, y: 1 } })); + expect(() => unescoSites(JSON.stringify(unknownShape), saved, {})).toThrow(/with a point/); + }); });