diff --git a/.github/workflows/android-apk.yml b/.github/workflows/android-apk.yml index 599cb96..a77a18e 100644 --- a/.github/workflows/android-apk.yml +++ b/.github/workflows/android-apk.yml @@ -45,6 +45,10 @@ jobs: cache: gradle - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Fetch Offware's shared data (stations, cities, sites, countries, moments) + # A file that can't be fetched or checked keeps its committed copy. Fails only when + # there is no worldwide stations file at all. + run: pnpm --filter postcards data - name: Build the web app run: pnpm --filter postcards build - name: Sync the Android project diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 44024a8..04322a1 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -9,6 +9,9 @@ on: push: branches: - main + # Weekly, after the shared data's Monday build, so the reference data stays current. + schedule: + - cron: "0 6 * * 1" workflow_dispatch: permissions: @@ -35,6 +38,10 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Fetch Offware's shared data (stations, cities, sites, countries, moments) + # A file that can't be fetched or checked keeps its committed copy. Fails only when + # there is no worldwide stations file at all. + run: pnpm --filter postcards data - name: Build (GitHub Pages subpath) run: pnpm --filter postcards build env: diff --git a/.gitignore b/.gitignore index c9e32cd..e7125fc 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ Thumbs.db .env .env.* !.env.example + +# Fetched at build time from the Offware places dataset (scripts/fetch-places.mjs) +apps/postcards/public/reference/railways-wikidata.json diff --git a/apps/postcards/package.json b/apps/postcards/package.json index 6683642..0df1054 100644 --- a/apps/postcards/package.json +++ b/apps/postcards/package.json @@ -14,8 +14,7 @@ "schema": "UPDATE_SCHEMA=1 vitest run tests/unit/schemaArtifact.spec.ts", "test:e2e": "playwright test", "format": "prettier --write .", - "reference": "node scripts/build-reference.mjs", - "railways": "node scripts/build-railways.mjs", + "data": "node scripts/fetch-data.mjs", "cap:sync": "cap sync", "cap:copy": "cap copy", "cap:add:android": "cap add android", @@ -58,15 +57,11 @@ "@types/react-dom": "^19.2.3", "@types/topojson-client": "^3.1.5", "@vitejs/plugin-react": "^6.0.3", - "airport-data": "^1.0.1", - "all-the-cities": "^3.1.0", - "country-state-city": "^3.2.1", "jsdom": "^29.1.1", "prettier": "^3.9.4", "typescript": "^6.0.3", "vite": "^8.1.3", "vite-plugin-pwa": "^1.3.0", - "vitest": "^4.1.10", - "world-countries": "^5.1.0" + "vitest": "^4.1.10" } } diff --git a/apps/postcards/public/reference/PROVENANCE.md b/apps/postcards/public/reference/PROVENANCE.md index 9185ccd..dd62f09 100644 --- a/apps/postcards/public/reference/PROVENANCE.md +++ b/apps/postcards/public/reference/PROVENANCE.md @@ -6,6 +6,12 @@ machine-readable provenance list ships inside the app at `src/lib/reference/data/provenance.json` and is displayed in Settings; this file mirrors it for people browsing the repository. +Every file is built in Offware's shared data, , which +lists each dataset with its licence in `v1/catalog.json`. `scripts/fetch-data.mjs` fetches +them at build time, and takes a file only while its dataset still carries the licence named +below. The files committed here are what a build keeps when the shared data can't be +reached. + | File | Dataset | License | Version / retrieved | |------|---------|---------|---------------------| | `cities.json` | GeoNames city gazetteer (population ≥ 15,000, via all-the-cities) — | CC BY 4.0 | all-the-cities 3.1.0 | @@ -13,7 +19,9 @@ mirrors it for people browsing the repository. | `subdivisions.json` | First-level regions: each city's GeoNames admin-1 code () and each code's GeoNames name (), both retrieved 2026-09-30; France's 13 regions by their French names; a code GeoNames does not name keeps one from dr5hn countries-states-cities — | CC BY 4.0 (GeoNames) / ODbL (dr5hn) | all-the-cities 3.1.0 + GeoNames 2026-09-30 + country-state-city | | `airports.json` | OpenFlights (IATA-coded, aggregated from OurAirports) — | ODbL 1.0 / OurAirports public domain | airport-data 1.0.1 | | `heritage.json` | UNESCO World Heritage List — all 1,248 sites incl. 47th session (2025) — | CC BY-SA 3.0 IGO — © UNESCO World Heritage Centre | 2025 list, retrieved 2026-07-11 | -| `landmarks.json` | Famous landmarks seed (full set via Wikidata build script) — | CC0 (facts) | seed | +| `landmarks.json` | Landmarks (museums, monuments, castles, palaces, parks; UNESCO sites excluded), the 10,000 best known, from Wikidata via Offware's shared data — . Written at build time by `scripts/fetch-data.mjs`; the committed file is a small seed used when the dataset can't be reached. Each id is `wd-`. | CC0 1.0 | fetched at build time | +| `railways-wikidata.json` | Railway stations worldwide from Wikidata (open mainline stations, instance of Q55488) via Offware's shared data — . Written at build time by `scripts/fetch-data.mjs`, not committed. Each id is the station's Wikidata QID. | CC0 1.0 | fetched at build time | +| `railways.json` | Railway stations in Europe from the Trainline EU open stations database — | ODbL 1.0 | `scripts/build-railways.mjs` | | `languages.json` | Per-country spoken languages (via world-countries) — | ODbL 1.0 | 5.1.0 | | `article-names.json` | Common country names (Wikivoyage/Wikipedia article titles, via world-countries) — | ODbL 1.0 | 5.1.0 | | `experiences.json` | Postcards "Moments" — app-authored name/hint/emoji only; anchor coordinates aggregated from Wikidata () / GeoNames (); experience concepts attributed to UNESCO World Heritage (), UNESCO Intangible Cultural Heritage () & Wikivoyage travel topics (). Per-item `sources` cite the specific record. | Mixed — name/hint/emoji CC0; Wikidata CC0; GeoNames CC BY 4.0; UNESCO CC BY-SA 3.0 IGO; Wikivoyage CC BY-SA 3.0 | V1 | diff --git a/apps/postcards/scripts/build-city-admin1.mjs b/apps/postcards/scripts/build-city-admin1.mjs deleted file mode 100644 index d55e28d..0000000 --- a/apps/postcards/scripts/build-city-admin1.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// Vendor GeoNames' current admin-1 code for every city the app ships, so -// build-reference.mjs names regions from codes and names of one GeoNames day. -// -// curl -LO https://download.geonames.org/export/dump/cities500.zip && unzip cities500.zip -// node scripts/build-city-admin1.mjs cities500.txt (from apps/postcards) -// -// Writes scripts/data/city-admin1.tsv.gz: one "\t." line per -// city of all-the-cities that cities500 still lists (CC BY 4.0, GeoNames). -import { createRequire } from "node:module"; -import { readFileSync, writeFileSync } from "node:fs"; -import { gzipSync } from "node:zlib"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const require = createRequire(import.meta.url); -const allCities = require("all-the-cities"); - -const source = process.argv[2]; -if (!source) { - console.error("usage: node scripts/build-city-admin1.mjs "); - process.exit(1); -} - -const wanted = new Set(allCities.map((c) => String(c.cityId))); -const rows = []; -for (const line of readFileSync(source, "utf8").split("\n")) { - const f = line.split("\t"); - if (f.length < 11 || !wanted.has(f[0])) continue; - rows.push([Number(f[0]), `${f[0]}\t${f[8]}.${f[10]}`]); -} -rows.sort((a, b) => a[0] - b[0]); - -const here = dirname(fileURLToPath(import.meta.url)); -const out = join(here, "data", "city-admin1.tsv.gz"); -writeFileSync(out, gzipSync(rows.map((r) => r[1]).join("\n") + "\n", { level: 9 })); -console.log(`cities with a current admin-1 code: ${rows.length} of ${wanted.size}`); diff --git a/apps/postcards/scripts/build-heritage-full.mjs b/apps/postcards/scripts/build-heritage-full.mjs deleted file mode 100644 index 84f699d..0000000 --- a/apps/postcards/scripts/build-heritage-full.mjs +++ /dev/null @@ -1,161 +0,0 @@ -// Build the FULL UNESCO World Heritage List -> public/reference/heritage.json -// -// Source (aggregator-only, Constitution I — the app authors no world facts): -// UNESCO World Heritage Centre, official World Heritage List -// (https://whc.unesco.org/en/list/ — syndication data, field names match the -// whc.unesco.org whc-sites export: name_en, iso_codes, category, coordinates, -// id_no, date_inscribed, …). 1,248 sites, complete through the 47th session -// of the World Heritage Committee (July 2025). -// License: -// CC BY-SA 3.0 IGO — UNESCO World Heritage Centre publishes the List under -// Creative Commons Attribution-ShareAlike 3.0 IGO (https://whc.unesco.org/en/licenses/6). -// Attribution: © UNESCO World Heritage Centre, https://whc.unesco.org -// Retrieved: 2026-07-11, via the public GitHub mirror of the UNESCO data -// https://raw.githubusercontent.com/BayoatHT/solve_cia/main/proj_004_cia/___proj_heritage/_raw_data/json/all_world_heritage.json -// (whc.unesco.org itself is unreachable from the build sandbox). The raw -// payload is vendored gzipped at scripts/data/whc-sites-2025.json.gz so this -// build is reproducible offline; spot-checked against the official -// whc-sites CSV export (2023 edition) for coordinate fidelity. -// -// Output: public/reference/heritage.json — an array of -// { id, name, countryIso2, lat, lon, category } -// category ∈ cultural | natural | mixed. -// Transnational sites emit ONE entry per country, same name, id suffixed -// with the country code (whs--fr, whs--es, …), reusing the -// site's coordinates. Sites the source ships without coordinates keep -// lat 0 / lon 0 — the app treats (0,0) as "no coords", it never guesses. -// Names are sanitized to inert plain text (HTML tags/entities stripped). -// -// Run: node scripts/build-heritage-full.mjs -// -// Countries are ISO 3166-1 alpha-2 from the source's iso_codes. The single -// site without any country code (Old City of Jerusalem, id_no 148 — UNESCO -// lists it under "Jerusalem (Site proposed by Jordan)", no ISO code) is -// skipped rather than assigned an invented code. - -import { readFileSync, writeFileSync } from "node:fs"; -import { gunzipSync } from "node:zlib"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const here = dirname(fileURLToPath(import.meta.url)); -const SRC = join(here, "data", "whc-sites-2025.json.gz"); -const OUT = join(here, "..", "public", "reference", "heritage.json"); - -const sites = JSON.parse(gunzipSync(readFileSync(SRC)).toString("utf8")); -if (!Array.isArray(sites) || sites.length < 1100) { - throw new Error(`unexpected source: ${Array.isArray(sites) ? sites.length : typeof sites} entries`); -} - -/** Strip HTML tags/entities the UNESCO feed sometimes embeds (e.g. ) — output must be inert. */ -function sanitizeName(raw) { - return String(raw) - .replace(/<[^>]*>/g, "") - .replace(/&/g, "&") - .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))) - .replace(/&(quot|apos|nbsp|ndash|mdash|rsquo|lsquo|eacute|egrave);/g, (_, e) => - ({ quot: '"', apos: "'", nbsp: " ", ndash: "–", mdash: "—", rsquo: "’", lsquo: "‘", eacute: "é", egrave: "è" })[e], - ) - .replace(/\s+/g, " ") - .trim(); -} - -/** Lowercase ASCII slug of a site name (diacritics folded, non-alphanumerics -> "-"). */ -function slugify(name) { - return name - .normalize("NFKD") - .replace(/[̀-ͯ]/g, "") - .replace(/[øØ]/g, "o") - .replace(/[æÆ]/g, "ae") - .replace(/[œŒ]/g, "oe") - .replace(/[ßẞ]/g, "ss") - .replace(/[đĐðÐ]/g, "d") - .replace(/[þÞ]/g, "th") - .replace(/[łŁ]/g, "l") - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, ""); -} - -const CATEGORY = { Cultural: "cultural", Natural: "natural", Mixed: "mixed" }; - -const out = []; -const skipped = []; -let noCoords = 0; -let transnational = 0; - -for (const s of sites) { - const name = sanitizeName(s.name_en); - const cat = CATEGORY[s.category]; - if (!name || !cat) throw new Error(`site ${s.id_no}: bad name/category ${JSON.stringify(s.category)}`); - - const isos = [...new Set( - String(s.iso_codes ?? "") - .split(",") - .map((c) => c.trim().toUpperCase()) - .filter((c) => /^[A-Z]{2}$/.test(c)), - )]; - if (isos.length === 0) { - skipped.push(`${s.id_no} ${name} (no ISO country code in source)`); - continue; - } - if (isos.length > 1) transnational++; - - const c = s.coordinates; - const hasCoords = c && typeof c.lat === "number" && typeof c.lon === "number" && - Number.isFinite(c.lat) && Number.isFinite(c.lon); - if (!hasCoords) noCoords++; - const lat = hasCoords ? Number(c.lat.toFixed(4)) : 0; - const lon = hasCoords ? Number(c.lon.toFixed(4)) : 0; - - const slug = slugify(name); - for (const iso of isos) { - out.push({ - id: isos.length > 1 ? `whs-${slug}-${iso.toLowerCase()}` : `whs-${slug}`, - name, - countryIso2: iso, - lat, - lon, - category: cat, - idNo: String(s.id_no), // temporary, for collision fixes below - }); - } -} - -// Different sites can slugify identically; disambiguate deterministically with -// the source's stable numeric id (never invented). -const counts = new Map(); -for (const e of out) counts.set(e.id, (counts.get(e.id) ?? 0) + 1); -let collisions = 0; -for (const e of out) { - if (counts.get(e.id) > 1) { - e.id = `${e.id}-${e.idNo}`; - collisions++; - } - delete e.idNo; -} - -out.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); - -// Validate before writing: unique ids, coordinate ranges, ISO codes, categories. -const ids = new Set(); -for (const e of out) { - if (ids.has(e.id)) throw new Error(`duplicate id: ${e.id}`); - ids.add(e.id); - if (!/^whs-[a-z0-9-]+$/.test(e.id)) throw new Error(`bad id: ${e.id}`); - if (!/^[A-Z]{2}$/.test(e.countryIso2)) throw new Error(`bad iso: ${e.id}`); - if (e.lat < -90 || e.lat > 90 || e.lon < -180 || e.lon > 180) throw new Error(`bad coords: ${e.id}`); - if (!["cultural", "natural", "mixed"].includes(e.category)) throw new Error(`bad category: ${e.id}`); - if (!e.name || /[<>]/.test(e.name)) throw new Error(`unsanitized name: ${e.id}`); -} - -writeFileSync(OUT, JSON.stringify(out) + "\n"); - -const perCountry = new Map(); -for (const e of out) perCountry.set(e.countryIso2, (perCountry.get(e.countryIso2) ?? 0) + 1); -const countryCounts = [...perCountry.entries()].sort((a, b) => b[1] - a[1]); -console.log(`wrote ${out.length} entries (${sites.length - skipped.length} sites, ${transnational} transnational) across ${perCountry.size} countries -> ${OUT}`); -console.log(`skipped: ${skipped.length ? skipped.join("; ") : "none"}`); -console.log(`sites without coordinates in source (kept at 0,0): ${noCoords}`); -console.log(`slug collisions disambiguated with UNESCO id_no: ${collisions}`); -console.log(`top countries: ${countryCounts.slice(0, 5).map(([k, v]) => `${k}=${v}`).join(" ")}; min: ${countryCounts.at(-1)?.[1]}`); diff --git a/apps/postcards/scripts/build-languages.mjs b/apps/postcards/scripts/build-languages.mjs deleted file mode 100644 index 5bb8c55..0000000 --- a/apps/postcards/scripts/build-languages.mjs +++ /dev/null @@ -1,41 +0,0 @@ -// Build per-country data from `world-countries` (ODbL): spoken languages (for -// Wikivoyage phrasebook/alphabet links) and the COMMON country name (the correct -// Wikivoyage article title — "Russia", not the ISO-official "Russian Federation"). -// Aggregator-only (Constitution I): we vendor a named, openly-licensed dataset -// with provenance; the app authors nothing. -// -// Outputs: -// public/reference/languages.json { "FR": [{ code: "fra", name: "French" }], ... } -// public/reference/article-names.json { "RU": "Russia", "MD": "Moldova", ... } -// -// Run: node scripts/build-languages.mjs (offline — reads the local npm package) - -import { createRequire } from "node:module"; -import { writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const require = createRequire(import.meta.url); -const countries = require("world-countries"); - -const out = {}; -const names = {}; -for (const c of countries) { - const iso2 = c.cca2; - if (!iso2) continue; - if (c.languages) { - const langs = Object.entries(c.languages).map(([code, name]) => ({ code, name })); - if (langs.length) out[iso2] = langs; - } - if (c.name?.common) names[iso2] = c.name.common; -} - -const dir = dirname(fileURLToPath(import.meta.url)); -const refDir = join(dir, "..", "public", "reference"); -writeFileSync(join(refDir, "languages.json"), JSON.stringify(out)); -writeFileSync(join(refDir, "article-names.json"), JSON.stringify(names)); -const langCount = new Set(Object.values(out).flat().map((l) => l.name)).size; -console.log( - `wrote languages for ${Object.keys(out).length} countries (${langCount} distinct) + ` + - `${Object.keys(names).length} common names -> ${refDir}`, -); diff --git a/apps/postcards/scripts/build-railways.mjs b/apps/postcards/scripts/build-railways.mjs deleted file mode 100644 index 17355ec..0000000 --- a/apps/postcards/scripts/build-railways.mjs +++ /dev/null @@ -1,170 +0,0 @@ -// Build the bundled RAILWAY STATION reference data. -// -// public/reference/railways.json railway stations (id, name, country, coords) -// -// SOURCES (pick with --source=; default: trainline) -// -// trainline Trainline EU open stations database (ODbL). A single, clean CSV of -// ~72k European stations with coordinates and ISO-3166-1 country codes. -// Fetched from GitHub raw, which is reachable from restricted networks -// where the worldwide endpoints below are not. This is the RECOMMENDED -// default bundled with the app. -// https://github.com/trainline-eu/stations -// -// wikidata Worldwide railway stations from Wikidata (CC0) via SPARQL. The most -// portable global source (QID + name + P17 country + P625 coords) but -// query.wikidata.org must be reachable (often blocked). See buildWikidata. -// -// Aggregator-only (Constitution I): this script RESHAPES existing, openly-licensed -// facts into the app's reference shape; it invents nothing. Provenance (named -// dataset + license + date) is written into the emitted file, and the chosen source -// is surfaced in-app so people can see — and swap — which dataset they loaded. -// -// node scripts/build-railways.mjs # trainline (default) -// node scripts/build-railways.mjs --source=wikidata -// -// The file is bundled under public/reference/ like cities/airports and loaded -// through the same reference-data seam. - -import { writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const OUT = join(dirname(fileURLToPath(import.meta.url)), "..", "public", "reference", "railways.json"); -const UA = "Postcards-reference-build/1.0 (local-first travel journal; contact via repo)"; - -const arg = process.argv.find((a) => a.startsWith("--source=")); -const SOURCE = arg ? arg.slice("--source=".length) : "trainline"; - -// ── Trainline (default) ────────────────────────────────────────────────────── -const TRAINLINE_CSV = - "https://raw.githubusercontent.com/trainline-eu/stations/master/stations.csv"; - -/** Parse Trainline's `;`-separated CSV. Fields are unquoted (the dataset uses no - * embedded semicolons in the columns we read), so a plain split per line is safe - * and avoids pulling in a CSV dependency for a build script. */ -function parseTrainline(text) { - const lines = text.split("\n"); - const header = lines[0].split(";"); - const col = (name) => header.indexOf(name); - const iName = col("name"); - const iLat = col("latitude"); - const iLon = col("longitude"); - const iCc = col("country"); - const iSugg = col("is_suggestable"); - const iCity = col("is_city"); - const iAir = col("is_airport"); - const iId = col("id"); - const out = []; - for (let i = 1; i < lines.length; i++) { - const line = lines[i]; - if (!line) continue; - const f = line.split(";"); - // Only real, bookable stations: suggestable, with coordinates, and NOT a - // city-grouping hub (those sit at city centroids and duplicate the city) nor - // an airport (we already bundle airports as their own place kind). - if (f[iSugg] !== "t" || f[iCity] === "t" || f[iAir] === "t") continue; - const lat = Number(f[iLat]); - const lon = Number(f[iLon]); - const cc = (f[iCc] || "").trim().toUpperCase(); - const name = (f[iName] || "").trim(); - if (!name || !/^[A-Z]{2}$/.test(cc) || !Number.isFinite(lat) || !Number.isFinite(lon)) continue; - // id namespaced to the source so it never collides with a QID/GeoNames id. - out.push({ id: `tl-${f[iId]}`, name, countryIso2: cc, subdivisionId: null, lat, lon }); - } - return out; -} - -async function buildTrainline() { - const res = await fetch(TRAINLINE_CSV, { headers: { "User-Agent": UA } }); - if (!res.ok) throw new Error(`Trainline CSV ${res.status} ${res.statusText}`); - const text = await res.text(); - const byId = new Map(); - for (const s of parseTrainline(text)) if (!byId.has(s.id)) byId.set(s.id, s); - const stations = [...byId.values()].sort((a, b) => a.name.localeCompare(b.name)); - return { - stations, - source: { - dataset: "Trainline EU open stations database (railway stations)", - url: "https://github.com/trainline-eu/stations", - license: "ODbL-1.0", - retrieved: new Date().toISOString().slice(0, 10), - note: "Suggestable stations with coordinates; airports and city-grouping hubs excluded. Coverage is Europe-focused.", - coverage: "europe", - }, - }; -} - -// ── Wikidata (worldwide; needs query.wikidata.org reachable) ────────────────── -const ENDPOINT = "https://query.wikidata.org/sparql"; -const PAGE = 5000; -const query = (limit, offset) => ` -SELECT ?station ?stationLabel ?iso2 ?lat ?lon WHERE { - ?station wdt:P31 wd:Q55488 ; - wdt:P625 ?coord ; - wdt:P17 ?country . - ?country wdt:P297 ?iso2 . - FILTER NOT EXISTS { ?station wdt:P576 ?abolished. } - FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915. } - FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883. } - FILTER( EXISTS { ?station wdt:P954 ?uic. } - || EXISTS { ?sl schema:about ?station ; schema:isPartOf ?wiki . - FILTER(CONTAINS(STR(?wiki), "wikipedia.org")) } ) - BIND(geof:latitude(?coord) AS ?lat) - BIND(geof:longitude(?coord) AS ?lon) - SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } -} -ORDER BY ?station -LIMIT ${limit} OFFSET ${offset}`; - -async function buildWikidata() { - const byId = new Map(); - for (let offset = 0; ; offset += PAGE) { - const url = `${ENDPOINT}?format=json&query=${encodeURIComponent(query(PAGE, offset))}`; - const res = await fetch(url, { - headers: { Accept: "application/sparql-results+json", "User-Agent": UA }, - }); - if (!res.ok) throw new Error(`Wikidata SPARQL ${res.status} ${res.statusText}`); - const rows = (await res.json()).results.bindings; - if (!rows.length) break; - for (const r of rows) { - const id = r.station.value.replace("http://www.wikidata.org/entity/", ""); - const name = r.stationLabel?.value?.trim(); - if (!name || name === id || byId.has(id)) continue; - const iso2 = r.iso2.value.toUpperCase(); - const lat = Number(r.lat.value); - const lon = Number(r.lon.value); - if (!/^[A-Z]{2}$/.test(iso2) || !Number.isFinite(lat) || !Number.isFinite(lon)) continue; - byId.set(id, { id, name, countryIso2: iso2, subdivisionId: null, lat, lon }); - } - process.stderr.write(` …${byId.size} stations so far\n`); - if (rows.length < PAGE) break; - } - const stations = [...byId.values()].sort((a, b) => a.name.localeCompare(b.name)); - return { - stations, - source: { - dataset: "Wikidata railway stations (instance of Q55488)", - url: "https://query.wikidata.org/", - license: "CC0-1.0", - retrieved: new Date().toISOString().slice(0, 10), - note: "Mainline stations with coordinates and a Wikipedia sitelink or UIC code; metro/tram excluded.", - coverage: "worldwide", - }, - }; -} - -const BUILDERS = { trainline: buildTrainline, wikidata: buildWikidata }; - -async function main() { - const build = BUILDERS[SOURCE]; - if (!build) throw new Error(`Unknown --source=${SOURCE} (expected: ${Object.keys(BUILDERS).join(", ")})`); - const { stations, source } = await build(); - writeFileSync(OUT, JSON.stringify({ _source: source, stations })); - process.stderr.write(`Wrote ${stations.length} railway stations (${SOURCE}) → ${OUT}\n`); -} - -main().catch((e) => { - process.stderr.write(String(e?.stack || e) + "\n"); - process.exitCode = 1; -}); diff --git a/apps/postcards/scripts/build-reference.mjs b/apps/postcards/scripts/build-reference.mjs deleted file mode 100644 index dca4e96..0000000 --- a/apps/postcards/scripts/build-reference.mjs +++ /dev/null @@ -1,215 +0,0 @@ -// Build the bundled reference data from openly-licensed sources: -// - public/reference/cities.json GeoNames cities >= 15k (all-the-cities, CC BY 4.0) -// - public/reference/subdivisions.json first-level regions (GeoNames admin-1 taxonomy): -// each city's current code and each code's name from -// one GeoNames day (CC BY 4.0), vendored at scripts/data/ -// by build-city-admin1.mjs and admin1CodesASCII.txt. -// - public/reference/airports.json IATA-coded airports (OpenFlights, aggregated from the -// public-domain OurAirports), via `airport-data`. -// A city GeoNames no longer lists takes the region of its nearest city that it does, and a -// code with no GeoNames name keeps one from the dr5hn countries-states-cities dataset -// (ODbL), matched by nearest centroid. Aggregator-only: reshapes existing data. -// -// Run: node scripts/build-reference.mjs (from apps/postcards) -import { createRequire } from "node:module"; -import { readFileSync, writeFileSync } from "node:fs"; -import { gunzipSync } from "node:zlib"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const require = createRequire(import.meta.url); -const allCities = require("all-the-cities"); -const { State } = require("country-state-city"); -const airportData = require("airport-data"); -const countries = require("i18n-iso-countries"); -countries.registerLocale(require("i18n-iso-countries/langs/en.json")); - -// 0 = ship EVERY city in the GeoNames-derived package (~135k) — full world -// coverage (small islands like Lombok included), at the cost of a ~15 MB asset -// (cached once by the service worker). -const MIN_POPULATION = 0; - -// France: GeoNames names four of the 13 metropolitan regions in English -// ("Brittany"), so all 13 take their own French names, by INSEE region code. -const FR_REGION_NAMES = { - 11: "Île-de-France", 24: "Centre-Val de Loire", 27: "Bourgogne-Franche-Comté", - 28: "Normandie", 32: "Hauts-de-France", 44: "Grand Est", 52: "Pays de la Loire", - 53: "Bretagne", 75: "Nouvelle-Aquitaine", 76: "Occitanie", 84: "Auvergne-Rhône-Alpes", - 93: "Provence-Alpes-Côte d'Azur", 94: "Corse", -}; - - -// dr5hn states per country, with usable centroids. -const statesByCountry = new Map(); -function statesOf(cc) { - if (!statesByCountry.has(cc)) { - const list = (State.getStatesOfCountry(cc) || []) - .map((s) => ({ name: s.name, lat: parseFloat(s.latitude), lon: parseFloat(s.longitude) })) - .filter((s) => Number.isFinite(s.lat) && Number.isFinite(s.lon) && !(s.lat === 0 && s.lon === 0)); - statesByCountry.set(cc, list); - } - return statesByCountry.get(cc); -} - -const here = dirname(fileURLToPath(import.meta.url)); -const refDir = join(here, "..", "public", "reference"); - -// GeoNames admin-1 names by "CC-code", from https://download.geonames.org/export/dump/admin1CodesASCII.txt -const ADMIN1_NAMES = new Map( - gunzipSync(readFileSync(join(here, "data", "admin1CodesASCII.txt.gz"))) - .toString("utf8") - .split("\n") - .filter(Boolean) - .map((line) => { - const [code, name] = line.split("\t"); - return [code.replace(".", "-"), name]; - }), -); - -// Each city's admin-1 code on the same GeoNames day as ADMIN1_NAMES, by geonameid. -// all-the-cities carries codes from an older snapshot, and GeoNames has since given -// some of them to other regions (Vietnam's 2025 provinces), so its own are not used. -const CITY_ADMIN1 = new Map( - gunzipSync(readFileSync(join(here, "data", "city-admin1.tsv.gz"))) - .toString("utf8") - .split("\n") - .filter(Boolean) - .map((line) => { - const [id, code] = line.split("\t"); - return [id, code]; - }), -); - -// --- Cities, each with its current region code where GeoNames still lists it --- -const cities = []; -const seen = new Set(); -for (const c of allCities) { - if ((c.population || 0) < MIN_POPULATION || !c.cityId || seen.has(c.cityId)) continue; - const [lon, lat] = c.loc.coordinates; - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; - seen.add(c.cityId); - const current = CITY_ADMIN1.get(String(c.cityId)); - const [cc, admin1] = current ? current.split(".") : []; - cities.push({ - id: String(c.cityId), name: c.name, countryIso2: c.country, - subdivisionId: cc === c.country && admin1 ? `${cc}-${admin1}` : null, - lat: Math.round(lat * 1e4) / 1e4, lon: Math.round(lon * 1e4) / 1e4, - // GeoNames uses 0 for "unknown population" — store null, never a fake 0. - population: c.population > 0 ? c.population : null, - // Only a city GeoNames no longer lists falls back to its nearest neighbour. - unlisted: !current && !!c.adminCode, - }); -} -const coded = new Map(); // country -> cities carrying a current code -for (const c of cities) { - if (!c.subdivisionId) continue; - if (!coded.has(c.countryIso2)) coded.set(c.countryIso2, []); - coded.get(c.countryIso2).push(c); -} -let borrowed = 0; -for (const c of cities) { - if (c.unlisted) { - let best = null, bestD = Infinity; - for (const o of coded.get(c.countryIso2) ?? []) { - const d = (o.lat - c.lat) ** 2 + (o.lon - c.lon) ** 2; - if (d < bestD) { bestD = d; best = o; } - } - if (best) { c.subdivisionId = best.subdivisionId; borrowed++; } - } - delete c.unlisted; -} -console.log(`cities GeoNames no longer lists, placed by nearest neighbour: ${borrowed}`); - -const regions = new Map(); // "CC-code" -> { cc, adminCode, sumLat, sumLon, n } -for (const c of cities) { - if (!c.subdivisionId) continue; - let r = regions.get(c.subdivisionId); - if (!r) { - const adminCode = c.subdivisionId.slice(c.countryIso2.length + 1); - regions.set(c.subdivisionId, (r = { cc: c.countryIso2, adminCode, sumLat: 0, sumLon: 0, n: 0 })); - } - r.sumLat += c.lat; - r.sumLon += c.lon; - r.n++; -} -cities.sort((a, b) => (b.population ?? 0) - (a.population ?? 0)); -// Two tiers: a small CORE file — the world's top CORE_COUNT cities by population — -// that the app bundles + precaches and blocks on at startup, and the FULL world -// gazetteer (~135k) that is downloaded ON DEMAND (like an offline map pack), never -// bundled and never auto-fetched. Keeping only the top 10k in the app keeps the -// install small; the long tail is a one-tap download in Settings. -const CORE_COUNT = 10000; -const core = cities.slice(0, CORE_COUNT); // already population-descending -writeFileSync(join(refDir, "cities.json"), JSON.stringify(core) + "\n"); -writeFileSync(join(refDir, "cities-all.json"), JSON.stringify(cities) + "\n"); -console.log(`core cities: ${core.length} | full: ${cities.length}`); - -// --- Name each region by its GeoNames code, else by nearest state centroid --- -function nearestName(cc, lat, lon) { - let best = null, bestD = Infinity; - for (const s of statesOf(cc)) { - const d = (s.lat - lat) ** 2 + (s.lon - lon) ** 2; - if (d < bestD) { bestD = d; best = s.name; } - } - return best; -} - -let named = 0; -const subdivisions = []; -for (const [id, r] of regions) { - const cLat = r.sumLat / r.n, cLon = r.sumLon / r.n; - const name = - (r.cc === "FR" ? FR_REGION_NAMES[r.adminCode] : undefined) ?? - ADMIN1_NAMES.get(id) ?? - nearestName(r.cc, cLat, cLon); - if (name) named++; - subdivisions.push({ id, countryIso2: r.cc, name: name ?? `${r.cc} region ${r.adminCode}` }); -} -subdivisions.sort((a, b) => a.id.localeCompare(b.id)); -writeFileSync(join(refDir, "subdivisions.json"), JSON.stringify(subdivisions) + "\n"); - -console.log(`cities: ${cities.length} | subdivisions: ${subdivisions.length}`); -console.log(`named: ${named}/${subdivisions.length} (${Math.round((named / subdivisions.length) * 100)}%)`); -console.log(`named by GeoNames code: ${subdivisions.filter((s) => ADMIN1_NAMES.has(s.id)).length}`); -console.log(`countries with regions: ${new Set(subdivisions.map((s) => s.countryIso2)).size}`); -console.log(`Paris subId: ${cities.find((c) => c.id === "2988507")?.subdivisionId}`); - -// --- Airports (IATA-coded, OpenFlights via airport-data) --- -// A few OpenFlights country names differ from ISO 3166 English names; map them to -// alpha-2 explicitly. "Netherlands Antilles" is deliberately absent — the country -// dissolved in 2010 into CW/SX/BQ and the source can't tell us which, so those few -// airports are dropped rather than guessed (Constitution I: never invent). -const AIRPORT_CC_OVERRIDES = { - "Congo (Kinshasa)": "CD", "Congo (Brazzaville)": "CG", Burma: "MM", Laos: "LA", - Syria: "SY", Micronesia: "FM", "Virgin Islands": "VI", "British Virgin Islands": "VG", - "Falkland Islands": "FK", Moldova: "MD", Macedonia: "MK", Swaziland: "SZ", Macau: "MO", - Brunei: "BN", "East Timor": "TL", "Midway Islands": "UM", "Johnston Atoll": "UM", - "Wake Island": "UM", -}; - -const airports = []; -const seenIata = new Set(); -let airportsDropped = 0; -for (const a of airportData) { - const iata = a.iata; - if (!iata || !/^[A-Z]{3}$/.test(iata) || seenIata.has(iata)) continue; - if (!Number.isFinite(a.latitude) || !Number.isFinite(a.longitude)) continue; - const cc = AIRPORT_CC_OVERRIDES[a.country] ?? countries.getAlpha2Code(a.country, "en"); - if (!cc) { - airportsDropped++; - continue; - } - seenIata.add(iata); - airports.push({ - id: iata, - name: a.name, - city: a.city && a.city !== a.name ? a.city : "", - countryIso2: cc, - lat: Math.round(a.latitude * 1e4) / 1e4, - lon: Math.round(a.longitude * 1e4) / 1e4, - }); -} -airports.sort((a, b) => a.id.localeCompare(b.id)); -writeFileSync(join(refDir, "airports.json"), JSON.stringify(airports) + "\n"); -console.log(`airports: ${airports.length} (dropped ${airportsDropped} with unresolved country)`); -console.log(`countries with airports: ${new Set(airports.map((a) => a.countryIso2)).size}`); \ No newline at end of file diff --git a/apps/postcards/scripts/build-sovereignty.mjs b/apps/postcards/scripts/build-sovereignty.mjs deleted file mode 100644 index 90c2cba..0000000 --- a/apps/postcards/scripts/build-sovereignty.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// Classify each ISO 3166-1 entry as a UN member state ("un") or a dependent -// territory ("territory"), from the world-countries dataset (mledoze, ODbL) — -// the same source already used for continents. Aggregator-only (Constitution I): -// we reshape the dataset's own `unMember` flag; we do not decide statehood. -// -// Output: src/lib/reference/data/sovereignty.json { "US": "un", "HK": "territory", ... } -// Run: node scripts/build-sovereignty.mjs (from apps/postcards) -import { createRequire } from "node:module"; -import { writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; - -const require = createRequire(import.meta.url); -const world = require("world-countries"); -const countries = require("i18n-iso-countries"); -countries.registerLocale(require("i18n-iso-countries/langs/en.json")); - -// Corrections against the authoritative UN membership list (193 members). These -// are named facts, not our own judgement: world-countries@5.1.0 marks the Holy -// See (VA) unMember:true, but the Vatican is a permanent observer, not a member -// (https://www.un.org/en/about-us/member-states). We aggregate the UN's own list. -const UN_MEMBER_OVERRIDES = { VA: "territory" }; - -const byCca2 = new Map(world.map((c) => [c.cca2, c])); -const out = {}; -for (const iso2 of Object.keys(countries.getNames("en"))) { - if (!countries.alpha2ToNumeric(iso2)) continue; // keep the same set as buildCountries - const c = byCca2.get(iso2); - out[iso2] = UN_MEMBER_OVERRIDES[iso2] ?? (c && c.unMember ? "un" : "territory"); -} - -const dir = dirname(fileURLToPath(import.meta.url)); -const path = join(dir, "..", "src", "lib", "reference", "data", "sovereignty.json"); -writeFileSync(path, JSON.stringify(out) + "\n"); -const un = Object.values(out).filter((v) => v === "un").length; -console.log(`wrote ${Object.keys(out).length} countries (${un} UN, ${Object.keys(out).length - un} territories) -> ${path}`); diff --git a/apps/postcards/scripts/data/admin1CodesASCII.txt.gz b/apps/postcards/scripts/data/admin1CodesASCII.txt.gz deleted file mode 100644 index 4e3d3ef..0000000 Binary files a/apps/postcards/scripts/data/admin1CodesASCII.txt.gz and /dev/null differ diff --git a/apps/postcards/scripts/data/city-admin1.tsv.gz b/apps/postcards/scripts/data/city-admin1.tsv.gz deleted file mode 100644 index 38f2b29..0000000 Binary files a/apps/postcards/scripts/data/city-admin1.tsv.gz and /dev/null differ diff --git a/apps/postcards/scripts/data/whc-sites-2025.json.gz b/apps/postcards/scripts/data/whc-sites-2025.json.gz deleted file mode 100644 index 6d08097..0000000 Binary files a/apps/postcards/scripts/data/whc-sites-2025.json.gz and /dev/null differ diff --git a/apps/postcards/scripts/fetch-data.mjs b/apps/postcards/scripts/fetch-data.mjs new file mode 100644 index 0000000..cad4ae7 --- /dev/null +++ b/apps/postcards/scripts/fetch-data.mjs @@ -0,0 +1,145 @@ +// Fetch Offware's shared data (https://github.com/offware-apps/data) into the bundled +// reference files, at build time. Every dataset the app ships comes from there: +// +// public/reference/railways-wikidata.json stations/railways.json Wikidata, CC0 +// public/reference/landmarks.json places/landmarks.json Wikidata, CC0 +// public/reference/railways.json stations-europe/railways.json Trainline, ODbL +// public/reference/cities.json cities/top.json GeoNames, CC BY 4.0 +// public/reference/cities-all.json cities/all.json GeoNames, CC BY 4.0 +// public/reference/subdivisions.json regions/regions.json GeoNames + dr5hn, ODbL +// public/reference/airports.json airports/airports.json OpenFlights, ODbL +// public/reference/heritage.json heritage/unesco.json UNESCO, CC BY-SA 3.0 IGO +// public/reference/languages.json countries/languages.json world-countries, ODbL +// public/reference/article-names.json countries/article-names.json world-countries, ODbL +// public/reference/experiences.json moments/moments.json per item, as cited +// src/lib/reference/data/country-names.json countries/names.json world-countries, ODbL +// src/lib/reference/data/continents.json countries/continents.json world-countries, ODbL +// src/lib/reference/data/sovereignty.json countries/sovereignty.json world-countries, ODbL +// +// Aggregator-only (Constitution I): the shared data reshapes named, openly-licensed +// datasets; this script only downloads and checks it. A file is written only when its +// dataset still carries the licence the app's provenance names (catalog.json), and its +// records have the shape the app reads. Otherwise the committed copy is kept, so the app +// always builds and works offline. With no worldwide stations file at all the script +// fails: they're the default station source, and a build without them would leave every +// new user with no stations. +// +// node scripts/fetch-data.mjs +// DATA_BASE=https://example.org/data/v1 node scripts/fetch-data.mjs + +import { existsSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const BASE = process.env.DATA_BASE ?? process.env.PLACES_BASE ?? "https://offware-apps.github.io/data/v1"; +const APP = join(dirname(fileURLToPath(import.meta.url)), ".."); +const REF = join(APP, "public", "reference"); +const DATA = join(APP, "src", "lib", "reference", "data"); +const LANDMARK_MAX = 10_000; +const UA = "Postcards-reference-build/1.0 (https://github.com/offware-apps/Postcards)"; + +/** The licence each dataset must still carry: the one the app's provenance names. */ +const LICENSES = { + stations: "CC0-1.0", + places: "CC0-1.0", + "stations-europe": "ODbL-1.0", + cities: "CC-BY-4.0", + regions: "ODbL-1.0", + airports: "ODbL-1.0", + heritage: "CC-BY-SA-3.0-IGO", + countries: "ODbL-1.0", + moments: "CC0-1.0 AND CC-BY-4.0 AND CC-BY-SA-3.0-IGO AND CC-BY-SA-4.0", +}; + +const finite = (n, max) => typeof n === "number" && Number.isFinite(n) && Math.abs(n) <= max; +const iso2 = (s) => typeof s === "string" && /^[A-Z]{2}$/.test(s); +const text = (s) => typeof s === "string" && s.trim() !== ""; +const validPoint = (p) => p && text(p.id) && text(p.name) && iso2(p.countryIso2) && finite(p.lat, 90) && finite(p.lon, 180); + +async function get(path) { + const res = await fetch(`${BASE}/${path}`, { headers: { "User-Agent": UA }, signal: AbortSignal.timeout(120_000) }); + if (!res.ok) throw new Error(`${path}: HTTP ${res.status}`); + return res.json(); +} + +/** A list of `item`s, all valid, at least `min` of them. */ +function list(json, min, valid) { + if (!Array.isArray(json)) throw new Error("not a list"); + if (json.length < min || !json.every(valid)) throw new Error(`${json.length} records, or some invalid`); + return json; +} + +/** An object keyed by country code, at least `min` keys, every value valid. */ +function byCountry(json, min, valid) { + if (!json || typeof json !== "object" || Array.isArray(json)) throw new Error("not an object"); + const entries = Object.entries(json); + if (entries.length < min || !entries.every(([k, v]) => iso2(k) && valid(v))) throw new Error(`${entries.length} countries, or some invalid`); + return json; +} + +function stationsFile(json, license, min) { + if (json?._source?.license !== license || !Array.isArray(json.stations)) throw new Error("unexpected shape or licence"); + const stations = json.stations.filter(validPoint); + if (stations.length < min) throw new Error(`only ${stations.length} valid stations`); + return { _source: json._source, stations }; +} + +function landmarks(json) { + // The best known LANDMARK_MAX, so the file stays small enough to load at start-up. + // Postcards' landmarks are one category; the dataset's finer kinds stay out of the app. + const out = list(json, 100, () => true) + .filter(validPoint) + .sort((a, b) => (b.sitelinks ?? 0) - (a.sitelinks ?? 0) || a.id.localeCompare(b.id)) + .slice(0, LANDMARK_MAX) + .map(({ id, name, countryIso2, lat, lon }) => ({ id, name, countryIso2, lat, lon, category: "landmark" })) + .sort((a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id)); + if (out.length < 100) throw new Error(`only ${out.length} valid landmarks`); + return out; +} + +const city = (c) => validPoint(c) && (c.population === null || finite(c.population, 1e9)) && (c.subdivisionId === null || text(c.subdivisionId)); + +/** [dataset, path in the shared data, file written, check and reshape]. */ +const FILES = [ + ["stations", "stations/railways.json", join(REF, "railways-wikidata.json"), (j) => stationsFile(j, "CC0-1.0", 1000)], + ["places", "places/landmarks.json", join(REF, "landmarks.json"), landmarks], + ["stations-europe", "stations-europe/railways.json", join(REF, "railways.json"), (j) => stationsFile(j, "ODbL-1.0", 10_000)], + ["cities", "cities/top.json", join(REF, "cities.json"), (j) => list(j, 5000, city)], + ["cities", "cities/all.json", join(REF, "cities-all.json"), (j) => list(j, 100_000, city)], + ["regions", "regions/regions.json", join(REF, "subdivisions.json"), (j) => list(j, 1000, (r) => text(r.id) && iso2(r.countryIso2) && text(r.name))], + ["airports", "airports/airports.json", join(REF, "airports.json"), (j) => list(j, 3000, (a) => validPoint(a) && /^[A-Z]{3}$/.test(a.id) && typeof a.city === "string")], + ["heritage", "heritage/unesco.json", join(REF, "heritage.json"), (j) => list(j, 1000, (h) => validPoint(h) && ["cultural", "natural", "mixed"].includes(h.category))], + ["countries", "countries/languages.json", join(REF, "languages.json"), (j) => byCountry(j, 200, (v) => Array.isArray(v) && v.every((l) => text(l.code) && text(l.name)))], + ["countries", "countries/article-names.json", join(REF, "article-names.json"), (j) => byCountry(j, 200, text)], + ["countries", "countries/names.json", join(DATA, "country-names.json"), (j) => byCountry(j, 200, text)], + ["countries", "countries/continents.json", join(DATA, "continents.json"), (j) => byCountry(j, 200, text)], + ["countries", "countries/sovereignty.json", join(DATA, "sovereignty.json"), (j) => byCountry(j, 240, (v) => v === "un" || v === "territory")], + ["moments", "moments/moments.json", join(REF, "experiences.json"), (j) => { + if (!Array.isArray(j?.experiences) || j.experiences.length < 50) throw new Error("no experiences"); + return j; + }], +]; + +let catalog = null; +try { + catalog = await get("catalog.json"); +} catch (e) { + process.stderr.write(`data: catalog not read, every file kept as it was (${e?.message ?? e})\n`); +} +const licenseOf = new Map((catalog?.datasets ?? []).map((d) => [d.id, d.license])); + +for (const [dataset, path, out, check] of FILES) { + try { + if (licenseOf.get(dataset) !== LICENSES[dataset]) throw new Error(`licence is ${licenseOf.get(dataset)}, provenance names ${LICENSES[dataset]}`); + const body = check(await get(path)); + writeFileSync(out, JSON.stringify(body) + "\n"); + process.stderr.write(`data: ${path} written\n`); + } catch (e) { + process.stderr.write(`data: ${path} kept as it was (${e?.message ?? e})\n`); + } +} + +if (!existsSync(join(REF, "railways-wikidata.json"))) { + process.stderr.write("data: no worldwide railway stations file, the default station source would be empty\n"); + process.exit(1); +} diff --git a/apps/postcards/src/lib/i18n/en.ts b/apps/postcards/src/lib/i18n/en.ts index f0cfc68..91fe2b8 100644 --- a/apps/postcards/src/lib/i18n/en.ts +++ b/apps/postcards/src/lib/i18n/en.ts @@ -84,10 +84,13 @@ export const en = { "Used everywhere a country is counted: totals, the passport, per-continent progress.", "settings.stations.title": "Railway stations", "settings.stations.recommended": "recommended", + "settings.stations.source.wikidata": "World — Wikidata", "settings.stations.source.trainline": "Europe — Trainline", "settings.stations.source.none": "None", + "settings.stations.desc.wikidata": + "Open railway stations worldwide from Wikidata (CC0), via Offware's shared data. Each station links back to its Wikidata item, where anyone can correct it.", "settings.stations.desc.trainline": - "~44,000 European stations from Trainline's open database (ODbL). The only worldwide-open set that loads on this build; a global set (Wikidata/OSM) can be added later.", + "~44,000 European stations from Trainline's open database (ODbL). Choose it to keep stations you marked from this list before the worldwide set arrived.", "settings.stations.desc.none": "No railway stations are loaded — they won't appear in search, on the map, or in stats.", "settings.map.advanced": "More map options & offline packs", "settings.optimize.title": "Optimisation", diff --git a/apps/postcards/src/lib/i18n/fr.ts b/apps/postcards/src/lib/i18n/fr.ts index 094153b..f15cbd9 100644 --- a/apps/postcards/src/lib/i18n/fr.ts +++ b/apps/postcards/src/lib/i18n/fr.ts @@ -78,10 +78,13 @@ export const fr: Messages = { "Utilisé partout où un pays est compté : totaux, passeport, progression par continent.", "settings.stations.title": "Gares ferroviaires", "settings.stations.recommended": "recommandé", + "settings.stations.source.wikidata": "Monde — Wikidata", "settings.stations.source.trainline": "Europe — Trainline", "settings.stations.source.none": "Aucune", + "settings.stations.desc.wikidata": + "Les gares ferroviaires ouvertes du monde entier, issues de Wikidata (CC0) via les données partagées d'Offware. Chaque gare renvoie à sa fiche Wikidata, que chacun peut corriger.", "settings.stations.desc.trainline": - "~44 000 gares européennes issues de la base ouverte de Trainline (ODbL). Le seul jeu ouvert disponible sur cette version ; un jeu mondial (Wikidata/OSM) pourra être ajouté plus tard.", + "~44 000 gares européennes issues de la base ouverte de Trainline (ODbL). Choisissez-la pour garder les gares marquées depuis cette liste avant l'arrivée du jeu mondial.", "settings.stations.desc.none": "Aucune gare n'est chargée — elles n'apparaîtront ni dans la recherche, ni sur la carte, ni dans les stats.", "settings.map.advanced": "Plus d'options de carte et cartes hors ligne", "settings.optimize.title": "Optimisation", diff --git a/apps/postcards/src/lib/i18n/ko.ts b/apps/postcards/src/lib/i18n/ko.ts index 0956acc..4d2c75b 100644 --- a/apps/postcards/src/lib/i18n/ko.ts +++ b/apps/postcards/src/lib/i18n/ko.ts @@ -78,10 +78,13 @@ export const ko: Messages = { "국가를 세는 모든 곳에 사용됩니다: 합계, 여권, 대륙별 진행률.", "settings.stations.title": "기차역", "settings.stations.recommended": "권장", + "settings.stations.source.wikidata": "전 세계 — Wikidata", "settings.stations.source.trainline": "유럽 — Trainline", "settings.stations.source.none": "없음", + "settings.stations.desc.wikidata": + "Offware 공유 데이터를 통해 Wikidata(CC0)에서 가져온 전 세계 운행 중인 기차역. 각 역은 누구나 수정할 수 있는 Wikidata 항목으로 연결됩니다.", "settings.stations.desc.trainline": - "Trainline 공개 데이터베이스(ODbL)의 유럽 기차역 약 44,000곳. 이 빌드에서 불러올 수 있는 유일한 공개 데이터이며, 전 세계 데이터(Wikidata/OSM)는 나중에 추가할 수 있습니다.", + "Trainline 공개 데이터베이스(ODbL)의 유럽 기차역 약 44,000곳. 전 세계 데이터가 추가되기 전에 이 목록에서 표시한 역을 유지하려면 선택하세요.", "settings.stations.desc.none": "기차역을 불러오지 않습니다 — 검색, 지도, 통계에 나타나지 않습니다.", "settings.map.advanced": "추가 지도 옵션 및 오프라인 팩", "settings.optimize.title": "최적화", diff --git a/apps/postcards/src/lib/reference/data/provenance.json b/apps/postcards/src/lib/reference/data/provenance.json index bdd007a..8c50304 100644 --- a/apps/postcards/src/lib/reference/data/provenance.json +++ b/apps/postcards/src/lib/reference/data/provenance.json @@ -30,9 +30,15 @@ "url": "https://openflights.org/data.html" }, { - "dataset": "Railway stations — Trainline EU open stations database (choosable in Settings)", + "dataset": "Railway stations worldwide — Wikidata via Offware shared data (default; choosable in Settings)", + "license": "CC0-1.0", + "version": "fetched at build time", + "url": "https://github.com/offware-apps/data" + }, + { + "dataset": "Railway stations — Trainline EU open stations database (Europe; choosable in Settings)", "license": "ODbL-1.0", - "version": "trainline · worldwide: --source=wikidata", + "version": "trainline", "url": "https://github.com/trainline-eu/stations" }, { @@ -54,10 +60,10 @@ "url": "https://en.wikivoyage.org" }, { - "dataset": "Famous landmarks seed (full set via Wikidata build script)", - "license": "CC0 (facts)", - "version": "seed", - "url": "https://www.wikidata.org" + "dataset": "Landmarks — the 10,000 best-known sights (Wikidata, via Offware's shared data)", + "license": "CC0-1.0", + "version": "fetched at build time", + "url": "https://github.com/offware-apps/data" }, { "dataset": "UNESCO World Heritage List — all 1,248 sites incl. 47th session (2025)", @@ -76,5 +82,11 @@ "license": "CC BY-SA 3.0", "version": "live", "url": "https://en.wikivoyage.org" + }, + { + "dataset": "Offware shared data: where the reference files above are built and fetched from", + "license": "Each dataset keeps its source's licence", + "version": "fetched at build time", + "url": "https://github.com/offware-apps/data" } ] \ No newline at end of file diff --git a/apps/postcards/src/lib/reference/referenceData.ts b/apps/postcards/src/lib/reference/referenceData.ts index 928a041..9b8a413 100644 --- a/apps/postcards/src/lib/reference/referenceData.ts +++ b/apps/postcards/src/lib/reference/referenceData.ts @@ -482,7 +482,7 @@ async function loadExtras(impl: ReferenceDataImpl): Promise { .then((r) => (r.ok ? (r.json() as Promise) : empty)) .catch(() => empty); // Railway stations: whichever dataset the user chose in Settings (default - // Trainline; "None" loads nothing). A { _source, stations:[…] } wrapper; + // Wikidata, worldwide; "None" loads nothing). A { _source, stations:[…] } wrapper; // absent file → []. Switching source later re-fetches via setStationData. // The biggest file here (5 MB, a long parse on a phone), so it waits for a // screen to ask (requestStations): on the map, once the map has loaded, so it diff --git a/apps/postcards/src/lib/reference/stationSources.ts b/apps/postcards/src/lib/reference/stationSources.ts index f1a210d..7f3c75f 100644 --- a/apps/postcards/src/lib/reference/stationSources.ts +++ b/apps/postcards/src/lib/reference/stationSources.ts @@ -3,15 +3,14 @@ // aggregator, never author) bundled as its own file under public/reference/; the // user picks which one they load — or "None" to carry none at all. // -// This catalogue is DATA-DRIVEN: the Settings picker lists exactly what's here, so -// adding a worldwide build output (e.g. railways-wikidata.json from -// `node scripts/build-railways.mjs --source=wikidata`) is a one-line addition that -// automatically shows up as a selectable source — no UI change needed. +// This catalogue is DATA-DRIVEN: the Settings picker lists exactly what's here. +// The worldwide set comes from Offware's shared data (Wikidata, CC0), +// fetched at build time by `scripts/fetch-data.mjs`. const BASE = import.meta.env.BASE_URL; /** Stable ids, persisted in localStorage — never renumber. "none" loads nothing. */ -export type StationSourceId = "trainline" | "none"; +export type StationSourceId = "wikidata" | "trainline" | "none"; export interface StationSourceDef { id: StationSourceId; @@ -24,23 +23,34 @@ export interface StationSourceDef { recommended: boolean; } -// Trainline (ODbL, Europe) is the recommended default — the only full open station -// set reachable from the build sandbox. A future worldwide file (Wikidata CC0 / -// OSM ODbL), produced by the build script on an open network, slots in here. +// Wikidata (CC0, worldwide) is the recommended default. Trainline (ODbL, Europe) +// stays selectable: its station ids differ, so people who marked stations from it +// keep it until they choose otherwise (see loadStationSource). export const STATION_SOURCES: readonly StationSourceDef[] = [ + { + id: "wikidata", + url: `${BASE}reference/railways-wikidata.json`, + license: "CC0-1.0", + coverage: "worldwide", + recommended: true, + }, { id: "trainline", url: `${BASE}reference/railways.json`, license: "ODbL-1.0", coverage: "europe", - recommended: true, + recommended: false, }, { id: "none", url: null, license: "", coverage: "none", recommended: false }, ]; -export const DEFAULT_STATION_SOURCE: StationSourceId = "trainline"; +export const DEFAULT_STATION_SOURCE: StationSourceId = "wikidata"; const STATION_SOURCE_KEY = "postcards-station-source"; +// Set once the app has been opened before (App.tsx). Before the worldwide set, +// Trainline was everyone's default, so an earlier user who never chose keeps it: +// their marked stations carry Trainline ids, which the worldwide set doesn't have. +const RETURNING_USER_KEY = "postcards-intro-seen"; export function isStationSource(v: unknown): v is StationSourceId { return STATION_SOURCES.some((s) => s.id === v); @@ -55,7 +65,12 @@ export function stationSourceById(id: StationSourceId): StationSourceDef { export function loadStationSource(): StationSourceId { try { const v = localStorage.getItem(STATION_SOURCE_KEY); - return isStationSource(v) ? v : DEFAULT_STATION_SOURCE; + if (isStationSource(v)) return v; + // Either way the choice is saved now: a new user's intro flag, set on this + // first run, must not turn them into a "returning" Trainline user next time. + const choice: StationSourceId = localStorage.getItem(RETURNING_USER_KEY) != null ? "trainline" : DEFAULT_STATION_SOURCE; + localStorage.setItem(STATION_SOURCE_KEY, choice); + return choice; } catch { return DEFAULT_STATION_SOURCE; } diff --git a/apps/postcards/tests/unit/gazetteer.spec.ts b/apps/postcards/tests/unit/gazetteer.spec.ts index 3144dd8..c916d66 100644 --- a/apps/postcards/tests/unit/gazetteer.spec.ts +++ b/apps/postcards/tests/unit/gazetteer.spec.ts @@ -1,12 +1,7 @@ import { describe, it, expect } from "vitest"; -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; -import { gunzipSync } from "node:zlib"; import { getReferenceData } from "../../src/lib/reference/referenceData"; const ref = getReferenceData(); -const here = dirname(fileURLToPath(import.meta.url)); describe("bundled gazetteer integrity (full GeoNames-derived world gazetteer)", () => { const cities = ref.allCities(); @@ -79,38 +74,8 @@ describe("bundled gazetteer integrity (full GeoNames-derived world gazetteer)", expect(tokyoRegion?.name.includes(" region ")).toBe(false); }); - // Both vendored files are one GeoNames day: each city's region code and each code's - // name. Mixing days put Hanoi under "Quang Tri" when Vietnam renumbered in 2025. - const vendored = (file: string) => - new Map( - gunzipSync(readFileSync(join(here, "..", "..", "scripts", "data", file))) - .toString("utf8") - .split("\n") - .filter(Boolean) - .map((line) => line.split("\t") as [string, string]), - ); - - it("gives every city GeoNames lists the region code it lists today", () => { - const current = vendored("city-admin1.tsv.gz"); - const wrong = cities - .filter((c) => { - const code = current.get(c.id); - const listed = code?.startsWith(`${c.countryIso2}.`) && !code.endsWith("."); - return listed && c.subdivisionId !== code!.replace(".", "-"); - }) - .map((c) => `${c.name} ${c.id}: ${c.subdivisionId}`); - expect(wrong).toEqual([]); - }); - - it("names every region by GeoNames' name for its code, France in French", () => { - const names = vendored("admin1CodesASCII.txt.gz"); - const wrong = [...names] - .map(([code, name]) => ({ id: code.replace(".", "-"), name })) - .map((s) => ({ ...s, ours: ref.subdivisionById(s.id)?.name })) - .filter((s) => s.ours !== undefined && !s.id.startsWith("FR-") && s.ours !== s.name) - .map((s) => `${s.id}: ${s.ours}`); - expect(wrong).toEqual([]); - }); + // That each city carries the region code GeoNames lists today, and each region GeoNames' + // name for it, is checked where the files are built: offware-apps/data. it("puts well-known cities in their own region", () => { const regionOf = (name: string, cc: string) => { diff --git a/apps/postcards/tests/unit/heritage.spec.ts b/apps/postcards/tests/unit/heritage.spec.ts index 656266f..513bcbc 100644 --- a/apps/postcards/tests/unit/heritage.spec.ts +++ b/apps/postcards/tests/unit/heritage.spec.ts @@ -11,7 +11,7 @@ import type { Visit } from "../../src/lib/schema/models"; // The shipped heritage.json is the FULL UNESCO World Heritage List (all ~1,248 // sites, one entry per country for transnational sites), built by -// scripts/build-heritage-full.mjs from the official UNESCO data (CC BY-SA 3.0 IGO). +// Offware's shared data (offware-apps/data) from the official UNESCO data (CC BY-SA 3.0 IGO). // The logic tests below still use small injected data so the wiring is verified // independently of the shipped file; the bundled file itself is validated at the end. const heritage: HeritageSite[] = [ diff --git a/apps/postcards/tests/unit/referenceStartup.spec.ts b/apps/postcards/tests/unit/referenceStartup.spec.ts index 83be1ee..cd3e8b3 100644 --- a/apps/postcards/tests/unit/referenceStartup.spec.ts +++ b/apps/postcards/tests/unit/referenceStartup.spec.ts @@ -39,7 +39,7 @@ describe("reference data at startup", () => { // The stations, the biggest file, are not asked for until a screen asks // (the map once it has loaded), so they never compete with it. - expect(late["railways.json"]).toBeUndefined(); + expect(late["railways-wikidata.json"]).toBeUndefined(); const gen = gazetteerGeneration(); const next = () => @@ -58,11 +58,11 @@ describe("reference data at startup", () => { // Not on idle either: only on request. await new Promise((r) => setTimeout(r, 1600)); - expect(late["railways.json"]).toBeUndefined(); + expect(late["railways-wikidata.json"]).toBeUndefined(); requestStations(); - await vi.waitFor(() => expect(late["railways.json"]).toBeDefined(), { timeout: 3000 }); + await vi.waitFor(() => expect(late["railways-wikidata.json"]).toBeDefined(), { timeout: 3000 }); landed = next(); - late["railways.json"]!({ + late["railways-wikidata.json"]!({ stations: [{ id: "Q1", name: "Lisboa Oriente", countryIso2: "PT", lat: 38.77, lon: -9.1 }], }); await landed; @@ -94,9 +94,9 @@ describe("reference data at startup", () => { late["languages.json"]!({}); late["article-names.json"]!({}); requestStations(); - await vi.waitFor(() => expect(late["railways.json"]).toBeDefined(), { timeout: 3000 }); + await vi.waitFor(() => expect(late["railways-wikidata.json"]).toBeDefined(), { timeout: 3000 }); const landed = new Promise((r) => window.addEventListener(GAZETTEER_UPGRADED_EVENT, r, { once: true })); - late["railways.json"]!({ + late["railways-wikidata.json"]!({ stations: [{ id: "Q1", name: "Lisboa Oriente", countryIso2: "PT", lat: 38.77, lon: -9.1 }], }); await landed; diff --git a/apps/postcards/tests/unit/stations.spec.ts b/apps/postcards/tests/unit/stations.spec.ts index b79e2c7..0fcf893 100644 --- a/apps/postcards/tests/unit/stations.spec.ts +++ b/apps/postcards/tests/unit/stations.spec.ts @@ -1,10 +1,11 @@ -import { describe, it, expect } from "vitest"; +import { beforeEach, describe, it, expect } from "vitest"; import { initReferenceDataSync, setStationData } from "../../src/lib/reference/referenceData"; import type { Station } from "../../src/lib/reference/types"; import { STATION_SOURCES, DEFAULT_STATION_SOURCE, isStationSource, + loadStationSource, stationSourceById, } from "../../src/lib/reference/stationSources"; import { PlaceRefSchema } from "../../src/lib/schema/models"; @@ -71,6 +72,11 @@ describe("station data sources (Settings choice)", () => { expect(STATION_SOURCES.filter((s) => s.recommended)).toHaveLength(1); }); + it("defaults to the worldwide Wikidata set", () => { + expect(DEFAULT_STATION_SOURCE).toBe("wikidata"); + expect(stationSourceById("wikidata")).toMatchObject({ coverage: "worldwide", license: "CC0-1.0" }); + }); + it("rejects unknown source ids", () => { expect(isStationSource("osm")).toBe(false); expect(isStationSource(null)).toBe(false); @@ -88,3 +94,24 @@ describe("station data sources (Settings choice)", () => { expect(ref.allStations()).toEqual([]); }); }); + +describe("the station source a person starts with", () => { + beforeEach(() => localStorage.clear()); + + it("gives a new user the worldwide set, and keeps it after the first run", () => { + expect(loadStationSource()).toBe("wikidata"); + localStorage.setItem("postcards-intro-seen", "1"); // set at the end of the first run + expect(loadStationSource()).toBe("wikidata"); + }); + + it("keeps an earlier user on Trainline, whose station ids their visits use", () => { + localStorage.setItem("postcards-intro-seen", "1"); + expect(loadStationSource()).toBe("trainline"); + }); + + it("never overrides a choice made in Settings", () => { + localStorage.setItem("postcards-intro-seen", "1"); + localStorage.setItem("postcards-station-source", "wikidata"); + expect(loadStationSource()).toBe("wikidata"); + }); +}); diff --git a/apps/postcards/vite.config.ts b/apps/postcards/vite.config.ts index 40403d6..aaff671 100644 --- a/apps/postcards/vite.config.ts +++ b/apps/postcards/vite.config.ts @@ -76,7 +76,9 @@ export default defineConfig({ // top-10k core and downloads the full set ON DEMAND (a one-tap Settings // action, like a tile pack). Runtime-cached below so it reopens offline // once fetched. Never bundled — that would defeat the small-install goal. - globIgnores: ["**/reference/cities-all.json"], + // The worldwide stations (fetched at build time from the places dataset) can + // outgrow the precache limit; they're runtime-cached below instead. + globIgnores: ["**/reference/cities-all.json", "**/reference/railways-wikidata.json"], maximumFileSizeToCacheInBytes: 8 * 1024 * 1024, // Control the page on the very first visit, so a tile pack or the full // city list downloaded in that session reaches the runtime caches below @@ -100,6 +102,17 @@ export default defineConfig({ cacheableResponse: { statuses: [200] }, }, }, + { + // Worldwide railway stations: cached when the map first asks for them, + // so they reopen offline; refreshed in the background after a release. + urlPattern: ({ url }) => url.pathname.endsWith("/reference/railways-wikidata.json"), + handler: "StaleWhileRevalidate", + options: { + cacheName: "stations-v1", + expiration: { maxEntries: 2 }, + cacheableResponse: { statuses: [200] }, + }, + }, { urlPattern: ({ url }) => url.hostname.endsWith("tile.openstreetmap.org"), handler: "CacheFirst", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7840096..5a263d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,15 +98,6 @@ importers: '@vitejs/plugin-react': specifier: ^6.0.3 version: 6.0.3(vite@8.1.3(@types/node@26.1.1)(terser@5.48.0)) - airport-data: - specifier: ^1.0.1 - version: 1.0.1 - all-the-cities: - specifier: ^3.1.0 - version: 3.1.0 - country-state-city: - specifier: ^3.2.1 - version: 3.2.1 jsdom: specifier: ^29.1.1 version: 29.1.1 @@ -125,9 +116,6 @@ importers: vitest: specifier: ^4.1.10 version: 4.1.10(@types/node@26.1.1)(jsdom@29.1.1)(vite@8.1.3(@types/node@26.1.1)(terser@5.48.0)) - world-countries: - specifier: ^5.1.0 - version: 5.1.0 packages: @@ -1297,15 +1285,9 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - airport-data@1.0.1: - resolution: {integrity: sha512-Y3k2q5I/CwLKW1mul7ECRQQSEQpBm5jebmgutjO/DLbnBp6CG7jaKD58uu8O8HS0oip23TurO6czXBh5Cet9zg==} - ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} - all-the-cities@3.1.0: - resolution: {integrity: sha512-Jx+mZR7lUVSX+qk785T3MRkAJYMvUpOmeNQSCbUl1WcCNpBUKJsQJ6uZzOl3/bWXVIUx1UhDFs8ASDHSI3rvzg==} - ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -1466,9 +1448,6 @@ packages: core-js-compat@3.49.0: resolution: {integrity: sha512-VQXt1jr9cBz03b331DFDCCP90b3fanciLkgiOoy8SBHy06gNf+vQ1A3WFLqG7I8TipYIKeYK9wxd0tUrvHcOZA==} - country-state-city@3.2.1: - resolution: {integrity: sha512-kxbanqMc6izjhc/EHkGPCTabSPZ2G6eG4/97akAYHJUN4stzzFEvQPZoF8oXDQ+10gM/O/yUmISCR1ZVxyb6EA==} - cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -1782,9 +1761,6 @@ packages: idb@8.0.3: resolution: {integrity: sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==} - ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - indent-string@4.0.0: resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} engines: {node: '>=8'} @@ -2208,10 +2184,6 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pbf@3.3.0: - resolution: {integrity: sha512-XDF38WCH3z5OV/OVa8GKUNtLAyneuzbCisx7QUCF8Q6Nutx0WnJrQe5O+kOtBlLfRNUws98Y58Lblp+NJG5T4Q==} - hasBin: true - pbf@4.0.2: resolution: {integrity: sha512-J0ajxARhZfpUEebxYs1vhMGMuLSXtBe1e+fFPDrf2uA2hgo+UshKfNUWOz92HJNz6/NFEXseQPddnHkTreWRqg==} hasBin: true @@ -2878,9 +2850,6 @@ packages: world-atlas@2.0.2: resolution: {integrity: sha512-IXfV0qwlKXpckz1FhwXVwKRjiIhOnWttOskm5CtxMsjgE/MXAYRHWJqgXOpM8IkcPBoXnyTU5lFHcYa5ChG0LQ==} - world-countries@5.1.0: - resolution: {integrity: sha512-CXR6EBvTbArDlDDIWU3gfKb7Qk0ck2WNZ234b/A0vuecPzIfzzxH+O6Ejnvg1sT8XuiZjVlzOH0h08ZtaO7g0w==} - wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -4240,8 +4209,6 @@ snapshots: acorn@8.17.0: {} - airport-data@1.0.1: {} - ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 @@ -4249,10 +4216,6 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - all-the-cities@3.1.0: - dependencies: - pbf: 3.3.0 - ansi-regex@5.0.1: {} ansi-styles@4.3.0: @@ -4401,8 +4364,6 @@ snapshots: dependencies: browserslist: 4.28.5 - country-state-city@3.2.1: {} - cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -4760,8 +4721,6 @@ snapshots: idb@8.0.3: {} - ieee754@1.2.1: {} - indent-string@4.0.0: {} inherits@2.0.4: {} @@ -5161,11 +5120,6 @@ snapshots: pathe@2.0.3: {} - pbf@3.3.0: - dependencies: - ieee754: 1.2.1 - resolve-protobuf-schema: 2.1.0 - pbf@4.0.2: dependencies: resolve-protobuf-schema: 2.1.0 @@ -5949,8 +5903,6 @@ snapshots: world-atlas@2.0.2: {} - world-countries@5.1.0: {} - wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 diff --git a/specs/022-shared-places/plan.md b/specs/022-shared-places/plan.md new file mode 100644 index 0000000..b34d02b --- /dev/null +++ b/specs/022-shared-places/plan.md @@ -0,0 +1,30 @@ +# Implementation Plan: Worldwide stations and landmarks from Offware's shared data + +**Spec**: `specs/022-shared-places/spec.md` + +## Constitution Check + +- **I. Aggregator, never an author**: pass. Data comes from Wikidata (CC0) through the places + dataset; ids are Wikidata items; provenance updated. +- **Local-first / offline**: pass. Files are bundled at build time; stations runtime-cached. +- **Privacy**: pass. The fetch happens in CI at build time, never from a user's device. +- **Data is inert**: pass. The build script validates shape, licence and coordinates. +- **Zero lock-in**: pass. Static JSON on GitHub Pages; `DATA_BASE` points it elsewhere. +- **Licences**: pass. Each file is taken only while its dataset carries the licence provenance + names; share-alike sources (ODbL, CC BY-SA) keep their licence and credit. + +## Design + +- `scripts/fetch-data.mjs` (`pnpm data`): reads `catalog.json`, then downloads and validates each + file (licence, shape, record count), keeps the 10,000 best-known landmarks, and writes the + files; a file that fails stays as committed. +- The build scripts for cities, regions, airports, UNESCO sites, Trainline stations, languages + and sovereignty, and their vendored sources, move to offware-apps/data (`src/reference/`), + with the gazetteer checks against the vendored GeoNames day. +- `src/lib/reference/stationSources.ts`: adds `wikidata` (default, recommended); the first + `loadStationSource()` without a saved choice pins Trainline for a returning user (intro already + seen) and Wikidata for a new one. +- `vite.config.ts`: `railways-wikidata.json` out of the precache, into a runtime cache. +- Workflows: `pages.yml` and `android-apk.yml` run `pnpm places` before building; `pages.yml` also + runs weekly after the dataset's Monday build. +- `railways-wikidata.json` is git-ignored: it is produced at build time. diff --git a/specs/022-shared-places/spec.md b/specs/022-shared-places/spec.md new file mode 100644 index 0000000..76f9656 --- /dev/null +++ b/specs/022-shared-places/spec.md @@ -0,0 +1,74 @@ +# Feature Specification: Reference data from Offware's shared data + +**Feature Branch**: `022-shared-places` + +**Created**: 2026-10-12 + +**Status**: Implemented + +**Input**: Maintainer: "update Postcards to use that centralized dataset. It uses only European +trains; make sure it uses worldwide train stations." The dataset is +[offware-apps/data](https://github.com/offware-apps/data), shared by the Offware apps: towns, +sights and railway stations from Wikidata (CC0), every record keeping its Wikidata id. + +## Overview + +Postcards' railway stations come from Trainline's open database, which covers Europe only, and its +landmarks are a hand-picked seed of ~70. Offware's shared data publishes, every week, +open railway stations worldwide and the best-known sights of every town, both from Wikidata. This +feature makes Postcards read both from it, keeping Constitution I: the app still authors nothing, +and every record's provenance (its Wikidata item) travels with it. + +## User Scenarios & Testing + +### User Story 1 - Mark a station anywhere in the world (Priority: P1) + +A traveller who took the train in Japan searches "Tokyo Station", finds it, and marks it visited, +offline once the stations have loaded. + +**Acceptance Scenarios**: + +1. **Given** a new install, **When** the user searches a station outside Europe, **Then** it is + found, from the worldwide set, which is the recommended default. +2. **Given** the user marked stations from the Europe (Trainline) set before this release and never + chose a source, **When** they update, **Then** they stay on Trainline, so their marked stations + keep resolving; Settings offers the worldwide set. +3. **Given** any user, **When** they pick a source in Settings, **Then** that choice is kept. + +### User Story 2 - See well-known sights beyond UNESCO's list (Priority: P2) + +Museums, monuments, castles, palaces and parks show among sites & landmarks: the 10,000 best known +worldwide (by how many Wikipedia editions cover them), replacing the seed. UNESCO sites keep coming +from UNESCO's own list, so none is listed twice. + +### User Story 3 - All reference data in one shared place (Priority: P2) + +Maintainer: "every data we use on every application on a central place, so it can be managed by +a central application." Cities, regions, airports, UNESCO sites, European stations, countries and +moments are built in the shared data too, not in Postcards. Postcards fetches them at build time, +and keeps its committed copies so it always builds and works offline. Nothing a user sees changes: +the shared files are the same as the ones Postcards built. + +## Requirements + +- **FR-001**: The build fetches every reference file from the shared data + (`scripts/fetch-data.mjs`): worldwide stations and landmarks (Wikidata), and cities, regions, + airports, UNESCO sites, European stations, countries and moments. +- **FR-002**: A file is written only when it validates: the dataset still carries the licence the + app's provenance names (`catalog.json`), the file has the expected shape and enough valid + records. Otherwise the committed file stays. The build fails only when no worldwide stations + file exists at all, since the default source would then be empty. +- **FR-003**: A "World — Wikidata" station source is the recommended default; Trainline and None + stay selectable. +- **FR-004**: A person who used the app before and never chose keeps Trainline (FR scenario 1.2). +- **FR-005**: The worldwide station file is runtime-cached (stale-while-revalidate), as it can + outgrow the precache size limit. +- **FR-006**: Provenance (Settings and `public/reference/PROVENANCE.md`) names each original + source and licence, and that it comes by way of the shared data. +- **FR-008**: The scripts that built these files, and their vendored sources, live in the shared + data, not in Postcards. +- **FR-007**: Strings in en, fr and ko. + +## Out of scope + +Converting Trainline-id visits to Wikidata ids; per-category landmark glyphs. diff --git a/specs/022-shared-places/tasks.md b/specs/022-shared-places/tasks.md new file mode 100644 index 0000000..914e103 --- /dev/null +++ b/specs/022-shared-places/tasks.md @@ -0,0 +1,14 @@ +# Tasks: Worldwide stations and landmarks from Offware's shared data + +All paths under `apps/postcards/` unless noted. + +- [x] T001 `scripts/fetch-data.mjs` + `pnpm data`, with validation and a never-failing fallback +- [x] T002 `wikidata` station source as the recommended default; returning users pinned to Trainline +- [x] T003 Unit tests: default source, new vs returning user, saved choice kept (`tests/unit/stations.spec.ts`) +- [x] T004 Runtime cache for `railways-wikidata.json` (`vite.config.ts`) +- [x] T005 en/fr/ko strings for the new source and the updated Trainline description +- [x] T006 Provenance: `src/lib/reference/data/provenance.json`, `public/reference/PROVENANCE.md` +- [x] T007 Workflows: fetch before building (`.github/workflows/pages.yml`, `android-apk.yml`); weekly Pages build +- [x] T008 Fetch every other reference file from the shared data, licence checked against `catalog.json` +- [x] T009 Remove the build scripts and vendored sources that moved to offware-apps/data; drop their dev dependencies +- [x] T010 Docs: wiki pages and provenance name the shared data diff --git a/wiki/Architecture.md b/wiki/Architecture.md index 4b39ae7..cc420c9 100644 --- a/wiki/Architecture.md +++ b/wiki/Architecture.md @@ -63,7 +63,7 @@ apps/postcards/ │ │ ├── experiences.json │ │ └── PROVENANCE.md │ └── icons/ # PWA icons -├── scripts/build-reference.mjs # regenerates public/reference/* from npm datasets +├── scripts/fetch-data.mjs # fetches public/reference/* from Offware's shared data ├── tests/ # unit (Vitest) + e2e (Playwright) ├── vite.config.ts capacitor.config.ts package.json └── android/ ios/ # Capacitor native projects (both committed) @@ -206,13 +206,16 @@ from named datasets and shipped as static assets, never invented in app code. ### The build pipeline -`scripts/build-reference.mjs` (`pnpm reference`) reshapes openly-licensed npm -datasets into `public/reference/*.json`: +The reference files are built in [offware-apps/data](https://github.com/offware-apps/data) +(`src/reference/`), the data every Offware app shares, and fetched at build time by +`scripts/fetch-data.mjs` (`pnpm data`), which keeps the committed copy of a file when the +shared data can't be reached or a dataset's licence no longer matches provenance. The +shared data reshapes openly-licensed datasets: - **`all-the-cities`** (GeoNames, CC BY 4.0) → cities, plus per-region centroid accumulation. -- **GeoNames `cities500` and `admin1CodesASCII`**, vendored under - `scripts/data/` from one retrieval day → each city's current region code and +- **GeoNames `cities500` and `admin1CodesASCII`**, vendored in the shared data + from one retrieval day → each city's current region code and each code's name; a city GeoNames no longer lists takes its nearest listed city's region, and France's regions keep their French names. - **`country-state-city`** (dr5hn, ODbL) → a name, by nearest centroid, only for @@ -221,6 +224,8 @@ datasets into `public/reference/*.json`: whose country can't be resolved are **dropped, never guessed**. - `i18n-iso-countries` + `world-countries` supply ISO codes, continents, and UN-member vs. territory status. +- The UNESCO World Heritage List, Trainline's European stations, Wikidata's worldwide + stations and landmarks, and the moments list. Provenance is recorded in machine-readable `src/lib/reference/data/provenance.json` (shown in Settings) and mirrored in `public/reference/PROVENANCE.md`. See diff --git a/wiki/Data-and-Provenance.md b/wiki/Data-and-Provenance.md index 81f9d1f..d511797 100644 --- a/wiki/Data-and-Provenance.md +++ b/wiki/Data-and-Provenance.md @@ -8,6 +8,12 @@ and version are recorded and shown to you. The app invents none of it. This page lists every reference dataset the app ships or fetches, its license, and how that provenance is recorded in the repo and surfaced in the app. +Every reference file is built in Offware's shared data, +[offware-apps/data](https://github.com/offware-apps/data), where each dataset keeps its +source's licence (`v1/catalog.json`). The app's build fetches them with +`scripts/fetch-data.mjs` and takes a file only while its dataset still carries the licence +listed here; otherwise the committed copy stays. + Related: [Architecture → Reference data](Architecture.md#reference-data-pipeline-two-stage-gazetteer-spatial-grid-index) (how the data is loaded, indexed, and queried), [Privacy and the Constitution](Privacy-and-Constitution.md) (Principle I, the rule itself), [Backup and restore](Features.md#backup-import--reset), and how diff --git a/wiki/Development.md b/wiki/Development.md index a8d0219..36aaac5 100644 --- a/wiki/Development.md +++ b/wiki/Development.md @@ -84,17 +84,18 @@ All scripts below live in `apps/postcards/package.json`. | `test:e2e` | `playwright test` — end-to-end suite (`tests/e2e/**`). | | `schema` | `UPDATE_SCHEMA=1 vitest run tests/unit/schemaArtifact.spec.ts` — regenerate the JSON Schema artifact from the Zod models. | | `format` | `prettier --write .` — format (config: `.prettierrc.json`, 100 cols, double quotes, trailing commas). | -| `reference` | `node scripts/build-reference.mjs` — rebuild bundled reference data from openly-licensed sources (see [Data and Provenance](Data-and-Provenance.md)). | +| `data` | `node scripts/fetch-data.mjs` — fetch the reference data from [Offware's shared data](https://github.com/offware-apps/data) into `public/reference/` and `src/lib/reference/data/` (see [Data and Provenance](Data-and-Provenance.md)). | | `cap:sync` / `cap:copy` | Capacitor sync / copy web assets into native shells. | | `cap:add:android` / `cap:add:ios` | Add a native platform. | | `cap:open:android` / `cap:open:ios` | Open the native project in Android Studio / Xcode. | | `native:android` | `pnpm build && cap sync android && cap open android`. | | `native:ios` | `pnpm build && cap sync ios && cap open ios`. | -The `reference` script and its siblings under `scripts/` (`build-heritage-*.mjs`, -`build-landmarks-seed.mjs`, `build-languages.mjs`, `build-sovereignty.mjs`) are -**build-time aggregators**: they reshape existing openly-licensed datasets into the -bundled `public/reference/*.json` files. The app itself never invents reference data. +The reference data is built in [offware-apps/data](https://github.com/offware-apps/data), +the data every Offware app shares: it reshapes existing openly-licensed datasets, each +keeping its licence. `pnpm data` fetches them at build time, checking each dataset's +licence, shape and record count, and keeps the committed copy of any file that fails, so +the app always builds. The app itself never invents reference data. --- diff --git a/wiki/Privacy-and-Constitution.md b/wiki/Privacy-and-Constitution.md index ee5fbca..18183b5 100644 --- a/wiki/Privacy-and-Constitution.md +++ b/wiki/Privacy-and-Constitution.md @@ -45,9 +45,10 @@ become the *authoritative source* of place names, boundaries, coordinates, or country lists. - **Reference data is built from named datasets, not invented.** - `public/reference/*.json` is a *built asset* regenerated by - `scripts/build-reference.mjs` from openly-licensed npm packages (GeoNames via - `all-the-cities`, dr5hn regions, OpenFlights airports, UNESCO heritage, …). + `public/reference/*.json` is a *built asset*, built in Offware's shared data + ([offware-apps/data](https://github.com/offware-apps/data)) from openly-licensed + datasets (GeoNames, dr5hn regions, OpenFlights airports, UNESCO heritage, Wikidata, …) + and fetched at build time by `scripts/fetch-data.mjs`. Provenance is recorded machine-readably in `src/lib/reference/data/provenance.json`, mirrored for humans in `public/reference/PROVENANCE.md`, **shown in Settings**, and stamped into every