diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33ae143..7b993af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,9 @@ jobs: node-version: 22 cache: npm - run: npm ci - - name: Fetch three countries from Wikidata - # Great Britain and the United States are the heaviest countries for the time limit. - run: npm run fetch -- MC GB US + - name: Fetch two countries from Wikidata + # Great Britain is the heaviest country read in one query. + run: npm run fetch -- MC GB env: FETCH_TRIES: "1" - run: npm run build @@ -42,3 +42,14 @@ jobs: du -sh public/v1 public/v1/*/*.json jq '.places | length' public/v1/places/index.json jq '.stations | length' public/v1/stations/railways.json + - name: Read Japan and the United States in batches + # Their towns time out in a single query: check the batches find them, and the + # worldwide stations query finds Japan's. + run: | + npm run fetch -- JP US + jq -e '[.cities[] | select(.countryIso2 == "JP")] | length > 500' data/raw/wikidata.json + jq -e '[.cities[] | select(.countryIso2 == "US")] | length > 5000' data/raw/wikidata.json + jq -e '[.stations[] | select(.countryIso2 == "JP")] | length > 1000' data/raw/wikidata.json + env: + FETCH_TRIES: "2" + FETCH_IN_PARTS: "1" diff --git a/scripts/fetch.ts b/scripts/fetch.ts index 3334a70..471cb0c 100644 --- a/scripts/fetch.ts +++ b/scripts/fetch.ts @@ -2,22 +2,35 @@ * Fetch towns, cities, their sights and railway stations from Wikidata to * data/raw/wikidata.json, which is committed: a full fetch takes about two hours, so it runs * monthly in its own workflow and the publish workflow builds from the saved copy. One - * query per country for towns and for stations, then the sights in batches of cities, - * slowly enough to respect the query service's limits. A batch that fails is retried in - * smaller ones; whatever keeps failing is left out and named. + * query per country for towns, one for the stations worldwide and their labels in batches, + * then the sights in batches of cities, slowly enough to respect the query service's limits. + * A country whose town query fails has its town ids read first, then their details in + * batches; a batch of sights that fails is read in smaller batches. A country whose towns + * still can't be read keeps what the last fetch had for it; whatever is left is left out and + * named. * Run: npm run fetch [-- FR DE] (default: every country) */ -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { COUNTRIES_QUERY, + COUNTRY_CODES_QUERY, + STATIONS_QUERY, + citiesByIdQuery, citiesQuery, + keepPrevious, + labelsQuery, + nameStations, parseCities, parseCountries, + parseCountryCodes, + parseLabels, parseSights, parseStations, + parseTownIds, sightsQuery, - stationsQuery, + townIdsQuery, + type City, type Raw, } from "../src/wikidata"; @@ -26,13 +39,21 @@ const ENDPOINT = "https://query.wikidata.org/sparql"; const USER_AGENT = "offware-data/0.1 (https://github.com/offware-apps/data)"; const OUT = resolve("data/raw/wikidata.json"); const BATCH = 20; +/** Towns whose details are read in one query, for a country read in batches. */ +const TOWN_BATCH = 500; +/** Stations labelled per query: 10,000 took up to 44 s of the query service's 60. */ +const LABEL_BATCH = 5_000; /** A failed batch of sights is retried this many cities at a time. */ const SMALL_BATCH = 4; /** FETCH_TRIES=1 makes a test run fail fast. */ const TRIES = Number(process.env.FETCH_TRIES) || 3; +/** Tries of a country's whole town query before its towns are read in batches. */ +const WHOLE_TRIES = Math.min(TRIES, 2); const PAUSE_MS = 1_000; /** Tiny territories time out now and then; more than this many missing countries fails the run. */ const MAX_MISSING_COUNTRIES = 2; +/** More countries than this kept from the last fetch means the query service or a query is broken: fail the run. */ +const MAX_KEPT_COUNTRIES = 10; /** Share of towns whose sights may be missing before the run fails. */ const MAX_MISSING_SIGHTS = 0.01; @@ -75,12 +96,34 @@ const raw: Raw = { fetchedAt: new Date().toISOString(), cities: [], sights: {}, const all = countries.length ? countries : parseCountries(await sparql(COUNTRIES_QUERY)); log(`${all.length} countries`); +/** FETCH_IN_PARTS=1 skips a country's whole town query, to test reading in batches. */ +const whole = (query: string, tries: number): Promise => + process.env.FETCH_IN_PARTS === "1" ? Promise.reject(new Error("read in parts")) : sparql(query, tries); + +/** A country with too many towns for one query: their ids first, then their details in batches. */ +async function townsInBatches(cc: string): Promise { + const ids = parseTownIds(await sparql(townIdsQuery(cc))); + log(`${cc} towns: reading ${ids.length} in batches of ${TOWN_BATCH}`); + const out: City[] = []; + for (let i = 0; i < ids.length; i += TOWN_BATCH) { + await sleep(PAUSE_MS); + out.push(...parseCities(await sparql(citiesByIdQuery(ids.slice(i, i + TOWN_BATCH))), cc)); + } + return out; +} + +/** The last fetch, committed: a country this run can't read keeps what it had there. */ +const previous = existsSync(OUT) ? (JSON.parse(readFileSync(OUT, "utf-8")) as Raw) : null; + const missing: string[] = []; for (const cc of all) { try { - // The quick way first; a country too large for it is read the slow, sure way. - const result = await sparql(citiesQuery(cc), 1).catch(() => sparql(citiesQuery(cc, true))); - const cities = parseCities(result, cc); + // The quick way first; a country too large for it is read the slow, sure way, and in + // batches when even that times out. + const cities = await whole(citiesQuery(cc), 1) + .catch(() => whole(citiesQuery(cc, true), WHOLE_TRIES)) + .then((result) => parseCities(result, cc)) + .catch(() => townsInBatches(cc)); raw.cities.push(...cities); log(`${cc}: ${cities.length} towns and cities`); } catch (err) { @@ -90,21 +133,21 @@ for (const cc of all) { await sleep(PAUSE_MS); } if (raw.cities.length === 0) throw new Error("no town or city could be fetched from Wikidata"); +const noTowns = keepPrevious(missing, raw.cities, previous?.cities ?? [], "towns and cities", log); -const missingStations: string[] = []; -const seen = new Set(); -for (const cc of all) { - try { - const stations = parseStations(await sparql(stationsQuery(cc))).filter((s) => !seen.has(s.wikidata)); - for (const s of stations) seen.add(s.wikidata); - raw.stations.push(...stations); - if (stations.length) log(`${cc}: ${stations.length} railway stations`); - } catch (err) { - missingStations.push(cc); - console.error(`[fetch] ${cc}: stations left out (${String(err)})`); - } +// Every country's stations come from the same few queries, so one that keeps failing stops +// the run here, before the sights, rather than publish a copy without stations. +const codes = parseCountryCodes(await sparql(COUNTRY_CODES_QUERY)); +await sleep(PAUSE_MS); +const stations = parseStations(await sparql(STATIONS_QUERY), codes, all); +const labels = new Map(); +for (let i = 0; i < stations.length; i += LABEL_BATCH) { await sleep(PAUSE_MS); + const batch = await sparql(labelsQuery(stations.slice(i, i + LABEL_BATCH).map((s) => s.wikidata))); + for (const [id, label] of parseLabels(batch)) labels.set(id, label); } +raw.stations = nameStations(stations, labels); +if (raw.stations.length === 0) throw new Error("no railway station could be fetched from Wikidata"); log(`${raw.stations.length} railway stations`); /** Sights of a batch of towns; a batch that fails is tried again a few towns at a time. */ @@ -138,13 +181,12 @@ mkdirSync(dirname(OUT), { recursive: true }); writeFileSync(OUT, JSON.stringify(raw), "utf-8"); const sightCount = Object.values(raw.sights).reduce((n, s) => n + s.length, 0); log(`${raw.cities.length} towns and cities, ${sightCount} sights, ${raw.stations.length} stations → ${OUT}`); -if (missing.length) console.log(`[fetch] countries without towns: ${missing.join(" ")}`); -if (missingStations.length) console.log(`[fetch] countries without stations: ${missingStations.join(" ")}`); +if (noTowns.length) console.log(`[fetch] countries without towns: ${noTowns.join(" ")}`); if (noSights.length) console.log(`[fetch] towns without sights: ${noSights.length}`); // A fetch with real holes would publish them: stop instead, and let the next run retry. if ( - missing.length > MAX_MISSING_COUNTRIES || - missingStations.length > MAX_MISSING_COUNTRIES || + noTowns.length > MAX_MISSING_COUNTRIES || + missing.length > MAX_KEPT_COUNTRIES || noSights.length > ids.length * MAX_MISSING_SIGHTS ) { console.error("[fetch] too much was left out to publish; nothing is published this run."); diff --git a/src/wikidata.ts b/src/wikidata.ts index ea086bd..2fddd88 100644 --- a/src/wikidata.ts +++ b/src/wikidata.ts @@ -106,35 +106,58 @@ export function parseCountries(result: unknown): string[] { } /** - * Towns and cities of one country with at least minPopulation() people. The towns are - * found first, in a named subquery the query service always runs before the rest, then - * their details read once each: a town with several populations, coordinates and labels - * would otherwise multiply into hundreds of rows, which made large countries time out. + * The towns and cities of one country with at least minPopulation() people, as a named + * subquery %towns, after others the query service runs first and in order: the country's + * items with a population and their classes, then which of those few classes are kinds of + * town (checked once per class, not once per item, which made large countries time out). */ -export function citiesQuery(countryIso2: string, byPopulation = false): string { +function towns(countryIso2: string, byPopulation: boolean): string { // byPopulation: for countries with millions of items (Great Britain, the United States), // read every item with a population and keep the country's, in that order, rather than - // read every item of the country. Slower for small countries, but never times out. - const towns = byPopulation + // read every item of the country. Slower for small countries, but lighter for large ones. + const candidates = byPopulation ? `hint:SubQuery hint:optimizer "None" . ${country(countryIso2)} ?city wdt:P1082 ?p0 . FILTER(?p0 >= ${minPopulation(countryIso2)}) - ?city wdt:P17 ?country ; wdt:P31 ?cls . - ${TOWN_CLASSES}` + ?city wdt:P17 ?country ; wdt:P31 ?cls .` : `${country(countryIso2)} ?city wdt:P17 ?country ; wdt:P1082 ?p0 ; wdt:P31 ?cls . - FILTER(?p0 >= ${minPopulation(countryIso2)}) - ${TOWN_CLASSES}`; - return `SELECT ?city ?cityLabel (MAX(?p) AS ?pop) (SAMPLE(?c) AS ?coord) (SAMPLE(?v) AS ?voy) (SAMPLE(?w) AS ?wp) (SAMPLE(?s) AS ?site) - (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names) + FILTER(?p0 >= ${minPopulation(countryIso2)})`; + return `WITH { + SELECT DISTINCT ?city ?cls WHERE { + ${candidates} + } +} AS %candidates +WITH { + SELECT DISTINCT ?cls WHERE { INCLUDE %candidates . } +} AS %classes +WITH { + SELECT ?cls WHERE { + hint:SubQuery hint:optimizer "None" . + INCLUDE %classes . + ${TOWN_CLASSES} + } +} AS %townClasses WITH { SELECT DISTINCT ?city WHERE { - ${towns} + INCLUDE %townClasses . + INCLUDE %candidates . } -} AS %towns +} AS %towns`; +} + +/** + * The columns parseCities reads, for the towns `source` binds to ?city. Each town's details + * are grouped into one row: a town with several populations, coordinates and labels would + * otherwise multiply into hundreds of rows. + */ +function details(source: string, withs = ""): string { + return `SELECT ?city ?cityLabel (MAX(?p) AS ?pop) (SAMPLE(?c) AS ?coord) (SAMPLE(?v) AS ?voy) (SAMPLE(?w) AS ?wp) (SAMPLE(?s) AS ?site) + (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names) +${withs} WHERE { - INCLUDE %towns . + ${source} ?city wdt:P1082 ?p ; wdt:P625 ?c . OPTIONAL { ?city rdfs:label ?alt . FILTER(LANG(?alt) IN (${LANGS.map((l) => `"${l}"`).join(", ")})) } OPTIONAL { ?v schema:about ?city ; schema:isPartOf . } @@ -145,6 +168,26 @@ WHERE { GROUP BY ?city ?cityLabel`; } +/** Towns and cities of one country, with their details, in one query. */ +export function citiesQuery(countryIso2: string, byPopulation = false): string { + return details("INCLUDE %towns .", towns(countryIso2, byPopulation)); +} + +/** + * The Wikidata ids of one country's towns and cities, without their details: for a country + * whose towns are too many to read with their details in one query. + */ +export function townIdsQuery(countryIso2: string): string { + return `SELECT ?city +${towns(countryIso2, true)} +WHERE { INCLUDE %towns . }`; +} + +/** A batch of towns and cities by Wikidata id, with their details. */ +export function citiesByIdQuery(ids: string[]): string { + return details(`VALUES ?city { ${ids.map((id) => `wd:${id}`).join(" ")} }`); +} + /** Kinds of sight, by Wikidata class. UNESCO World Heritage sites are kept whatever their class. */ export const SIGHT_CLASSES: [string, string][] = [ ["Q33506", "museum"], @@ -225,6 +268,23 @@ function url(v: string | undefined, host?: RegExp): string | undefined { const VOY = /^en\.wikivoyage\.org$/; const WP = /^en\.wikipedia\.org$/; +/** Adds what the last fetch had for each missing country; returns the countries it had nothing for. */ +export function keepPrevious(missing: string[], into: T[], old: T[], what: string, log: (msg: string) => void): string[] { + return missing.filter((cc) => { + const kept = old.filter((x) => x.countryIso2 === cc); + into.push(...kept); + if (kept.length) log(`${cc}: ${kept.length} ${what} kept from the last fetch`); + return kept.length === 0; + }); +} + +/** Wikidata ids from townIdsQuery. */ +export function parseTownIds(result: unknown): string[] { + return bindings(result) + .map((b) => qid(b.city?.value)) + .filter((id) => /^Q\d+$/.test(id)); +} + /** Cities from citiesQuery; when an id repeats (two populations or coordinates), the largest wins. */ export function parseCities(result: unknown, countryIso2: string): City[] { const byId = new Map(); @@ -295,47 +355,84 @@ export function parseSights(result: unknown): Record { // ---------------------------------------------------------------- railway stations +/** Every item with an ISO 3166-1 code, dissolved ones too: the countries stations name. */ +export const COUNTRY_CODES_QUERY = `SELECT ?country ?iso2 WHERE { ?country wdt:P297 ?iso2 . }`; + /** - * Mainline railway stations of one country that are still open, with coordinates and - * either a UIC code or a page on a Wikimedia project (which leaves out most tram and metro - * stops). One query per country, run in the written order: every railway station on - * Wikidata, then the country's, then their labels. Reading a large country's items first - * timed out. + * Mainline railway stations worldwide that are still open, with their countries, + * coordinates and either a UIC code or a page on a Wikimedia project (which leaves out most + * tram and metro stops). One query reads every railway station on Wikidata, in the written + * order, in about half a minute. The countries' ISO codes and the labels are read by their + * own queries: reading the labels here, or the codes with the Netherlands' exception, took + * this one past the query service's one-minute limit. */ -export function stationsQuery(countryIso2: string): string { - return `SELECT ?station ?stationLabel ?iso2 ?coord -WITH { - SELECT ?station (SAMPLE(?c) AS ?coord) WHERE { - hint:SubQuery hint:optimizer "None" . - ${country(countryIso2)} - ?station wdt:P31 wd:Q55488 . - ?station wdt:P17 ?country ; wdt:P625 ?c ; wikibase:sitelinks ?links . - FILTER NOT EXISTS { ?station wdt:P576 ?closed . } - FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915 . } - FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883 . } - FILTER(?links > 0 || EXISTS { ?station wdt:P954 ?uic . }) - } - GROUP BY ?station -} AS %stations -WHERE { - INCLUDE %stations . - BIND("${countryIso2}" AS ?iso2) +export const STATIONS_QUERY = `SELECT ?station ?country (SAMPLE(?c) AS ?coord) WHERE { + hint:Query hint:optimizer "None" . + ?station wdt:P31 wd:Q55488 . + ?station wdt:P17 ?country ; wdt:P625 ?c ; wikibase:sitelinks ?links . + FILTER NOT EXISTS { ?station wdt:P576 ?closed . } + FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915 . } + FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883 . } + FILTER(?links > 0 || EXISTS { ?station wdt:P954 ?uic . }) +} +GROUP BY ?station ?country`; + +/** Labels of a batch of items. */ +export function labelsQuery(ids: string[]): string { + return `SELECT ?item ?itemLabel WHERE { + VALUES ?item { ${ids.map((id) => `wd:${id}`).join(" ")} } ${labelService} }`; } -/** Stations from stationsQuery; a station listed twice keeps the first. */ -export function parseStations(result: unknown): Station[] { - const out: Station[] = []; - const seen = new Set(); +/** A station before nameStations gives it its label. */ +type UnnamedStation = Omit; + +/** Item → ISO code, from COUNTRY_CODES_QUERY, with the codes COUNTRY_ITEMS moves to another item. */ +export function parseCountryCodes(result: unknown): Map { + const codes = new Map(); + for (const b of bindings(result)) { + const id = qid(b.country?.value); + const cc = b.iso2?.value ?? ""; + if (/^Q\d+$/.test(id) && /^[A-Z]{2}$/.test(cc) && !(cc in COUNTRY_ITEMS)) codes.set(id, cc); + } + for (const [cc, id] of Object.entries(COUNTRY_ITEMS)) codes.set(id, cc); + return codes; +} + +/** + * Stations from STATIONS_QUERY in these countries, country by country in their order, a + * station in several kept in the first. They are named by nameStations. + */ +export function parseStations(result: unknown, codes: Map, countries: string[]): UnnamedStation[] { + const rank = new Map(countries.map((cc, i) => [cc, i])); + const best = new Map(); for (const b of bindings(result)) { const id = qid(b.station?.value); + const cc = codes.get(qid(b.country?.value)) ?? ""; + const r = rank.get(cc); const at = point(b.coord?.value); - const name = b.stationLabel?.value?.trim() ?? ""; - const cc = b.iso2?.value?.toUpperCase() ?? ""; - if (!/^Q\d+$/.test(id) || !at || !name || name === id || !/^[A-Z]{2}$/.test(cc) || seen.has(id)) continue; - seen.add(id); - out.push({ wikidata: id, name, countryIso2: cc, ...at }); + if (!/^Q\d+$/.test(id) || r === undefined || !at || (best.get(id)?.rank ?? Infinity) <= r) continue; + best.set(id, { rank: r, station: { wikidata: id, countryIso2: cc, ...at } }); } - return out; + return [...best.values()].sort((a, b) => a.rank - b.rank).map((s) => s.station); +} + +/** Item → label, from labelsQuery; an item with no label in the languages above has none. */ +export function parseLabels(result: unknown): Map { + const labels = new Map(); + for (const b of bindings(result)) { + const id = qid(b.item?.value); + const label = b.itemLabel?.value?.trim() ?? ""; + if (label && label !== id) labels.set(id, label); + } + return labels; +} + +/** Stations named by their labels, from parseLabels; a station without one is left out. */ +export function nameStations(stations: UnnamedStation[], labels: Map): Station[] { + return stations.flatMap(({ wikidata, ...rest }) => { + const name = labels.get(wikidata); + return name ? [{ wikidata, name, ...rest }] : []; + }); } diff --git a/tests/fixtures/country-codes.json b/tests/fixtures/country-codes.json new file mode 100644 index 0000000..090f30b --- /dev/null +++ b/tests/fixtures/country-codes.json @@ -0,0 +1,49 @@ +{ + "head": { + "vars": [] + }, + "results": { + "bindings": [ + { + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q142" + }, + "iso2": { + "type": "literal", + "value": "FR" + } + }, + { + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q39" + }, + "iso2": { + "type": "literal", + "value": "CH" + } + }, + { + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q17" + }, + "iso2": { + "type": "literal", + "value": "JP" + } + }, + { + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q29999" + }, + "iso2": { + "type": "literal", + "value": "NL" + } + } + ] + } +} diff --git a/tests/fixtures/station-labels.json b/tests/fixtures/station-labels.json new file mode 100644 index 0000000..7e38fe6 --- /dev/null +++ b/tests/fixtures/station-labels.json @@ -0,0 +1,69 @@ +{ + "head": { + "vars": [] + }, + "results": { + "bindings": [ + { + "item": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q801124" + }, + "itemLabel": { + "type": "literal", + "value": "Tokyo Station" + } + }, + { + "item": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q745942" + }, + "itemLabel": { + "type": "literal", + "value": "Paris Gare de Lyon" + } + }, + { + "item": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q1" + }, + "itemLabel": { + "type": "literal", + "value": "Frontier" + } + }, + { + "item": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q5" + }, + "itemLabel": { + "type": "literal", + "value": "Q5" + } + }, + { + "item": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q1124795" + }, + "itemLabel": { + "type": "literal", + "value": "Amsterdam Centraal" + } + }, + { + "item": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q3" + }, + "itemLabel": { + "type": "literal", + "value": "Border" + } + } + ] + } +} diff --git a/tests/fixtures/stations.json b/tests/fixtures/stations.json index 98226f7..6c2b7e3 100644 --- a/tests/fixtures/stations.json +++ b/tests/fixtures/stations.json @@ -6,16 +6,26 @@ "bindings": [ { "station": { - "type": "literal", - "value": "http://www.wikidata.org/entity/Q745942" + "type": "uri", + "value": "http://www.wikidata.org/entity/Q801124" }, - "stationLabel": { - "type": "literal", - "value": "Paris Gare de Lyon" + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q17" }, - "iso2": { + "coord": { "type": "literal", - "value": "FR" + "value": "Point(139.7671 35.6812)" + } + }, + { + "station": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q745942" + }, + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q142" }, "coord": { "type": "literal", @@ -24,58 +34,102 @@ }, { "station": { - "type": "literal", - "value": "http://www.wikidata.org/entity/Q745942" + "type": "uri", + "value": "http://www.wikidata.org/entity/Q1" }, - "stationLabel": { - "type": "literal", - "value": "Paris Gare de Lyon" + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q55" }, - "iso2": { + "coord": { "type": "literal", - "value": "CH" + "value": "Point(1 1)" + } + }, + { + "station": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q1" + }, + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q39" }, "coord": { "type": "literal", - "value": "Point(2.3733 48.8443)" + "value": "Point(1 1)" } }, { "station": { - "type": "literal", - "value": "http://www.wikidata.org/entity/Q801124" + "type": "uri", + "value": "http://www.wikidata.org/entity/Q5" }, - "stationLabel": { - "type": "literal", - "value": "Tokyo Station" + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q142" }, - "iso2": { + "coord": { "type": "literal", - "value": "jp" + "value": "Point(1 1)" + } + }, + { + "station": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q2" + }, + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q29999" }, "coord": { "type": "literal", - "value": "Point(139.7671 35.6812)" + "value": "Point(4.9 52.37)" } }, { "station": { - "type": "literal", - "value": "http://www.wikidata.org/entity/Q5" + "type": "uri", + "value": "http://www.wikidata.org/entity/Q1124795" + }, + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q55" }, - "stationLabel": { + "coord": { "type": "literal", - "value": "Q5" + "value": "Point(4.9003 52.3789)" + } + }, + { + "station": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q3" + }, + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q39" }, - "iso2": { + "coord": { "type": "literal", - "value": "FR" + "value": "Point(6.14 46.21)" + } + }, + { + "station": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q3" + }, + "country": { + "type": "uri", + "value": "http://www.wikidata.org/entity/Q55" }, "coord": { "type": "literal", - "value": "Point(1 1)" + "value": "Point(6.14 46.21)" } } ] } -} \ No newline at end of file +} diff --git a/tests/places.test.ts b/tests/places.test.ts index 8023b31..0528694 100644 --- a/tests/places.test.ts +++ b/tests/places.test.ts @@ -2,13 +2,17 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { PACK_MAX, buildFiles, placeIds, slug, type Place, type PlacesIndex } from "../src/places"; import type { Catalog } from "../src/catalog"; -import { citiesQuery, minPopulation, parseCities, parseCountries, parseSights, parseStations, point, sightsQuery, stationsQuery, type Raw } from "../src/wikidata"; +import { citiesByIdQuery, citiesQuery, keepPrevious, labelsQuery, minPopulation, nameStations, parseCities, parseCountries, parseCountryCodes, parseLabels, parseSights, parseStations, parseTownIds, point, sightsQuery, townIdsQuery, type Raw } from "../src/wikidata"; const fixture = (name: string): unknown => JSON.parse(readFileSync(new URL(`./fixtures/${name}`, import.meta.url), "utf-8")); +const codes = parseCountryCodes(fixture("country-codes.json")); +const stations = (countries: string[]) => + nameStations(parseStations(fixture("stations.json"), codes, countries), parseLabels(fixture("station-labels.json"))); + const raw = (): Raw => { const cities = [...parseCities(fixture("cities-fr.json"), "FR"), ...parseCities(fixture("cities-es.json"), "ES")]; - return { fetchedAt: "2026-10-12T00:00:00.000Z", cities, sights: parseSights(fixture("sights.json")), stations: parseStations(fixture("stations.json")) }; + return { fetchedAt: "2026-10-12T00:00:00.000Z", cities, sights: parseSights(fixture("sights.json")), stations: stations(["FR", "JP"]) }; }; describe("Wikidata results", () => { @@ -41,12 +45,23 @@ describe("Wikidata results", () => { it("reads countries and stations worldwide", () => { expect(parseCountries(fixture("countries.json"))).toEqual(["FR", "JP"]); - expect(parseStations(fixture("stations.json"))).toEqual([ + expect(stations(["FR", "JP"])).toEqual([ { wikidata: "Q745942", name: "Paris Gare de Lyon", countryIso2: "FR", lat: 48.8443, lon: 2.3733 }, { wikidata: "Q801124", name: "Tokyo Station", countryIso2: "JP", lat: 35.6812, lon: 139.7671 }, ]); }); + it("keeps a station in several countries in the first, and drops unlabelled ones", () => { + expect(stations(["CH", "FR", "JP", "NL"]).map((s) => `${s.countryIso2} ${s.name}`)).toEqual([ + "CH Frontier", + "CH Border", + "FR Paris Gare de Lyon", + "JP Tokyo Station", + "NL Amsterdam Centraal", + ]); + expect(labelsQuery(["Q1", "Q2"])).toContain("VALUES ?item { wd:Q1 wd:Q2 }"); + }); + it("keeps smaller towns in Europe than elsewhere", () => { expect(minPopulation("FR")).toBe(5000); expect(minPopulation("JP")).toBe(15000); @@ -63,8 +78,28 @@ describe("Wikidata results", () => { it("finds Dutch towns and stations by the country they name", () => { expect(citiesQuery("NL")).toContain("BIND(wd:Q55 AS ?country)"); expect(citiesQuery("NL", true)).toContain("BIND(wd:Q55 AS ?country)"); - expect(stationsQuery("NL")).toContain("BIND(wd:Q55 AS ?country)"); - expect(stationsQuery("NL")).not.toContain("P297"); + expect(codes.get("Q55")).toBe("NL"); + expect(codes.has("Q29999")).toBe(false); + }); + + it("reads a large country's town ids alone, then their details by id", () => { + const ids = townIdsQuery("US"); + expect(ids).toContain('wdt:P297 "US"'); + expect(ids).toContain("?p0 >= 15000"); + expect(ids).toContain("wd:Q15284"); + // Every item with a population first, then the country's: the order light enough for the US. + expect(ids).toMatch(/wdt:P1082 \?p0 \.[\s\S]*wdt:P17 \?country/); + expect(ids).not.toContain("rdfs:label"); + const result = { results: { bindings: [{ city: { value: "http://www.wikidata.org/entity/Q60" } }, { city: { value: "x" } }] } }; + expect(parseTownIds(result)).toEqual(["Q60"]); + expect(citiesByIdQuery(["Q60", "Q65"])).toContain("VALUES ?city { wd:Q60 wd:Q65 }"); + }); + + it("keeps the last fetch's data for a country this run missed, and names the ones it had none for", () => { + const old = [{ countryIso2: "IT", id: 1 }, { countryIso2: "US", id: 2 }]; + const into: { countryIso2: string; id: number }[] = []; + expect(keepPrevious(["IT", "JP"], into, old, "towns", () => {})).toEqual(["JP"]); + expect(into.map((x) => x.id)).toEqual([1]); }); });