diff --git a/.github/workflows/probe-europe.yml b/.github/workflows/probe-europe.yml new file mode 100644 index 0000000..dd9ee9a --- /dev/null +++ b/.github/workflows/probe-europe.yml @@ -0,0 +1,31 @@ +name: Europe probe + +# Measures every rail feed of the 33 Interrail countries (via Transitous's mirror): +# trips, stations, bundle sizes, and the search engine's memory and speed over all of them. +# The numbers size the search server. Results are in the run's summary. +on: + workflow_dispatch: + pull_request: + paths: + - scripts/probe-europe.ts + - .github/workflows/probe-europe.yml + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - name: Read every country's rail feeds and time the search + run: /usr/bin/time -f "%e s, %M KB peak memory" node --expose-gc --max-old-space-size=13312 --import tsx scripts/probe-europe.ts + - uses: actions/upload-artifact@v4 + if: always() + with: + name: europe-probe + path: data/probe/summary.md + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index e3afb72..b70613f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules/ data/raw/ public/ +data/probe/ +data/probe-raw/ diff --git a/package.json b/package.json index 55b5323..996be70 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "build:api": "tsx scripts/build-api.ts", "search": "tsx scripts/search.ts", "start": "tsx src/server.ts", + "probe:europe": "tsx scripts/probe-europe.ts", "check": "npm run typecheck && npm test" }, "devDependencies": { diff --git a/scripts/probe-europe.ts b/scripts/probe-europe.ts new file mode 100644 index 0000000..679dcee --- /dev/null +++ b/scripts/probe-europe.ts @@ -0,0 +1,297 @@ +/** + * How big is every train in Europe? Reads the European feeds Transitous republishes + * (https://api.transitous.org/gtfs/, which asks reusers to download from there), keeps the + * rail routes, and prints per country the trips, stations and bundle size, then how long + * the search engine takes to load them all and answer. It sizes the search server. + * + * npm run probe:europe download the feeds one at a time + * npm run probe:europe -- read zips already downloaded there + * + * PROBE_FROM (default today) and PROBE_DAYS (default 31) set the window; PROBE_ONLY=fr,es limits the countries. + * PROBE_PAIRS="Madrid,Valencia;Lyon,Annecy" sets the searches timed. + * Writes data/probe/.json and a Markdown summary (also to $GITHUB_STEP_SUMMARY). + */ +import { execFileSync } from "node:child_process"; +import { appendFileSync, createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream as WebReadableStream } from "node:stream/web"; +import { gzipSync } from "node:zlib"; +import { encodeBundle, type Bundle } from "../src/bundle"; +import { addDays, readGtfsTimetable, type GtfsOperator } from "../src/gtfs"; +import type { Timetable } from "../src/format"; +import { mergeTimetables } from "../src/merge"; +import { Network } from "../src/search"; + +const MIRROR = "https://api.transitous.org/gtfs/"; +const HEADERS = { "User-Agent": "train-api probe (https://github.com/offware-apps/train-api)" }; +const OUT = resolve("data/probe"); +const WORK = resolve("data/probe-raw"); +const DAYS = Number(process.env.PROBE_DAYS ?? 31); +const ONLY = new Set((process.env.PROBE_ONLY ?? "").split(",").map((s) => s.trim()).filter(Boolean)); + +/** The 33 Interrail countries, by the region code Transitous files start with; "eu" holds cross-border feeds. */ +const COUNTRIES: Record = { + at: "Austria", be: "Belgium", ba: "Bosnia and Herzegovina", bg: "Bulgaria", hr: "Croatia", cz: "Czechia", + dk: "Denmark", ee: "Estonia", fi: "Finland", fr: "France", de: "Germany", gb: "Great Britain", gr: "Greece", + hu: "Hungary", ie: "Ireland", it: "Italy", lv: "Latvia", lt: "Lithuania", lu: "Luxembourg", me: "Montenegro", + nl: "Netherlands", mk: "North Macedonia", no: "Norway", pl: "Poland", pt: "Portugal", ro: "Romania", + rs: "Serbia", sk: "Slovakia", si: "Slovenia", es: "Spain", se: "Sweden", ch: "Switzerland", tr: "Turkey", + eu: "Cross-border", +}; + +/** Files no reader uses; shapes.txt alone runs to gigabytes in some national feeds. */ +const UNUSED = ["shapes.txt", "transfers.txt", "frequencies.txt", "pathways.txt", "levels.txt", "translations.txt", "fare_*.txt"]; + +/** GTFS rail: 2, and the extended railway types 100–117. Trams, metros and buses are left out. */ +function isRail(routeType: string): boolean { + const t = Number(routeType); + return t === 2 || (t >= 100 && t <= 117); +} + +const rail = (id: string): GtfsOperator => ({ + id, + source: "Transitous", + route: (r) => (isRail(r.route_type ?? "") ? ((r.route_short_name ?? "").trim().split(/\s+/)[0] ?? "") || "train" : null), + passes: () => ({ interrail: { bookable: true, seat: "unknown" } }), +}); + +/** Every zip the mirror lists, following its folders one level down. */ +async function listMirror(): Promise { + const page = async (url: string): Promise => { + const res = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(60_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); + const html = await res.text(); + const links = [...html.matchAll(/href="([^"?#]+)"/g)].map((m) => new URL(m[1] as string, url).href).filter((u) => u.startsWith(url) && u !== url); + if (links.length === 0) console.log(`[probe] ${url} lists no links; it starts:\n${html.slice(0, 2000)}`); + return links; + }; + const top = await page(MIRROR); + const zips = top.filter((u) => u.endsWith(".zip")); + for (const dir of top.filter((u) => u.endsWith("/"))) zips.push(...(await page(dir)).filter((u) => u.endsWith(".zip"))); + return [...new Set(zips)]; +} + +/** "de_DELFI.gtfs.zip" → "de"; null for a region outside the 33 countries. */ +function countryOf(file: string): string | null { + const code = /^([a-z]{2})[_-]/i.exec(file)?.[1]?.toLowerCase(); + const c = code === "uk" ? "gb" : code; + return c && c in COUNTRIES ? c : null; +} + +const mb = (n: number): string => (n / 1e6).toFixed(1); +const secs = (ms: number): string => (ms / 1000).toFixed(1); + +async function download(url: string, out: string): Promise { + for (let i = 1; ; i++) { + try { + const res = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(1_800_000) }); + if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`); + await pipeline(Readable.fromWeb(res.body as WebReadableStream), createWriteStream(out)); + return statSync(out).size; + } catch (err) { + if (i >= 3) throw err; + console.error(`[probe] ${url}: attempt ${i} failed (${String(err)})`); + await new Promise((r) => setTimeout(r, 5_000 * i)); + } + } +} + +interface FeedRow { + file: string; + country: string; + zipBytes: number; + trips: number; + readMs: number; + error?: string; +} + +const from = process.env.PROBE_FROM ?? new Date().toISOString().slice(0, 10); +const local = process.argv[2] ? resolve(process.argv[2]) : null; +mkdirSync(OUT, { recursive: true }); +mkdirSync(WORK, { recursive: true }); + +const sources = local + ? readdirSync(local).filter((f) => f.endsWith(".zip")).map((f) => join(local, f)) + : await listMirror(); +const byCountry = new Map(); +const skipped: string[] = []; +for (const s of sources) { + const file = s.split("/").pop() as string; + const c = countryOf(file); + if (!c || (ONLY.size > 0 && !ONLY.has(c))) { + skipped.push(file); + continue; + } + byCountry.set(c, [...(byCountry.get(c) ?? []), s]); +} +console.log(`[probe] ${sources.length} feeds listed, ${sources.length - skipped.length} in ${byCountry.size} countries; window ${from} + ${DAYS} days`); +if (skipped.length) console.log(`[probe] other regions: ${skipped.join(", ")}`); + +const feeds: FeedRow[] = []; +const summary: { country: string; shared: number; sharedExamples: string; feeds: number; trips: number; variants: number; stations: number; located: number; bytes: number; gz: number }[] = []; +const t0 = Date.now(); +for (const [country, list] of [...byCountry].sort()) { + const read: Timetable[] = []; + for (const src of list) { + const file = src.split("/").pop() as string; + const row: FeedRow = { file, country, zipBytes: 0, trips: 0, readMs: 0 }; + feeds.push(row); + const zip = local ? src : join(WORK, file); + const dir = join(WORK, file.replace(/\.zip$/, "")); + try { + row.zipBytes = local ? statSync(src).size : await download(src, zip); + rmSync(dir, { recursive: true, force: true }); + // unzip exits 11 when an excluded name matches nothing; the files are out all the same. + try { + execFileSync("unzip", ["-q", "-o", "-j", zip, "-d", dir, "-x", ...UNUSED.map((u) => `*${u}`)], { stdio: "ignore" }); + } catch (err) { + if (!existsSync(join(dir, "stop_times.txt"))) throw err; + } + if (!local) rmSync(zip, { force: true }); + const t = Date.now(); + const tt = await readGtfsTimetable(dir, rail(country), from, DAYS, { v1: false }); + row.readMs = Date.now() - t; + row.trips = tt.trips.length; + read.push(tt); + } catch (err) { + row.error = String(err).slice(0, 200); + } finally { + rmSync(dir, { recursive: true, force: true }); + if (!local) rmSync(zip, { force: true }); + } + console.log( + `[probe] ${country} ${file}: ${mb(row.zipBytes)} MB zip, ${row.trips} rail trips, read in ${secs(row.readMs)} s${row.error ? `, FAILED ${row.error}` : ""}`, + ); + } + const { trips, coords, sameName } = mergeTimetables(read); + read.length = 0; + if (trips.length === 0) continue; + const bundle = encodeBundle(country, "interrail", trips, coords); + const json = JSON.stringify(bundle); + writeFileSync(join(OUT, `${country}.json`), json); + // Names that stand for stations in different towns, now told apart. + const shared = [...(sameName ?? [])].sort((a, b) => b[1].length - a[1].length); + const row = { + country, + shared: shared.length, + sharedExamples: shared.slice(0, 6).map(([n, l]) => `${n} (${l.length})`).join(", "), + feeds: list.length, + trips: trips.length, + variants: bundle.trips.length, + stations: bundle.stations.length, + located: bundle.stations.filter((s) => s[1] !== null).length, + bytes: json.length, + gz: gzipSync(json).length, + }; + summary.push(row); + console.log(`[probe] ${country}: ${row.variants} trips in the bundle, ${row.stations} stations (${row.located} located), ${mb(row.bytes)} MB, ${mb(row.gz)} MB gzipped`); +} +const readAll = Date.now() - t0; + +// The engine over every country, as a server would hold it. +const memory = (): string => { + global.gc?.(); + const m = process.memoryUsage(); + return `heap ${mb(m.heapUsed)} MB, rss ${mb(m.rss)} MB`; +}; +const before = memory(); +const t1 = Date.now(); +const bundles = summary.map((s) => JSON.parse(readFileSync(join(OUT, `${s.country}.json`), "utf-8")) as Bundle); +const parseMs = Date.now() - t1; +const t2 = Date.now(); +const net = new Network(bundles, { utcOffsets: { tr: 120 } }); +const buildMs = Date.now() - t2; +const loaded = memory(); + +const day = addDays(from, 1); +const PAIRS: [string, string][] = process.env.PROBE_PAIRS + ? process.env.PROBE_PAIRS.split(";").map((p) => p.split(",") as [string, string]) + : [ + ["Paris", "Berlin"], + ["Madrid", "Warszawa"], + ["Lisboa", "Helsinki"], + ["Amsterdam", "Roma"], + ["Zürich", "Budapest"], + ["Bruxelles", "København"], + ["Lyon", "Annecy"], + ["München", "Garmisch-Partenkirchen"], +]; +const searches: string[] = []; +// The place a name resolved to, with a few of its stations, to check the lookup. +const seenAs = (m: NonNullable>): string => + `${m.name} [${m.places.flatMap((i) => net.places[i]?.stations.map((s) => `${s.operator}:${s.name}`) ?? []).slice(0, 4).join("; ")}]`; +for (const [a, b] of PAIRS) { + const from = net.resolve(a); + const to = net.resolve(b); + if (!from || !to) { + searches.push(`| ${a} → ${b} | ${from ? "" : `no "${a}"`} ${to ? "" : `no "${b}"`} | | | |`); + continue; + } + const times: number[] = []; + let found: ReturnType = []; + for (let i = 0; i < 2; i++) { + const t = Date.now(); + found = net.journeys(from, to, day, { maxTrains: 6 }); + times.push(Date.now() - t); + } + const best = found.reduce<(typeof found)[number] | undefined>((x, j) => (!x || j.duration < x.duration ? j : x), undefined); + const route = best ? `${Math.floor(best.duration / 60)}h${String(best.duration % 60).padStart(2, "0")}, ${best.legs.length} trains` : "none"; + searches.push(`| ${a} → ${b}: ${seenAs(from)} → ${seenAs(to)} | ${found.length} | ${route} | ${times[0]} ms | ${times[1]} ms |`); +} +const firstDay = (() => { + const t = Date.now(); + const p = net.resolve("Paris"); + const b = net.resolve("Berlin"); + if (p && b) net.journeys(p, b, addDays(from, 3), { maxTrains: 6 }); + return Date.now() - t; +})(); +const paris = net.resolve("Paris"); +const t4 = Date.now(); +const reach = paris ? net.reachable(paris, day, { maxTrains: 4, step: 120 }) : new Map(); +const reachMs = Date.now() - t4; +const searched = memory(); + +const total = summary.reduce((n, s) => ({ trips: n.trips + s.variants, stations: n.stations + s.stations, bytes: n.bytes + s.bytes, gz: n.gz + s.gz }), { + trips: 0, + stations: 0, + bytes: 0, + gz: 0, +}); +const lines = [ + `## Every train in Europe: probe of ${from} + ${DAYS} days`, + "", + `${feeds.length} feeds read in ${secs(readAll)} s; ${feeds.filter((f) => f.error).length} failed.`, + "", + "| Country | Feeds | Trips | Stations (located) | Bundle MB | Gzipped MB | Names told apart (towns) |", + "|---|---|---|---|---|---|---|", + ...summary.map( + (s) => `| ${COUNTRIES[s.country]} | ${s.feeds} | ${s.variants} | ${s.stations} (${s.located}) | ${mb(s.bytes)} | ${mb(s.gz)} | ${s.shared}${s.shared ? `: ${s.sharedExamples}` : ""} |`, + ), + `| **All** | ${feeds.length} | ${total.trips} | ${total.stations} | ${mb(total.bytes)} | ${mb(total.gz)} | ${summary.reduce((n, s) => n + s.shared, 0)} |`, + "", + `Missing countries: ${Object.keys(COUNTRIES).filter((c) => !summary.some((s) => s.country === c)).map((c) => COUNTRIES[c]).join(", ") || "none"}.`, + "", + "### Engine over every country", + "", + `- Memory before loading: ${before}.`, + `- Parse every bundle: ${secs(parseMs)} s. Build the network: ${secs(buildMs)} s, ${net.places.length} places. Then ${loaded}.`, + `- The first search below also builds that day's timetable. Paris → Berlin on another day, built afresh: ${firstDay} ms.`, + `- Everything reachable from Paris in a day, up to 4 trains: ${reach.size} places in ${reachMs} ms. Then ${searched}.`, + "", + "| Search, tomorrow, up to 6 trains | Journeys | Fastest | First run | Again |", + "|---|---|---|---|---|", + ...searches, + "", + "### Feeds", + "", + "| Feed | Zip MB | Rail trips | Read s | Error |", + "|---|---|---|---|---|", + ...feeds.map((f) => `| ${f.file} | ${mb(f.zipBytes)} | ${f.trips} | ${secs(f.readMs)} | ${f.error ?? ""} |`), +]; +const md = lines.join("\n"); +console.log(`\n${md}`); +writeFileSync(join(OUT, "summary.md"), md); +if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${md}\n`); +rmSync(WORK, { recursive: true, force: true }); diff --git a/src/format.ts b/src/format.ts index 10d9426..7edaf2c 100644 --- a/src/format.ts +++ b/src/format.ts @@ -93,4 +93,6 @@ export interface Timetable { trips: Trip[]; /** Station name → [latitude, longitude], when the source publishes it. */ coords: Map; + /** Names whose stations lie in different places (a GTFS source): every position, for each. */ + sameName?: Map; } diff --git a/src/gtfs.ts b/src/gtfs.ts index 54b61f0..a029a4b 100644 --- a/src/gtfs.ts +++ b/src/gtfs.ts @@ -100,6 +100,17 @@ export interface GtfsOperator { passes(family: string): Partial>; } +/** Stations of one name further apart than this are different places sharing the name. */ +export const SAME_NAME_APART_KM = 3; + +/** Kilometres between two [lat, lon], close enough at station scale. */ +export function km(a: [number, number], b: [number, number]): number { + const rad = Math.PI / 180; + const x = (b[1] - a[1]) * rad * Math.cos(((a[0] + b[0]) / 2) * rad); + const y = (b[0] - a[0]) * rad; + return 6371 * Math.hypot(x, y); +} + /** "YYYYMMDD" → "YYYY-MM-DD". */ function isoDate(d: string): string { return `${d.slice(0, 4)}-${d.slice(4, 6)}-${d.slice(6, 8)}`; @@ -239,8 +250,15 @@ export async function readGtfs(dir: string, op: GtfsOperator, from: string, days * or the route's when the feed gives none (some feeds only name the line, others put the * train number there). v1 cuts the `narrow` families to the long-distance stations to * keep its stop pairs few; v2 keeps every stop, since a trip costs one entry per stop. + * `opts.v1: false` skips the rows. */ -export async function readGtfsTimetable(dir: string, op: GtfsOperator, from: string, days = 31): Promise { +export async function readGtfsTimetable( + dir: string, + op: GtfsOperator, + from: string, + days = 31, + opts: { v1?: boolean } = {}, +): Promise { const agencies = new Map(); await readTable(dir, "agency", (r) => agencies.set(r.agency_id ?? "", r.agency_name ?? "")); const onlyAgency = agencies.size === 1 ? [...agencies.values()][0] ?? "" : ""; @@ -274,10 +292,12 @@ export async function readGtfsTimetable(dir: string, op: GtfsOperator, from: str const lon = Number(r.stop_lon); if (r.stop_lat && r.stop_lon && Number.isFinite(lat) && Number.isFinite(lon) && (lat || lon)) at.set(id, [lat, lon]); }); - const stationOf = (stop: string): string => names.get(parents.get(stop) ?? stop) ?? names.get(stop) ?? ""; const coords = new Map(); + // Calls hold the station's id first; it becomes the station's name below. const calls = new Map(); + // Each station used, with where it is: its own position, else its first platform's. + const used = new Map(); await readTable( dir, "stop_times", @@ -287,19 +307,43 @@ export async function readGtfsTimetable(dir: string, op: GtfsOperator, from: str const dep = gtfsMinutes(r.departure_time || r.arrival_time || ""); if (Number.isNaN(arr) || Number.isNaN(dep)) return; const stop = r.stop_id ?? ""; - const station = stationOf(stop); - // A station's place is its own, else its first platform's. - if (station && !coords.has(station)) { - const p = at.get(parents.get(stop) ?? stop) ?? at.get(stop); - if (p) coords.set(station, p); - } + const id = parents.get(stop) ?? stop; + if (!used.get(id)) used.set(id, at.get(id) ?? at.get(stop)); const list = calls.get(trip) ?? []; - list.push({ seq: Number(r.stop_sequence), station, arr, dep }); + list.push({ seq: Number(r.stop_sequence), station: id, arr, dep }); calls.set(trip, list); }, { column: "trip_id", keep: (id) => trips.has(id) }, ); + // One name can stand for stations in different towns ("Gare SNCF"): each place after + // the first takes a number, "Gare SNCF (2)", so trips don't jump between towns. + const byName = new Map(); + const label = new Map(); + for (const [id, p] of used) { + const name = names.get(id) ?? ""; + if (!name) { + label.set(id, ""); + continue; + } + const places = byName.get(name) ?? []; + byName.set(name, places); + const near = p ? places.find((q) => km(p, q.at) <= SAME_NAME_APART_KM) : places[0]; + if (near) { + label.set(id, near.label); + continue; + } + const own = places.length === 0 ? name : `${name} (${places.length + 1})`; + label.set(id, own); + if (p) { + places.push({ at: p, label: own }); + coords.set(own, p); + } + } + for (const list of calls.values()) for (const c of list) c.station = label.get(c.station) ?? ""; + const sameName = new Map(); + for (const [name, places] of byName) if (places.length > 1) sameName.set(name, places.map((q) => q.at)); + const wide = new Set(); if (op.narrow) { for (const [tripId, trip] of trips) { @@ -317,8 +361,10 @@ export async function readGtfsTimetable(dir: string, op: GtfsOperator, from: str const passes = op.passes(trip.family); const all = list.filter((c, i) => c.station && c.station !== list[i - 1]?.station); if (all.length >= 2) whole.push(tripOf(op.id, all, trip.runs, str(trip.trainNo), str(trip.family), passes)); + // v1 costs a row per pair of stops: a feed of every regional train reads trips only. + if (opts.v1 === false) continue; const keep = op.narrow?.(trip.family) ? all.filter((c) => wide.has(c.station)) : all; pushStopPairs(out, str, op.id, keep, trip.runs, trip.trainNo, trip.family, passes); } - return { trains: out, trips: whole, coords }; + return { trains: out, trips: whole, coords, sameName }; } diff --git a/src/merge.ts b/src/merge.ts new file mode 100644 index 0000000..db4f321 --- /dev/null +++ b/src/merge.ts @@ -0,0 +1,42 @@ +import type { Timetable } from "./format"; +import { SAME_NAME_APART_KM, km } from "./gtfs"; + +/** + * Several feeds of one country as one timetable. A station name two feeds both use is one + * station when they place it within a few km of each other (a regional feed and the + * national one), and two otherwise: the later one takes a number, "Gare SNCF (2)", as + * {@link readGtfsTimetable} does inside one feed. + */ +export function mergeTimetables(list: Timetable[]): Timetable { + const out: Timetable = { trains: [], trips: [], coords: new Map(), sameName: new Map() }; + // Every place each name stands for so far, with the name it was given. + const places = new Map(); + const seen = (label: string, at: [number, number]): void => { + const base = label.replace(/ \(\d+\)$/, ""); + const list = places.get(base) ?? []; + if (!list.some((p) => p.label === label)) list.push({ at, label }); + places.set(base, list); + }; + for (const tt of list) { + const rename = new Map(); + for (const [name, at] of tt.coords) { + const base = name.replace(/ \(\d+\)$/, ""); + const known = places.get(base) ?? []; + const near = known.find((p) => km(p.at, at) <= SAME_NAME_APART_KM); + let label = near?.label; + if (!label) { + let n = known.length + 1; + label = n === 1 ? base : `${base} (${n})`; + while (out.coords.has(label)) label = `${base} (${++n})`; + } + if (label !== name) rename.set(name, label); + if (!out.coords.has(label)) out.coords.set(label, at); + seen(label, at); + } + const to = (s: string): string => rename.get(s) ?? s; + for (const t of tt.trips) out.trips.push(rename.size ? { ...t, stops: t.stops.map((s) => ({ ...s, station: to(s.station) })) } : t); + for (const t of tt.trains) out.trains.push(rename.size ? { ...t, origin: to(t.origin), destination: to(t.destination) } : t); + } + for (const [base, list] of places) if (list.length > 1) out.sameName?.set(base, list.map((p) => p.at)); + return out; +} diff --git a/src/search.ts b/src/search.ts index dede748..51c6077 100644 --- a/src/search.ts +++ b/src/search.ts @@ -334,6 +334,47 @@ export class Network { } rawPlace[i] = p; }); + // A name the place's other stations agree on beats a busier one: first the name most + // of them normalize to ("Zürich HB" over the quay's "Zürich, Bahnhofquai"), then one + // sharing a word with most (Munich is "München Hbf", not the tram stop "Hauptbahnhof + // (U, Tram)" that merged into it), then the busiest. + const byPlace: number[][] = this.places.map(() => []); + raw.forEach((_, i) => byPlace[rawPlace[i] as number]?.push(i)); + const words = raw.map((s) => new Set(s.norm.split(" ").filter((w) => w.length >= 3))); + byPlace.forEach((list, p) => { + if (list.length < 3) return; + const located = (i: number): boolean => raw[i]?.lat !== null && raw[i]?.lon !== null; + const score = (i: number): [number, number, number] => { + let same = 0; + let shared = 0; + for (const j of list) { + if (j === i) continue; + if (raw[j]?.norm === raw[i]?.norm) same++; + if ([...(words[i] as Set)].some((w) => words[j]?.has(w))) shared++; + } + return [same, shared, calls[i] as number]; + }; + const better = (a: [number, number, number], b: [number, number, number]): boolean => + a[0] !== b[0] ? a[0] > b[0] : a[1] !== b[1] ? a[1] > b[1] : a[2] > b[2]; + const cur = namedBy[p] as number; + let best = cur; + let bestScore = score(cur); + for (const i of list) { + if (located(i) !== located(cur)) continue; + const sc = score(i); + if (better(sc, bestScore)) { + best = i; + bestScore = sc; + } + } + if (best === cur) return; + const s = raw[best] as RawStation; + const place = this.places[p] as Place; + namedBy[p] = best; + place.name = s.name; + place.lat = s.lat; + place.lon = s.lon; + }); this.placeOf = stationRaw.map((ids) => Int32Array.from(ids, (i) => rawPlace[i] as number)); const n = this.places.length; const change = opts.change ?? 10; diff --git a/tests/bundle.test.ts b/tests/bundle.test.ts index 1a4c48c..1e71c4e 100644 --- a/tests/bundle.test.ts +++ b/tests/bundle.test.ts @@ -1,4 +1,7 @@ import { describe, it, expect } from "vitest"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { buildIndexV2, buildOperatorBundles, tripsFromTrains } from "../src/api"; import { decodeBundle, encodeBundle, isBundle, type Bundle } from "../src/bundle"; @@ -9,6 +12,7 @@ import { RENFE } from "../src/operators/renfe"; const DIR = fileURLToPath(new URL("./fixtures/gtfs-mini", import.meta.url)); const GB = fileURLToPath(new URL("./fixtures/gb-schedule.json", import.meta.url)); +const RENFE_ANY = { ...RENFE, route: () => "RE" }; const INTERRAIL = { interrail: { bookable: true, seat: "unknown" as const } }; const trip = (over: Partial = {}): Trip => ({ @@ -68,6 +72,40 @@ describe("trip bundles", () => { expect(coords.get("Barcelona-Sants")).toEqual([41.38, 2.14]); }); + it("tells apart stations of one name in different towns", async () => { + const dir = mkdtempSync(join(tmpdir(), "gtfs-")); + const files: Record = { + agency: "agency_id,agency_name\nA,Rail", + routes: "route_id,agency_id,route_short_name,route_type\nR,A,RE,2", + trips: "route_id,service_id,trip_id,trip_short_name\nR,S,T1,1\nR,S,T2,2", + calendar_dates: "service_id,date,exception_type\nS,20261005,1", + // Two "Hauptbahnhof" 300 km apart, and one station's two platforms 200 m apart. + stops: [ + "stop_id,stop_name,stop_lat,stop_lon,parent_station", + "H1,Hauptbahnhof,52.52,13.37,", + "H2,Hauptbahnhof,50.11,8.66,", + "P1,Halle,51.48,11.98,", + "P2,Halle,51.4818,11.98,", + ].join("\n"), + stop_times: [ + "trip_id,arrival_time,departure_time,stop_id,stop_sequence", + "T1,08:00:00,08:00:00,H1,1", + "T1,09:00:00,09:00:00,P1,2", + "T2,10:00:00,10:00:00,P2,1", + "T2,12:00:00,12:00:00,H2,2", + ].join("\n"), + }; + for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, `${name}.txt`), body); + const { trips, coords, sameName } = await readGtfsTimetable(dir, RENFE_ANY, "2026-10-05", 1, { v1: false }); + expect([...(sameName ?? new Map()).entries()]).toEqual([["Hauptbahnhof", [[52.52, 13.37], [50.11, 8.66]]]]); + // Halle's two platforms stay one station; the second Hauptbahnhof is its own. + expect(trips.map((t) => t.stops.map((x) => x.station))).toEqual([ + ["Hauptbahnhof", "Halle"], + ["Halle", "Hauptbahnhof (2)"], + ]); + expect(coords.get("Hauptbahnhof (2)")).toEqual([50.11, 8.66]); + }); + it("keeps every stop of a narrowed family, which v1 cuts", async () => { const op = { ...RENFE, narrow: (f: string) => f === "AVE" }; const { trains, trips } = await readGtfsTimetable(DIR, op, "2026-10-05", 7); diff --git a/tests/merge.test.ts b/tests/merge.test.ts new file mode 100644 index 0000000..d433fd4 --- /dev/null +++ b/tests/merge.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect } from "vitest"; +import type { Timetable, Trip } from "../src/format"; +import { mergeTimetables } from "../src/merge"; + +const trip = (from: string, to: string): Trip => ({ + operator: "fr", + trainNo: "1", + dates: ["2026-11-02"], + stops: [ + { station: from, arr: 480, dep: 480 }, + { station: to, arr: 540, dep: 540 }, + ], + passes: {}, +}); + +const feed = (trips: Trip[], coords: [string, [number, number]][]): Timetable => ({ trains: [], trips, coords: new Map(coords) }); + +describe("mergeTimetables", () => { + it("keeps one station for a name two feeds place together, and numbers one placed elsewhere", () => { + const a = feed([trip("Gare SNCF", "Lyon Part-Dieu")], [["Gare SNCF", [45.9, 6.12]], ["Lyon Part-Dieu", [45.76, 4.86]]]); + // Annecy's again 500 m off, and another town's "Gare SNCF" 100 km away. + const b = feed([trip("Gare SNCF", "Lyon Part-Dieu")], [["Gare SNCF", [45.903, 6.124]], ["Lyon Part-Dieu", [45.76, 4.86]]]); + const c = feed([trip("Gare SNCF", "Lyon Part-Dieu")], [["Gare SNCF", [45.19, 5.71]], ["Lyon Part-Dieu", [45.76, 4.86]]]); + const m = mergeTimetables([a, b, c]); + expect(m.trips.map((t) => t.stops[0]?.station)).toEqual(["Gare SNCF", "Gare SNCF", "Gare SNCF (2)"]); + expect(m.coords.get("Gare SNCF")).toEqual([45.9, 6.12]); + expect(m.coords.get("Gare SNCF (2)")).toEqual([45.19, 5.71]); + expect([...(m.sameName ?? new Map()).keys()]).toEqual(["Gare SNCF"]); + // The feeds' own trips are left as they were. + expect(c.trips[0]?.stops[0]?.station).toBe("Gare SNCF"); + }); +}); diff --git a/tests/search.test.ts b/tests/search.test.ts index a5d8d86..6f458d2 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -87,6 +87,33 @@ describe("station matching", () => { expect(atocha).toMatchObject({ name: "Madrid-Puerta de Atocha", lat: 40.4066, calls: 3 }); }); + it("prefers a name its stations share over a busier one without the town", () => { + const tram = bundle( + "de", + [1, 2, 3, 4].map((i) => run(`T${i}`, [["Hauptbahnhof (U, Tram)", "07:00"], ["Pasing", "07:10"]])).concat([ + run("RB1", [["München Hbf", "08:00"], ["Garmisch-Partenkirchen", "09:22"]]), + ]), + { "Hauptbahnhof (U, Tram)": [48.1402, 11.5601], "München Hbf": [48.1403, 11.5600], Pasing: [48.15, 11.46], "Garmisch-Partenkirchen": [47.49, 11.1] }, + ); + const obb = bundle("at", [run("RJ1", [["München Hauptbahnhof", "10:00"], ["Salzburg Hbf", "11:30"]])], { + "München Hauptbahnhof": [48.1401, 11.5602], + "Salzburg Hbf": [47.81, 13.05], + }); + const net = new Network([tram, obb]); + expect(net.places.find((p) => p.stations.length === 3)?.name).toBe("München Hbf"); + }); + + it("keeps the name most of a place's stations use", () => { + const at = (op: string, name: string, n: number): SearchBundle => + bundle(op, Array.from({ length: n }, (_, i) => run(`${op}${i}`, [[name, "07:00"], ["Elsewhere " + op, "08:00"]])), { + [name]: [47.3779, 8.5403], + ["Elsewhere " + op]: [46 + n / 10, 7], + }); + // The quay's stop is busiest, but three operators call the place "Zürich HB". + const net = new Network([at("ch", "Zürich HB", 2), at("de", "Zürich HB", 1), at("at", "Zürich HB", 1), at("x", "Zürich, Bahnhofquai", 9)]); + expect(net.places.find((p) => p.stations.length === 4)?.name).toBe("Zürich HB"); + }); + it("groups a town's stations, placed or not, and finds the town first", () => { const paris = net.resolve("paris"); expect(paris?.kind).toBe("city");