diff --git a/package.json b/package.json index 359b85c..aa99572 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/player", - "version": "0.4.1", + "version": "0.5.0", "description": "One web player for every source a Profullstack site serves: MP4, HLS, MPEG-2 transport streams and audio, with one control bar, on desktop, mobile, PWA and television.", "keywords": [ "video", @@ -36,6 +36,10 @@ "types": "./dist/codecs.d.ts", "default": "./dist/codecs.js" }, + "./m3u": { + "types": "./dist/m3u.d.ts", + "default": "./dist/m3u.js" + }, "./react": { "types": "./dist/react/index.d.ts", "default": "./dist/react/index.js" diff --git a/src/m3u.ts b/src/m3u.ts new file mode 100644 index 0000000..4f01a71 --- /dev/null +++ b/src/m3u.ts @@ -0,0 +1,290 @@ +/** + * The m3u parser, with no player attached. + * + * genrewatch and tipoffwatch each carry a copy of this, for the same reason they + * each carried a copy of the codec table: the two sites are ports of one another + * and the playlist a reader hands over is read identically by both. Every rule + * below was written against a real provider list. + * + * A subpath rather than the root export, and deliberately runtime-neutral: no + * DOM, no `node:` imports, no `Buffer`. This runs on a Bun server ingesting a + * reader's catalogue into Postgres, which is the opposite end of the stack from + * the rest of this package -- so it must not drag a player, an engine or a + * dynamic import in behind it. + * + * ## Why the streaming form exists + * + * `parseM3u(text)` needs the whole file as one string, and the caller needed it + * as one string anyway to hash it. On a 300,000-entry catalogue that is a + * several-hundred-megabyte string, a second copy to hash it, and an array with + * one string per line on top -- which pushed the site that did it into a garbage + * collection spiral that pegged three cores and stopped it accepting TCP + * connections at all, every five minutes, for as long as it was left running. + * + * {@link parseM3uStream} never holds the file. It consumes chunks, hands each one + * straight back to the caller for hashing, and keeps only the entries it has + * decided to keep. + */ + +/** + * Hard ceiling on entries taken from one list. + * + * Was 20,000 and silently truncating: a reader importing a 300,000-entry + * catalogue got 20,000 rows, no error, and no way to tell which were missing. + */ +export const MAX_CHANNELS = 300_000; + +/** What an entry is, which decides how the page offers it and how it is played. */ +export type EntryKind = 'live' | 'vod' | 'series'; + +export interface M3uEntry { + title: string; + group: string | null; + url: string; + kind: EntryKind; +} + +export interface ParseOptions { + /** Stop after this many entries. */ + max?: number; +} + +/** + * Pull `key="value"` pairs out of the attribute block of an #EXTINF line. + * + * Only the block BEFORE the last comma is scanned. Attribute values routinely + * contain commas (`group-title="Movies, Drama"`), so splitting the line on the + * first comma and calling the rest the title -- the obvious implementation -- + * truncates the title of every channel whose group contains one. + */ +function parseAttrs(head: string): Record { + const out: Record = {}; + for (const m of head.matchAll(/([a-zA-Z0-9_-]+)="([^"]*)"/g)) { + const [, key, value] = m; + // Both groups are mandatory in the pattern, so this is only ever appeasing + // noUncheckedIndexedAccess -- but it costs nothing and the alternative is a + // non-null assertion in the one function every entry passes through. + if (key === undefined || value === undefined) continue; + out[key.toLowerCase()] = value; + } + return out; +} + +/** + * Live channel, film, or episode of something. + * + * Read from the URL first because providers are consistent about their path + * shapes and wildly inconsistent about their group names. The group is the + * fallback, not the other way round. + * + * This has to be worked out at parse time and stored: both sites seal the URL at + * rest, so asking later would mean inspecting an encrypted blob -- which is + * exactly what happened once, found no "/movie/" in a base64 string, and + * answered 'live' for every entry on every list. + */ +export function entryKind({ + url, + group, +}: { url?: string | null; group?: string | null } = {}): EntryKind { + const u = String(url ?? '').toLowerCase(); + if (/\/series\//.test(u)) return 'series'; + if (/\/(movie|movies|vod)\//.test(u)) return 'vod'; + if (/\.(mkv|mp4|avi|m4v)(\?|$)/.test(u)) return 'vod'; + if (/\/live\//.test(u) || /\.(ts|m3u8)(\?|$)/.test(u)) return 'live'; + + // Nothing in the URL says. Fall back to the group, which usually does. + const g = String(group ?? '').toLowerCase(); + if (/\b(vod|on ?demand|movies?|films?)\b/.test(g)) return 'vod'; + if (/\b(series|shows?|tv ?shows?)\b/.test(g)) return 'series'; + return 'live'; +} + +export interface M3uParser { + /** + * Feed one line. Returns false once the parser is full, which is the caller's + * signal that further lines are wasted work -- not that it may stop reading, + * because the bytes behind them usually still have to be hashed. + */ + push(line: string): boolean; + /** Everything kept so far. The same array throughout; not copied per push. */ + readonly entries: M3uEntry[]; + /** True once `max` entries have been kept. */ + readonly full: boolean; +} + +/** + * A line-at-a-time m3u parser. + * + * The whole-file version used to look ahead from each `#EXTINF` for its URL, + * which is why it needed an array of every line. The lookahead is really a + * two-state machine -- "waiting for an #EXTINF" and "holding one, waiting for its + * URL" -- and written that way it needs no more than the line in front of it. + * + * Only `#EXTINF` followed by a URL counts. Everything else (`#EXTM3U`, + * `#EXT-X-SESSION-DATA`, comments, blank lines) is skipped rather than guessed + * at, because a playlist that half-parses is worse than one that does not. + */ +export function createM3uParser({ max = MAX_CHANNELS }: ParseOptions = {}): M3uParser { + const entries: M3uEntry[] = []; + + /** `#EXTGRP:` is the other way providers state a group; it applies until changed. */ + let currentGroup: string | null = null; + /** The #EXTINF we are holding while we look for the URL that belongs to it. */ + let pending: { name: string; attrGroup: string | null } | null = null; + + const parser: M3uParser = { + get entries() { + return entries; + }, + get full() { + return entries.length >= max; + }, + push(raw: string): boolean { + if (entries.length >= max) return false; + const line = raw.trim(); + + if (line.startsWith('#EXTGRP:')) { + const g = line.slice('#EXTGRP:'.length).trim() || null; + /* + * An empty #EXTGRP clears the group, EXCEPT while an #EXTINF is waiting + * for its URL -- there it leaves the previous one standing. That is not a + * nicety; it is what the whole-file version did, because its inner + * lookahead loop used `|| currentGroup` where the outer loop did not, and + * a reader's genre index is built from these. + */ + currentGroup = pending ? (g ?? currentGroup) : g; + return true; + } + + if (pending) { + // Blank lines and any other directive sit between an #EXTINF and its URL + // on real lists -- #EXTVLCOPT especially. A second #EXTINF lands here too + // and is skipped as a directive, so of two in a row the first one wins. + if (!line || line.startsWith('#')) return true; + + const url = line; + const { name, attrGroup } = pending; + pending = null; + // A relative or non-http URL is not something either site can seal, proxy + // or hand to a player, so it is dropped along with its #EXTINF. + if (!/^https?:\/\//i.test(url)) return true; + if (!name) return true; + + const group = attrGroup || currentGroup || null; + entries.push({ title: name, group, url, kind: entryKind({ url, group }) }); + return true; + } + + if (!line.startsWith('#EXTINF')) return true; + + /* + * The title is everything after the LAST comma, not the first: the + * attribute block before it usually contains commas of its own. + */ + const comma = line.lastIndexOf(','); + if (comma < 0) return true; + const attrs = parseAttrs(line.slice(0, comma)); + // `tvg-name` is the fallback because a handful of providers ship an empty + // display title and put the real one in the attributes. + const name = line.slice(comma + 1).trim() || attrs['tvg-name'] || ''; + // Held even when the name is empty, so the URL line that follows is + // consumed as this entry's rather than mistaken for the next one's. + pending = { name, attrGroup: attrs['group-title'] || null }; + return true; + }, + }; + + return parser; +} + +/** + * Split an M3U held in memory into entries. + * + * Kept for callers that genuinely have the whole thing already -- a paste into a + * form, a fixture in a test. Anything reading from a network response wants + * {@link parseM3uStream} instead: this signature cannot avoid holding the file, + * and on a real catalogue that is the problem rather than the parsing. + */ +export function parseM3u(text: string, opts: ParseOptions = {}): M3uEntry[] { + const parser = createM3uParser(opts); + for (const line of String(text ?? '').split(/\r?\n/)) { + if (!parser.push(line)) break; + } + return parser.entries; +} + +export interface StreamOptions extends ParseOptions { + /** + * Called with every chunk, in order, before it is decoded. + * + * This is how the file gets hashed without anyone holding it: the caller feeds + * its own digest and never sees a whole-file string. Throwing from here aborts + * the parse and cancels the underlying stream, which is where a size ceiling + * belongs -- the policy and its error message are the caller's, not ours. + */ + onChunk?: (chunk: Uint8Array | string) => void; +} + +export interface StreamResult { + entries: M3uEntry[]; + /** Bytes seen. String chunks are counted by length, having no encoding here. */ + bytes: number; + /** True if `max` was reached and later entries were dropped. */ + truncated: boolean; +} + +/** + * Parse an m3u as it arrives, holding neither the file nor its lines. + * + * Pass a fetch body directly: `parseM3uStream(res.body, { onChunk })`. Chunks may + * be bytes or strings; bytes are decoded with a streaming TextDecoder so a + * multi-byte character split across a chunk boundary survives, and a `\r\n` split + * the same way is handled by carrying the tail of each chunk into the next. + * + * The stream is always read to the end, even once `max` entries have been kept, + * because `onChunk` is usually a hash and a hash of most of a file is worth + * nothing. Past that point the decoding and splitting stop, so the tail of an + * oversized list costs only the read. + */ +export async function parseM3uStream( + chunks: AsyncIterable, + { max = MAX_CHANNELS, onChunk }: StreamOptions = {} +): Promise { + const parser = createM3uParser({ max }); + const decoder = new TextDecoder('utf-8'); + let carry = ''; + let bytes = 0; + let truncated = false; + + for await (const chunk of chunks) { + bytes += typeof chunk === 'string' ? chunk.length : chunk.byteLength; + onChunk?.(chunk); + + // Full already: keep reading so the caller's digest stays whole, but stop + // paying for decode and split. + if (truncated) continue; + + const text = typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true }); + if (!text) continue; + + const parts = (carry + text).split(/\r?\n/); + // The last piece may be half a line; it is only complete at end of stream. + carry = parts.pop() ?? ''; + for (const line of parts) { + if (!parser.push(line)) { + truncated = true; + carry = ''; + break; + } + } + } + + if (!truncated) { + // Flush whatever the decoder was holding, then the final unterminated line -- + // a file whose last entry has no trailing newline is ordinary. + const tail = carry + decoder.decode(); + if (tail) parser.push(tail); + } + + return { entries: parser.entries, bytes, truncated: truncated || parser.full }; +} diff --git a/test/m3u.test.ts b/test/m3u.test.ts new file mode 100644 index 0000000..b00fad1 --- /dev/null +++ b/test/m3u.test.ts @@ -0,0 +1,262 @@ +import { describe, it, expect } from 'vitest'; +import { MAX_CHANNELS, createM3uParser, entryKind, parseM3u, parseM3uStream } from '../src/m3u'; + +/** + * The parsing cases are ported knowledge, not invented coverage: each one is a + * provider list that was read wrongly in production on one of the two sites. + * The streaming cases below them are new, and they are all about the seams -- + * a chunk boundary is free to land in the middle of a line, a CRLF or a + * multi-byte character, and each of those has its own way of losing an entry. + */ + +/** Feed a string as bytes, in fixed-size pieces, as a network body would arrive. */ +async function* inChunks(text: string, size: number): AsyncGenerator { + const bytes = new TextEncoder().encode(text); + for (let i = 0; i < bytes.length; i += size) yield bytes.slice(i, i + size); +} + +const LIST = [ + '#EXTM3U', + '#EXTINF:-1 tvg-id="bbc1" group-title="UK | Entertainment",BBC One HD', + 'http://example.test/live/u/p/1.ts', +].join('\n'); + +describe('parseM3u', () => { + it('reads the title, the group and the URL', () => { + expect(parseM3u(LIST)).toEqual([ + { + title: 'BBC One HD', + group: 'UK | Entertainment', + url: 'http://example.test/live/u/p/1.ts', + kind: 'live', + }, + ]); + }); + + it('does not let a comma inside an attribute eat the title', () => { + // Splitting on the FIRST comma produced a title of ` Action",Die Hard` for + // every channel whose group contained one, and provider groups routinely do. + const [ch] = parseM3u( + ['#EXTINF:-1 group-title="Movies, Action",Die Hard', 'https://example.test/vod/1.mp4'].join( + '\n' + ) + ); + expect(ch?.title).toBe('Die Hard'); + expect(ch?.group).toBe('Movies, Action'); + }); + + it('applies #EXTGRP until it is changed', () => { + const list = [ + '#EXTGRP:Documentary', + '#EXTINF:-1,Planet Earth', + 'https://example.test/1.ts', + '#EXTINF:-1,Blue Planet', + 'https://example.test/2.ts', + '#EXTGRP:Kids', + '#EXTINF:-1,Bluey', + 'https://example.test/3.ts', + ].join('\n'); + expect(parseM3u(list).map((c) => [c.title, c.group])).toEqual([ + ['Planet Earth', 'Documentary'], + ['Blue Planet', 'Documentary'], + ['Bluey', 'Kids'], + ]); + }); + + it('lets an explicit group-title beat an inherited #EXTGRP', () => { + const list = [ + '#EXTGRP:Kids', + '#EXTINF:-1 group-title="Horror",The Thing', + 'https://x.test/1.ts', + ].join('\n'); + expect(parseM3u(list)[0]?.group).toBe('Horror'); + }); + + it('skips an entry with no usable URL rather than guessing', () => { + const list = [ + '#EXTINF:-1,Broken', + 'rtmp://example.test/nope', + '#EXTINF:-1,Fine', + 'https://ok.test/x.ts', + ].join('\n'); + expect(parseM3u(list).map((c) => c.title)).toEqual(['Fine']); + }); + + it('steps over the directives providers put between an entry and its URL', () => { + const list = [ + '#EXTINF:-1,With options', + '#EXTVLCOPT:network-caching=1000', + '', + 'https://ok.test/x.ts', + ].join('\n'); + expect(parseM3u(list).map((c) => c.title)).toEqual(['With options']); + }); + + it('falls back to tvg-name when the display title is empty', () => { + const list = ['#EXTINF:-1 tvg-name="Named",', 'https://ok.test/x.ts'].join('\n'); + expect(parseM3u(list)[0]?.title).toBe('Named'); + }); + + it('keeps the first of two #EXTINF lines sharing one URL', () => { + // The whole-file parser skipped the second as a directive while looking ahead + // for the URL, so the first one claimed it. Preserved deliberately. + const list = ['#EXTINF:-1,First', '#EXTINF:-1,Second', 'https://ok.test/x.ts'].join('\n'); + expect(parseM3u(list).map((c) => c.title)).toEqual(['First']); + }); + + it('reads an entry with no trailing newline', () => { + expect(parseM3u('#EXTINF:-1,Last\nhttps://ok.test/x.ts')).toHaveLength(1); + }); + + it('stops at max', () => { + const many = Array.from({ length: 50 }, (_, i) => + [`#EXTINF:-1,Ch ${i}`, `https://ok.test/${i}.ts`].join('\n') + ).join('\n'); + expect(parseM3u(many)).toHaveLength(50); + expect(parseM3u(many, { max: 10 })).toHaveLength(10); + }); + + it('defaults to a ceiling that does not silently truncate a real catalogue', () => { + expect(MAX_CHANNELS).toBe(300_000); + }); +}); + +describe('entryKind', () => { + it('reads the URL before the group, because paths are the consistent half', () => { + expect(entryKind({ url: 'https://x.test/series/1.mkv' })).toBe('series'); + expect(entryKind({ url: 'https://x.test/movie/1.mkv' })).toBe('vod'); + expect(entryKind({ url: 'https://x.test/live/1.ts' })).toBe('live'); + expect(entryKind({ url: 'https://x.test/x.mp4' })).toBe('vod'); + }); + + it('falls back to the group when the URL says nothing', () => { + expect(entryKind({ url: 'https://x.test/a/b', group: 'VOD | Films' })).toBe('vod'); + expect(entryKind({ url: 'https://x.test/a/b', group: 'TV Shows' })).toBe('series'); + expect(entryKind({ url: 'https://x.test/a/b', group: 'News' })).toBe('live'); + }); +}); + +describe('parseM3uStream', () => { + const big = Array.from({ length: 200 }, (_, i) => + [ + `#EXTINF:-1 group-title="Grp ${i % 7}",Channel ${i}`, + `https://example.test/live/${i}.ts`, + ].join('\n') + ).join('\n'); + + it('agrees with the whole-file parser, at every chunk size', async () => { + // The point of the exercise: streaming must not change a single entry. Sizes + // chosen to land boundaries inside lines, attributes and URLs alike. + const expected = parseM3u(big); + for (const size of [1, 2, 3, 7, 13, 64, 1024, 1_000_000]) { + const got = await parseM3uStream(inChunks(big, size)); + expect(got.entries, `chunk size ${size}`).toEqual(expected); + } + }); + + it('survives a CRLF split across a chunk boundary', async () => { + const crlf = LIST.replace(/\n/g, '\r\n'); + // Cut the stream precisely between the \r and the \n. + const at = crlf.indexOf('\r\n', crlf.indexOf('#EXTINF')) + 1; + const bytes = new TextEncoder().encode(crlf); + async function* split() { + yield bytes.slice(0, at); + yield bytes.slice(at); + } + expect((await parseM3uStream(split())).entries).toEqual(parseM3u(crlf)); + }); + + it('survives a multi-byte character split across a chunk boundary', async () => { + const list = ['#EXTINF:-1,Ürdü Kanalı — 4K', 'https://ok.test/x.ts'].join('\n'); + const bytes = new TextEncoder().encode(list); + // Every possible cut point, so no split of the two- and three-byte sequences + // is left untried. + for (let at = 1; at < bytes.length; at++) { + async function* split() { + yield bytes.slice(0, at); + yield bytes.slice(at); + } + const got = await parseM3uStream(split()); + expect( + got.entries.map((c) => c.title), + `cut at ${at}` + ).toEqual(['Ürdü Kanalı — 4K']); + } + }); + + it('hands every chunk to onChunk, in order and unaltered', async () => { + // This is the file's hash. A digest over most of a file is worth nothing, so + // "every chunk" is the whole property. + const seen: number[] = []; + await parseM3uStream(inChunks(big, 64), { + onChunk: (c) => { + seen.push(...(c as Uint8Array)); + }, + }); + // Decoded rather than compared byte for byte: identical content is the claim, + // and a decoded string says so where two typed arrays only agree. + expect(new TextDecoder().decode(new Uint8Array(seen))).toBe(big); + }); + + it('keeps hashing past max, so a truncated list still gets a whole digest', async () => { + const seen: number[] = []; + const got = await parseM3uStream(inChunks(big, 64), { + max: 5, + onChunk: (c) => { + seen.push(...(c as Uint8Array)); + }, + }); + expect(got.entries).toHaveLength(5); + expect(got.truncated).toBe(true); + // Read to the end regardless. + expect(seen.length).toBe(new TextEncoder().encode(big).byteLength); + expect(got.bytes).toBe(seen.length); + }); + + it('reports bytes, which is what a caller schedules the next poll from', async () => { + const got = await parseM3uStream(inChunks(big, 999)); + expect(got.bytes).toBe(new TextEncoder().encode(big).byteLength); + expect(got.truncated).toBe(false); + }); + + it('accepts string chunks as well as bytes', async () => { + async function* strings() { + yield '#EXTINF:-1,A\nhttps://ok.test/'; + yield 'a.ts\n#EXTINF:-1,B\nhttps://ok.test/b.ts'; + } + const got = await parseM3uStream(strings()); + expect(got.entries.map((c) => c.title)).toEqual(['A', 'B']); + }); + + it('lets the caller abort mid-stream by throwing from onChunk', async () => { + // Where a size ceiling belongs: the policy and the wording are the caller's. + let read = 0; + await expect( + parseM3uStream(inChunks(big, 32), { + onChunk: () => { + read += 1; + if (read > 3) throw new Error('that list is larger than we store'); + }, + }) + ).rejects.toThrow(/larger than we store/); + expect(read).toBe(4); + }); + + it('handles an empty body', async () => { + async function* nothing(): AsyncGenerator {} + const got = await parseM3uStream(nothing()); + expect(got.entries).toEqual([]); + expect(got.bytes).toBe(0); + }); +}); + +describe('createM3uParser', () => { + it('says when it is full, so a caller can stop paying to decode', () => { + const p = createM3uParser({ max: 1 }); + expect(p.push('#EXTINF:-1,One')).toBe(true); + expect(p.push('https://ok.test/1.ts')).toBe(true); + expect(p.full).toBe(true); + expect(p.push('#EXTINF:-1,Two')).toBe(false); + expect(p.entries).toHaveLength(1); + }); +}); diff --git a/tsup.config.ts b/tsup.config.ts index 2e7130b..0d8462a 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -14,7 +14,10 @@ import { defineConfig } from 'tsup'; * anyone who already has one. */ export default defineConfig({ - entry: ['src/index.ts', 'src/react/index.tsx', 'src/codecs.ts'], + // m3u.ts is its own entry for the same reason codecs.ts is, only more so: it is + // imported by a Bun server, and it must be reachable without the root export + // pulling a player and its dynamic engine imports into a process that has no DOM. + entry: ['src/index.ts', 'src/react/index.tsx', 'src/codecs.ts', 'src/m3u.ts'], format: ['esm'], target: 'es2022', splitting: true,