From e46432d16ac699f546112e28d3b8c54a68fe211e Mon Sep 17 00:00:00 2001 From: Aurantium <280805072+Aurantium-47@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:57:47 -0700 Subject: [PATCH] feat(csv-parse): add an async iterator API Parse asynchronous chunk iterables with the existing CSV core, including EOF, early termination, types, packaged distributions, tests and documentation. Refs: #447 Assisted-by: OpenAI GPT-6 --- packages/csv-parse/README.md | 33 +- packages/csv-parse/dist/cjs/iterator.cjs | 2015 +++++++++ packages/csv-parse/dist/cjs/iterator.d.cts | 20 + packages/csv-parse/dist/esm/iterator.d.ts | 20 + packages/csv-parse/dist/esm/iterator.js | 3983 ++++++++++++++++ packages/csv-parse/dist/iife/iterator.js | 3992 ++++++++++++++++ packages/csv-parse/dist/umd/iterator.js | 3993 +++++++++++++++++ packages/csv-parse/lib/iterator.d.ts | 20 + packages/csv-parse/lib/iterator.js | 27 + packages/csv-parse/package.json | 22 +- packages/csv-parse/rollup.config.js | 34 + packages/csv-parse/samples/api.iterator.js | 16 + packages/csv-parse/test/api.iterator.js | 260 ++ packages/csv-parse/test/api.types.iterator.ts | 46 + .../test/option.group_columns_by_name.js | 2 +- 15 files changed, 14480 insertions(+), 3 deletions(-) create mode 100644 packages/csv-parse/dist/cjs/iterator.cjs create mode 100644 packages/csv-parse/dist/cjs/iterator.d.cts create mode 100644 packages/csv-parse/dist/esm/iterator.d.ts create mode 100644 packages/csv-parse/dist/esm/iterator.js create mode 100644 packages/csv-parse/dist/iife/iterator.js create mode 100644 packages/csv-parse/dist/umd/iterator.js create mode 100644 packages/csv-parse/lib/iterator.d.ts create mode 100644 packages/csv-parse/lib/iterator.js create mode 100644 packages/csv-parse/samples/api.iterator.js create mode 100644 packages/csv-parse/test/api.iterator.js create mode 100644 packages/csv-parse/test/api.types.iterator.ts diff --git a/packages/csv-parse/README.md b/packages/csv-parse/README.md index 180e320df..da4592e58 100644 --- a/packages/csv-parse/README.md +++ b/packages/csv-parse/README.md @@ -35,7 +35,38 @@ It implements the Node.js [`stream.Transform` API](http://nodejs.org/api/stream. Run `npm install csv` to install the full CSV module or run `npm install csv-parse` if you are only interested by the CSV parser. -Use the callback and sync APIs for simplicity or the stream based API for scalability. +Use the callback and sync APIs for simplicity, or the stream and async iterator APIs for incremental processing. + +### Async iterator API + +The `csv-parse/iterator` entrypoint parses an asynchronous iterable of string or +`Uint8Array` chunks, including Node.js Buffers, without a stream wrapper: + +```js +import { parse } from "csv-parse/iterator"; + +async function* chunks() { + yield "name,age\nAlice,"; + yield new TextEncoder().encode("20\nBob,30"); +} + +for await (const record of parse(chunks(), { columns: true, cast: true })) { + console.log(record); +} +``` + +All records from a chunk are yielded before the next chunk is requested. The +final record does not need a trailing newline. Parsing and input errors are +thrown during iteration. Once iteration has started, breaking from the loop, +calling the generator's `return()`, or reaching a `to`/`to_line` limit closes the +input iterator. +Strings are encoded as UTF-8; use byte chunks with the `encoding` option for +other encodings. The existing parser options and streaming record shapes apply. + +CommonJS consumers can use `require("csv-parse/iterator")`. For browsers, use +`csv-parse/browser/esm/iterator`; this distribution includes the required byte +buffer support and does not depend on Node.js streams. Parsing still processes +each input chunk synchronously, so callers should supply suitably sized chunks. ## Example diff --git a/packages/csv-parse/dist/cjs/iterator.cjs b/packages/csv-parse/dist/cjs/iterator.cjs new file mode 100644 index 000000000..566e27e71 --- /dev/null +++ b/packages/csv-parse/dist/cjs/iterator.cjs @@ -0,0 +1,2015 @@ +'use strict'; + +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if (Array.isArray(message)) message = message.join(" ").trim(); + super(message); + if (Error.captureStackTrace !== undefined) { + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for (const context of contexts) { + for (const key in context) { + const value = context[key]; + this[key] = Buffer.isBuffer(value) + ? value.toString(options.encoding) + : value == null + ? value + : JSON.parse(JSON.stringify(value)); + } + } + } +} + +const is_object = function (obj) { + return typeof obj === "object" && obj !== null && !Array.isArray(obj); +}; + +const normalize_columns_array = function (columns) { + const normalizedColumns = []; + for (let i = 0, l = columns.length; i < l; i++) { + const column = columns[i]; + if (column === undefined || column === null || column === false) { + normalizedColumns[i] = { disabled: true }; + } else if (typeof column === "string" || typeof column === "number") { + normalizedColumns[i] = { name: `${column}` }; + } else if (is_object(column)) { + if (typeof column.name !== "string") { + throw new CsvError("CSV_OPTION_COLUMNS_MISSING_NAME", [ + "Option columns missing name:", + `property "name" is required at position ${i}`, + "when column is an object literal", + ]); + } + normalizedColumns[i] = column; + } else { + throw new CsvError("CSV_INVALID_COLUMN_DEFINITION", [ + "Invalid column definition:", + "expect a string or a literal object,", + `got ${JSON.stringify(column)} at position ${i}`, + ]); + } + } + return normalizedColumns; +}; + +class ResizeableBuffer { + constructor(size = 100) { + this.size = size; + this.length = 0; + this.buf = Buffer.allocUnsafe(size); + } + prepend(val) { + if (Buffer.isBuffer(val)) { + const length = this.length + val.length; + if (length >= this.size) { + this.resize(); + if (length >= this.size) { + throw Error("INVALID_BUFFER_STATE"); + } + } + const buf = this.buf; + this.buf = Buffer.allocUnsafe(this.size); + val.copy(this.buf, 0); + buf.copy(this.buf, val.length); + this.length += val.length; + } else { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + const buf = this.clone(); + this.buf[0] = val; + buf.copy(this.buf, 1, 0, length); + } + } + append(val) { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + this.buf[length] = val; + } + clone() { + return Buffer.from(this.buf.slice(0, this.length)); + } + resize() { + const length = this.length; + this.size = this.size * 2; + const buf = Buffer.allocUnsafe(this.size); + this.buf.copy(buf, 0, 0, length); + this.buf = buf; + } + toString(encoding) { + if (encoding) { + return this.buf.toString(encoding, 0, this.length); + } else { + return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)); + } + } + toJSON() { + return this.toString("utf8"); + } + reset() { + this.length = 0; + } +} + +const init_state = function (options) { + // ECMAScript WhiteSpace + LineTerminator codepoints, encoded under + // `options.encoding`. Aligns trimming with `String.prototype.trim()`. + // https://tc39.es/ecma262/#sec-white-space + // https://tc39.es/ecma262/#sec-line-terminators + // + // Codepoints unrepresentable in the target encoding are dropped: Node's + // Buffer substitutes them with `?` (0x3F), and including those would cause + // literal `?` bytes in the input to be trimmed under `latin1`/`ascii`. + const timchars = [ + // Basic Latin + 0x0020, // [Space](https://www.fileformat.info/info/unicode/char/0020/index.htm) + 0x0009, // [CHARACTER TABULATION (HT)](https://www.fileformat.info/info/unicode/char/0009/index.htm) + 0x000a, // [LINE FEED (LF)](https://www.fileformat.info/info/unicode/char/000a/index.htm) + 0x000d, // [CARRIAGE RETURN (CR)](https://www.fileformat.info/info/unicode/char/000d/index.htm) + 0x000c, // [FORM FEED (FF)](https://www.fileformat.info/info/unicode/char/000c/index.htm) + 0x000b, // [LINE TABULATION (VT)](https://www.fileformat.info/info/unicode/char/000b/index.htm) + // Latin-1 Supplement + 0x00a0, // [NO-BREAK SPACE (NBSP)](https://www.fileformat.info/info/unicode/char/00a0/index.htm) + // Ogham + 0x1680, // [OGHAM SPACE MARK](https://www.fileformat.info/info/unicode/char/1680/index.htm) + // General Punctuation + 0x2000, // [EN QUAD](https://www.fileformat.info/info/unicode/char/2000/index.htm) + 0x2001, // [EM QUAD](https://www.fileformat.info/info/unicode/char/2001/index.htm) + 0x2002, // [EN SPACE](https://www.fileformat.info/info/unicode/char/2002/index.htm) + 0x2003, // [EM SPACE](https://www.fileformat.info/info/unicode/char/2003/index.htm) + 0x2004, // [THREE-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2004/index.htm) + 0x2005, // [FOUR-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2005/index.htm) + 0x2006, // [SIX-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2006/index.htm) + 0x2007, // [FIGURE SPACE](https://www.fileformat.info/info/unicode/char/2007/index.htm) + 0x2008, // [PUNCTUATION SPACE](https://www.fileformat.info/info/unicode/char/2008/index.htm) + 0x2009, // [THIN SPACE](https://www.fileformat.info/info/unicode/char/2009/index.htm) + 0x200a, // [HAIR SPACE](https://www.fileformat.info/info/unicode/char/200a/index.htm) + 0x2028, // [LINE SEPARATOR](https://www.fileformat.info/info/unicode/char/2028/index.htm) + 0x2029, // [PARAGRAPH SEPARATOR](https://www.fileformat.info/info/unicode/char/2029/index.htm) + 0x202f, // [NARROW NO-BREAK SPACE (NNBSP)](https://www.fileformat.info/info/unicode/char/202f/index.htm) + 0x205f, // [MEDIUM MATHEMATICAL SPACE (MMSP)](https://www.fileformat.info/info/unicode/char/205f/index.htm) + 0x3000, // [IDEOGRAPHIC SPACE](https://www.fileformat.info/info/unicode/char/3000/index.htm) + 0xfeff, // [ZERO WIDTH NO-BREAK SPACE (BOM)](https://www.fileformat.info/info/unicode/char/feff/index.htm) + ].reduce((acc, codepoint) => { + const encoded = Buffer.from( + String.fromCharCode(codepoint), + options.encoding, + ); + if (codepoint !== 0x3f && encoded.length === 1 && encoded[0] === 0x3f) { + return acc; + } + acc.push(encoded); + return acc; + }, []); + // First-byte lookup table for `__isCharTrimable`. Non-whitespace bytes + // (the common case) bail out in O(1) without scanning every timchar. + const timcharFirstBytes = new Uint8Array(256); + for (const t of timchars) timcharFirstBytes[t[0]] = 1; + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + delimiterBufPrevious: undefined, + delimiterDiscovered: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: + Buffer.isBuffer(options.escape) && + Buffer.isBuffer(options.quote) && + Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) + ? options.columns.length + : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...(options.delimiter + ? options.delimiter.map((delimiter) => delimiter.length) + : []), + // Auto discovery of delimiter is limited to 1 character + options.delimiter_auto ? 1 : 0, + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + // Skip if the remaining buffer can be a multi-byte trim character + ...timchars.map((t) => t.length), + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: + options.record_delimiter.length === 0 + ? 0 + : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [ + Buffer.from(" ", options.encoding)[0], + Buffer.from("\t", options.encoding)[0], + ], + wasQuoting: false, + wasRowDelimiter: false, + timchars: timchars, + timcharFirstBytes: timcharFirstBytes, + }; +}; + +const underscore = function (str) { + return str.replace(/([A-Z])/g, function (_, match) { + return "_" + match.toLowerCase(); + }); +}; + +const normalize_options = function (opts) { + const options = {}; + // Merge with user options + for (const opt in opts) { + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if (options.encoding === undefined || options.encoding === true) { + options.encoding = "utf8"; + } else if (options.encoding === null || options.encoding === false) { + options.encoding = null; + } else if ( + typeof options.encoding !== "string" && + options.encoding !== null + ) { + throw new CsvError( + "CSV_INVALID_OPTION_ENCODING", + [ + "Invalid option encoding:", + "encoding must be a string or null to return a buffer,", + `got ${JSON.stringify(options.encoding)}`, + ], + options, + ); + } + // Normalize option `bom` + if ( + options.bom === undefined || + options.bom === null || + options.bom === false + ) { + options.bom = false; + } else if (options.bom !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_BOM", + [ + "Invalid option bom:", + "bom must be true,", + `got ${JSON.stringify(options.bom)}`, + ], + options, + ); + } + // Normalize option `cast` + options.cast_function = null; + if ( + options.cast === undefined || + options.cast === null || + options.cast === false || + options.cast === "" + ) { + options.cast = undefined; + } else if (typeof options.cast === "function") { + options.cast_function = options.cast; + options.cast = true; + } else if (options.cast !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_CAST", + [ + "Invalid option cast:", + "cast must be true or a function,", + `got ${JSON.stringify(options.cast)}`, + ], + options, + ); + } + // Normalize option `cast_date` + if ( + options.cast_date === undefined || + options.cast_date === null || + options.cast_date === false || + options.cast_date === "" + ) { + options.cast_date = false; + } else if (options.cast_date === true) { + options.cast_date = function (value) { + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + } else if (typeof options.cast_date !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_CAST_DATE", + [ + "Invalid option cast_date:", + "cast_date must be true or a function,", + `got ${JSON.stringify(options.cast_date)}`, + ], + options, + ); + } + // Normalize option `columns` + options.cast_first_line_to_header = undefined; + if (options.columns === true) { + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + } else if (typeof options.columns === "function") { + options.cast_first_line_to_header = options.columns; + options.columns = true; + } else if (Array.isArray(options.columns)) { + options.columns = normalize_columns_array(options.columns); + } else if ( + options.columns === undefined || + options.columns === null || + options.columns === false + ) { + options.columns = false; + } else { + throw new CsvError( + "CSV_INVALID_OPTION_COLUMNS", + [ + "Invalid option columns:", + "expect an array, a function or true,", + `got ${JSON.stringify(options.columns)}`, + ], + options, + ); + } + // Normalize option `group_columns_by_name` + if ( + options.group_columns_by_name === undefined || + options.group_columns_by_name === null || + options.group_columns_by_name === false + ) { + options.group_columns_by_name = false; + } else if (options.group_columns_by_name !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "expect a boolean,", + `got ${JSON.stringify(options.group_columns_by_name)}`, + ], + options, + ); + } else if (options.columns === false) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "the `columns` mode must be activated.", + ], + options, + ); + } + // Normalize option `comment` + if ( + options.comment === undefined || + options.comment === null || + options.comment === false || + options.comment === "" + ) { + options.comment = null; + } else { + if (typeof options.comment === "string") { + options.comment = Buffer.from(options.comment, options.encoding); + } + if (!Buffer.isBuffer(options.comment)) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment:", + "comment must be a buffer or a string,", + `got ${JSON.stringify(options.comment)}`, + ], + options, + ); + } + } + // Normalize option `comment_no_infix` + if ( + options.comment_no_infix === undefined || + options.comment_no_infix === null || + options.comment_no_infix === false + ) { + options.comment_no_infix = false; + } else if (options.comment_no_infix !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment_no_infix:", + "value must be a boolean,", + `got ${JSON.stringify(options.comment_no_infix)}`, + ], + options, + ); + } + // Normalize option `delimiter_auto` + if ( + options.delimiter_auto === undefined || + options.delimiter_auto === null || + options.delimiter_auto === false + ) { + options.delimiter_auto = false; + } else if (options.delimiter_auto === true) { + options.delimiter_auto = {}; + } else if (!is_object(options.delimiter_auto)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "delimiter_auto must be a boolean or a configuration object,", + `got ${JSON.stringify(options.delimiter_auto)}`, + ], + options, + ); + } + if (options.delimiter_auto) { + if (options.delimiter_auto.preferred === undefined) + options.delimiter_auto.preferred = { + [",".charCodeAt(0)]: 1.8, + ["\t".charCodeAt(0)]: 1.8, + [";".charCodeAt(0)]: 1.6, + [" ".charCodeAt(0)]: 1.6, + [":".charCodeAt(0)]: 1.5, + [".".charCodeAt(0)]: 1.4, + ["/".charCodeAt(0)]: 1.4, + }; + else if (!is_object(options.delimiter_auto.preferred)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "preferred must be an object,", + `got ${JSON.stringify(options.delimiter_auto.preferred)}`, + ], + options, + ); + } + if (options.delimiter_auto.score === undefined) + options.delimiter_auto.score = (info, options) => { + return ( + (info.total - info.std) * (options.preferred[info.char_code] || 1) + ); + }; + else if (typeof options.delimiter_auto.score !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "score must be a function,", + `got ${JSON.stringify(options.delimiter_auto.score)}`, + ], + options, + ); + } + if (options.delimiter_auto.size === undefined) + options.delimiter_auto.size = 2048; + else if (typeof options.delimiter_auto.size !== "number") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "size must be a number,", + `got ${JSON.stringify(options.delimiter_auto.size)}`, + ], + options, + ); + } + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if (options.delimiter_auto !== false) { + options.delimiter = []; + } + if (!Array.isArray(options.delimiter)) { + if ( + options.delimiter === undefined || + options.delimiter === null || + options.delimiter === false + ) { + options.delimiter = Buffer.from(",", options.encoding); + } + options.delimiter = [options.delimiter]; + } + options.delimiter = options.delimiter.map(function (delimiter) { + if (typeof delimiter === "string") { + delimiter = Buffer.from(delimiter, options.encoding); + } + if (!Buffer.isBuffer(delimiter) || delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER", + [ + "Invalid option delimiter:", + "delimiter must be a non empty string or buffer or array of string|buffer,", + `got ${delimiter_json}`, + ], + options, + ); + } + return delimiter; + }); + // Normalize option `escape` + if (options.escape === undefined || options.escape === true) { + options.escape = Buffer.from('"', options.encoding); + } else if (typeof options.escape === "string") { + options.escape = Buffer.from(options.escape, options.encoding); + } else if (options.escape === null || options.escape === false) { + options.escape = null; + } + if (options.escape !== null) { + if (!Buffer.isBuffer(options.escape)) { + throw new Error( + `Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`, + ); + } + } + // Normalize option `from` + if (options.from === undefined || options.from === null) { + options.from = 1; + } else { + if (typeof options.from === "string" && /\d+/.test(options.from)) { + options.from = parseInt(options.from); + } + if (Number.isInteger(options.from)) { + if (options.from < 0) { + throw new Error( + `Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`, + ); + } + } + // Normalize option `from_line` + if (options.from_line === undefined || options.from_line === null) { + options.from_line = 1; + } else { + if ( + typeof options.from_line === "string" && + /\d+/.test(options.from_line) + ) { + options.from_line = parseInt(options.from_line); + } + if (Number.isInteger(options.from_line)) { + if (options.from_line <= 0) { + throw new Error( + `Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`, + ); + } + } + // Normalize options `ignore_last_delimiters` + if ( + options.ignore_last_delimiters === undefined || + options.ignore_last_delimiters === null + ) { + options.ignore_last_delimiters = false; + } else if (typeof options.ignore_last_delimiters === "number") { + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if (options.ignore_last_delimiters === 0) { + options.ignore_last_delimiters = false; + } + } else if (typeof options.ignore_last_delimiters !== "boolean") { + throw new CsvError( + "CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS", + [ + "Invalid option `ignore_last_delimiters`:", + "the value must be a boolean value or an integer,", + `got ${JSON.stringify(options.ignore_last_delimiters)}`, + ], + options, + ); + } + if (options.ignore_last_delimiters === true && options.columns === false) { + throw new CsvError( + "CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS", + [ + "The option `ignore_last_delimiters`", + "requires the activation of the `columns` option", + ], + options, + ); + } + // Normalize option `info` + if ( + options.info === undefined || + options.info === null || + options.info === false + ) { + options.info = false; + } else if (options.info !== true) { + throw new Error( + `Invalid Option: info must be true, got ${JSON.stringify(options.info)}`, + ); + } + // Normalize option `max_record_size` + if ( + options.max_record_size === undefined || + options.max_record_size === null || + options.max_record_size === false + ) { + options.max_record_size = 0; + } else if ( + Number.isInteger(options.max_record_size) && + options.max_record_size >= 0 + ) ; else if ( + typeof options.max_record_size === "string" && + /\d+/.test(options.max_record_size) + ) { + options.max_record_size = parseInt(options.max_record_size); + } else { + throw new Error( + `Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`, + ); + } + // Normalize option `objname` + if ( + options.objname === undefined || + options.objname === null || + options.objname === false + ) { + options.objname = undefined; + } else if (Buffer.isBuffer(options.objname)) { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if (options.encoding === null) ; else { + options.objname = options.objname.toString(options.encoding); + } + } else if (typeof options.objname === "string") { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + } else if (typeof options.objname === "number") ; else { + throw new Error( + `Invalid Option: objname must be a string or a buffer, got ${options.objname}`, + ); + } + if (options.objname !== undefined) { + if (typeof options.objname === "number") { + if (options.columns !== false) { + throw Error( + "Invalid Option: objname index cannot be combined with columns or be defined as a field", + ); + } + } else { + // A string or a buffer + if (options.columns === false) { + throw Error( + "Invalid Option: objname field must be combined with columns or be defined as an index", + ); + } + } + } + // Normalize option `on_record` + if (options.on_record === undefined || options.on_record === null) { + options.on_record = undefined; + } else if (typeof options.on_record !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_ON_RECORD", + [ + "Invalid option `on_record`:", + "expect a function,", + `got ${JSON.stringify(options.on_record)}`, + ], + options, + ); + } + // Normalize option `on_skip` + // options.on_skip ??= (err, chunk) => { + // this.emit('skip', err, chunk); + // }; + if ( + options.on_skip !== undefined && + options.on_skip !== null && + typeof options.on_skip !== "function" + ) { + throw new Error( + `Invalid Option: on_skip must be a function, got ${JSON.stringify(options.on_skip)}`, + ); + } + // Normalize option `quote` + if ( + options.quote === null || + options.quote === false || + options.quote === "" + ) { + options.quote = null; + } else { + if (options.quote === undefined || options.quote === true) { + options.quote = Buffer.from('"', options.encoding); + } else if (typeof options.quote === "string") { + options.quote = Buffer.from(options.quote, options.encoding); + } + if (!Buffer.isBuffer(options.quote)) { + throw new Error( + `Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`, + ); + } + } + // Normalize option `raw` + if ( + options.raw === undefined || + options.raw === null || + options.raw === false + ) { + options.raw = false; + } else if (options.raw !== true) { + throw new Error( + `Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`, + ); + } + // Normalize option `record_delimiter` + if (options.record_delimiter === undefined) { + options.record_delimiter = []; + } else if ( + typeof options.record_delimiter === "string" || + Buffer.isBuffer(options.record_delimiter) + ) { + if (options.record_delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = [options.record_delimiter]; + } else if (!Array.isArray(options.record_delimiter)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = options.record_delimiter.map(function (rd, i) { + if (typeof rd !== "string" && !Buffer.isBuffer(rd)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } else if (rd.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } + if (typeof rd === "string") { + rd = Buffer.from(rd, options.encoding); + } + return rd; + }); + // Normalize option `relax_column_count` + if (typeof options.relax_column_count === "boolean") ; else if ( + options.relax_column_count === undefined || + options.relax_column_count === null + ) { + options.relax_column_count = false; + } else { + throw new Error( + `Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`, + ); + } + if (typeof options.relax_column_count_less === "boolean") ; else if ( + options.relax_column_count_less === undefined || + options.relax_column_count_less === null + ) { + options.relax_column_count_less = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`, + ); + } + if (typeof options.relax_column_count_more === "boolean") ; else if ( + options.relax_column_count_more === undefined || + options.relax_column_count_more === null + ) { + options.relax_column_count_more = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`, + ); + } + // Normalize option `relax_quotes` + if (typeof options.relax_quotes === "boolean") ; else if ( + options.relax_quotes === undefined || + options.relax_quotes === null + ) { + options.relax_quotes = false; + } else { + throw new Error( + `Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`, + ); + } + // Normalize option `skip_empty_lines` + if (typeof options.skip_empty_lines === "boolean") ; else if ( + options.skip_empty_lines === undefined || + options.skip_empty_lines === null + ) { + options.skip_empty_lines = false; + } else { + throw new Error( + `Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`, + ); + } + // Normalize option `skip_records_with_empty_values` + if (typeof options.skip_records_with_empty_values === "boolean") ; else if ( + options.skip_records_with_empty_values === undefined || + options.skip_records_with_empty_values === null + ) { + options.skip_records_with_empty_values = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`, + ); + } + // Normalize option `skip_records_with_error` + if (typeof options.skip_records_with_error === "boolean") ; else if ( + options.skip_records_with_error === undefined || + options.skip_records_with_error === null + ) { + options.skip_records_with_error = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`, + ); + } + // Normalize option `rtrim` + if ( + options.rtrim === undefined || + options.rtrim === null || + options.rtrim === false + ) { + options.rtrim = false; + } else if (options.rtrim !== true) { + throw new Error( + `Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`, + ); + } + // Normalize option `ltrim` + if ( + options.ltrim === undefined || + options.ltrim === null || + options.ltrim === false + ) { + options.ltrim = false; + } else if (options.ltrim !== true) { + throw new Error( + `Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`, + ); + } + // Normalize option `trim` + if ( + options.trim === undefined || + options.trim === null || + options.trim === false + ) { + options.trim = false; + } else if (options.trim !== true) { + throw new Error( + `Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`, + ); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if (options.trim === true && opts.ltrim !== false) { + options.ltrim = true; + } else if (options.ltrim !== true) { + options.ltrim = false; + } + if (options.trim === true && opts.rtrim !== false) { + options.rtrim = true; + } else if (options.rtrim !== true) { + options.rtrim = false; + } + // Normalize option `to` + if (options.to === undefined || options.to === null) { + options.to = -1; + } else if (options.to !== -1) { + if (typeof options.to === "string" && /\d+/.test(options.to)) { + options.to = parseInt(options.to); + } + if (Number.isInteger(options.to)) { + if (options.to <= 0) { + throw new Error( + `Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`, + ); + } + } + // Normalize option `to_line` + if (options.to_line === undefined || options.to_line === null) { + options.to_line = -1; + } else if (options.to_line !== -1) { + if (typeof options.to_line === "string" && /\d+/.test(options.to_line)) { + options.to_line = parseInt(options.to_line); + } + if (Number.isInteger(options.to_line)) { + if (options.to_line <= 0) { + throw new Error( + `Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`, + ); + } + } + return options; +}; + +// Discussed in [issue #400](https://github.com/adaltas/node-csv/issues/400) +// See https://github.com/python/cpython/blob/ea1b1c579f600cc85d145c60862b2e6b98701b24/Lib/csv.py#L349 +const delimiter_discover = function (records, options) { + // Normalize the configuration + if (!options) { + ({ delimiter_auto: options } = normalize_options({ delimiter_auto: true })); + } + // Convert String to Buffer + if (typeof records === "string") { + records = Buffer.from(records); + } + // Convert Buffer to an array of records + if (Buffer.isBuffer(records)) { + records = ((data) => { + const records = []; + const parser = transform({ delimiter: [] }); + const push = (record) => records.push(record); + const close = () => {}; + const error = parser.parse(data, true, push, close); + if (error !== undefined) throw error; + return records; + })(records); + } + // Info array initialization, 127 entries, one per char code + const info = Array(127) + .fill() + .map(() => ({ lines: [] })); + // Traverse each records, count occurences per char code + records.map(([record], line) => { + for (let i = 0, l = record.length; i < l; i++) { + // Count the character frequency + const code = record.charCodeAt(i); + if (info[code].lines[line] === undefined) info[code].lines[line] = 0; + info[code].lines[line]++; + } + }); + // Traverse each char code, compute the score + info.map((info, i) => { + info.char_code = i; + info.std = std(info.lines); + info.total = info.lines.reduce((acc, val) => acc + val, 0); + info.preferred = !!options.preferred[i]; + info.score = options.score(info, options); + }); + // Extract the dominant character + const result = info.reduce( + (acc, info) => (acc.score > info.score ? acc : info), + {}, + ); + return String.fromCharCode(result.char_code); +}; + +const std = function (array) { + const n = array.length; + if (n === 0) return 0; + const mean = array.reduce((a, b) => a + b) / n; + return Math.sqrt( + array.map((x) => Math.pow(x - mean, 2)).reduce((a, b) => a + b) / n, + ); +}; + +const isRecordEmpty = function (record) { + return record.every( + (field) => + field == null || (field.toString && field.toString().trim() === ""), + ); +}; + +const cr = 13; // `\r`, carriage return, 0x0D in hexadécimal, 13 in decimal +const nl = 10; // `\n`, newline, 0x0A in hexadecimal, 10 in decimal + +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + utf8: Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + utf16le: Buffer.from([255, 254]), +}; + +const transform = function (original_options = {}) { + const info = { + bytes: 0, + bytes_records: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0, + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function (i, bufLen, end) { + if (end) return false; + const { encoding, escape, quote } = this.options; + const { quoting, needMoreDataSize, recordDelimiterMaxLength } = + this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + // If "record_delimiter" is yet to be discovered: + // 1. It is equals to `[]` and "recordDelimiterMaxLength" equals `0` + // 2. We set the length to windows line ending in the current encoding + // Note, that encoding is known from user or bom discovery at that point + // recordDelimiterMaxLength, + recordDelimiterMaxLength === 0 + ? Buffer.from("\r\n", encoding).length + : recordDelimiterMaxLength, + // Skip if remaining buffer can be an escaped quote + quoting ? (escape === null ? 0 : escape.length) + quote.length : 0, + // Skip if remaining buffer can be record delimiter following the closing quote + quoting ? quote.length + recordDelimiterMaxLength : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function (nextBuf, end, push, close) { + const { + bom, + comment_no_infix, + delimiter_auto, + encoding, + from_line, + ltrim, + max_record_size, + raw, + relax_quotes, + rtrim, + skip_empty_lines, + to, + to_line, + } = this.options; + let { comment, escape, quote, record_delimiter } = this.options; + const { + bomSkipped, + delimiterDiscovered, + delimiterBufPrevious, + rawBuffer, + escapeIsQuote, + } = this.state; + // Automatic delimiter discovery + if (!delimiterDiscovered && delimiter_auto) { + let delimiterBuf; + if (delimiterBufPrevious === undefined) { + delimiterBuf = nextBuf; + } else if ( + delimiterBufPrevious !== undefined && + nextBuf === undefined + ) { + delimiterBuf = delimiterBufPrevious; + } else { + delimiterBuf = Buffer.concat([delimiterBufPrevious, nextBuf]); + } + // Ensure that nextBuf is not concatenated a second time during buffer reconciliation + nextBuf = undefined; + // this.delimiterBufPrevious = delimiterBuf; + if (end || delimiterBuf.length > delimiter_auto.size) { + this.options.delimiter = [ + Buffer.from( + delimiter_discover(delimiterBuf, this.options.delimiter_auto), + ), + ]; + this.state.previousBuf = delimiterBuf; + this.state.delimiterBufPrevious = undefined; + this.state.delimiterDiscovered = true; + } else { + this.state.delimiterBufPrevious = delimiterBuf; + return; + } + } + // Previous buffers reconciliation + const { previousBuf } = this.state; + let buf; + if (previousBuf === undefined) { + if (nextBuf === undefined) { + // Handle empty string + close(); + return; + } else { + buf = nextBuf; + } + } else if (previousBuf !== undefined && nextBuf === undefined) { + buf = previousBuf; + } else { + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if (bomSkipped === false) { + if (bom === false) { + this.state.bomSkipped = true; + } else if (buf.length < 3) { + // No enough data + if (end === false) { + // Wait for more data + this.state.previousBuf = buf; + return; + } + } else { + for (const encoding in boms) { + if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) { + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + const options = normalize_options({ + ...this.original_options, + encoding: encoding, + }); + // Properties are merged with the existing options instance + for (const key in options) { + this.options[key] = options[key]; + } + // Options will re-evaluate the Buffer with the new encoding + ({ comment, escape, quote } = this.options); + break; + } + } + this.state.bomSkipped = true; + } + } + const bufLen = buf.length; + let pos; + for (pos = 0; pos < bufLen; pos++) { + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if (this.__needMoreData(pos, bufLen, end)) { + break; + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if (to_line !== -1 && this.info.lines > to_line) { + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if (this.state.quoting === false && record_delimiter.length === 0) { + const record_delimiterCount = this.__autoDiscoverRecordDelimiter( + buf, + pos, + ); + if (record_delimiterCount) { + record_delimiter = this.options.record_delimiter; + } + } + const chr = buf[pos]; + if (raw === true) { + rawBuffer.append(chr); + } + if ( + (chr === cr || chr === nl) && + this.state.wasRowDelimiter === false + ) { + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if (this.state.escaping === true) { + this.state.escaping = false; + } else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if ( + escape !== null && + this.state.quoting === true && + this.__isEscape(buf, pos, chr) && + pos + escape.length < bufLen + ) { + if (escapeIsQuote) { + if (this.__isQuote(buf, pos + escape.length)) { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } else { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if (this.state.commenting === false && this.__isQuote(buf, pos)) { + if (this.state.quoting === true) { + const nextChr = buf[pos + quote.length]; + const isNextChrTrimable = + rtrim && this.__isCharTrimable(buf, pos + quote.length); + const isNextChrComment = + comment !== null && + this.__compareBytes(comment, buf, pos + quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter( + buf, + pos + quote.length, + nextChr, + ); + const isNextChrRecordDelimiter = + record_delimiter.length === 0 + ? this.__autoDiscoverRecordDelimiter(buf, pos + quote.length) + : this.__isRecordDelimiter(nextChr, buf, pos + quote.length); + // Escape a quote + // Treat next char as a regular character + if ( + escape !== null && + this.__isEscape(buf, pos, chr) && + this.__isQuote(buf, pos + escape.length) + ) { + pos += escape.length - 1; + } else if ( + !nextChr || + isNextChrDelimiter || + isNextChrRecordDelimiter || + isNextChrComment || + isNextChrTrimable + ) { + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + } else if (relax_quotes === false) { + const err = this.__error( + new CsvError( + "CSV_INVALID_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + "instead of delimiter, record delimiter, trimable character", + "(if activated) or comment", + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; + } + } else { + if (this.state.field.length !== 0) { + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if (relax_quotes === false) { + const info = this.__infoField(); + const bom = Object.keys(boms) + .map((b) => + boms[b].equals(this.state.field.toString()) ? b : false, + ) + .filter(Boolean)[0]; + const err = this.__error( + new CsvError( + "INVALID_OPENING_QUOTE", + [ + "Invalid Opening Quote:", + `a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`, + bom ? `(${bom} bom)` : undefined, + ], + this.options, + info, + { + field: this.state.field, + }, + ), + ); + if (err !== undefined) return err; + } + } else { + this.state.quoting = true; + pos += quote.length - 1; + continue; + } + } + } + if (this.state.quoting === false) { + const recordDelimiterLength = this.__isRecordDelimiter( + chr, + buf, + pos, + ); + if (recordDelimiterLength !== 0) { + // Do not emit comments which take a full line + const skipCommentLine = + this.state.commenting && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0; + if (skipCommentLine) { + this.info.comment_lines++; + // Skip full comment line + } else { + // Activate records emission if above from_line + if ( + this.state.enabled === false && + this.info.lines + + (this.state.wasRowDelimiter === true ? 1 : 0) >= + from_line + ) { + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if ( + skip_empty_lines === true && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0 + ) { + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + this.info.bytes = + this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + if (to !== -1 && this.info.records >= to) { + this.state.stop = true; + close(); + return; + } + } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if (this.state.commenting) { + continue; + } + if ( + comment !== null && + (comment_no_infix === false || + (this.state.record.length === 0 && + this.state.field.length === 0)) + ) { + const commentCount = this.__compareBytes(comment, buf, pos, chr); + if (commentCount !== 0) { + this.state.commenting = true; + continue; + } + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if (delimiterLength !== 0) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + pos += delimiterLength - 1; + continue; + } + } + } + if (this.state.commenting === false) { + if ( + max_record_size !== 0 && + this.state.record_length + this.state.field.length > max_record_size + ) { + return this.__error( + new CsvError( + "CSV_MAX_RECORD_SIZE", + [ + "Max Record Size:", + "record exceed the maximum number of tolerated bytes", + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } + } + const lappend = + ltrim === false || + this.state.quoting === true || + this.state.field.length !== 0 || + !this.__isCharTrimable(buf, pos); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if (lappend === true && rappend === true) { + this.state.field.append(chr); + } else if (rtrim === true && !this.__isCharTrimable(buf, pos)) { + return this.__error( + new CsvError( + "CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + "found non trimable byte after quote", + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } else { + if (lappend === false) { + pos += this.__isCharTrimable(buf, pos) - 1; + } + continue; + } + } + if (end === true) { + // Ensure we are not ending in a quoting state + if (this.state.quoting === true) { + const err = this.__error( + new CsvError( + "CSV_QUOTE_NOT_CLOSED", + [ + "Quote Not Closed:", + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + // Skip last line if it has no characters + if ( + this.state.wasQuoting === true || + this.state.record.length !== 0 || + this.state.field.length !== 0 + ) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + } else if (this.state.wasRowDelimiter === true) { + this.info.empty_lines++; + } else if (this.state.commenting === true) { + this.info.comment_lines++; + } + } + } else { + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function (push) { + const { + columns, + group_columns_by_name, + encoding, + info, + from, + relax_column_count, + relax_column_count_less, + relax_column_count_more, + raw, + skip_records_with_empty_values, + } = this.options; + const { enabled, record } = this.state; + if (enabled === false) { + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if (columns === true) { + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if (columns === false && this.info.records === 0) { + this.state.expectedRecordLength = recordLength; + } + if (recordLength !== this.state.expectedRecordLength) { + const err = + columns === false + ? new CsvError( + "CSV_RECORD_INCONSISTENT_FIELDS_LENGTH", + [ + "Invalid Record Length:", + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ) + : new CsvError( + "CSV_RECORD_INCONSISTENT_COLUMNS", + [ + "Invalid Record Length:", + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ); + if ( + relax_column_count === true || + (relax_column_count_less === true && + recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && + recordLength > this.state.expectedRecordLength) + ) { + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + } else { + const finalErr = this.__error(err); + if (finalErr) return finalErr; + } + } + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + if (this.state.recordHasError === true) { + this.__resetRecord(); + this.state.recordHasError = false; + return; + } + this.info.records++; + if (from === 1 || this.info.records >= from) { + const { objname } = this.options; + // With columns, records are object + if (columns !== false) { + const obj = {}; + // Transform record array to an object + for (let i = 0, l = record.length; i < l; i++) { + if (columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if ( + group_columns_by_name === true && + Object.hasOwn(obj, columns[i].name) + ) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } + } else { + Object.defineProperty(obj, columns[i].name, { + value: record[i], + enumerable: true, + writable: true, + configurable: true, + }); + } + } + // Without objname (default) + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: obj }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj], + push, + ); + if (err) { + return err; + } + } + // Without columns, records are array + } else { + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: record }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? record : [record[objname], record], + push, + ); + if (err) { + return err; + } + } + } + } + this.__resetRecord(); + }, + __firstLineToColumns: function (record) { + const { firstLineToHeaders } = this.state; + try { + const headers = + firstLineToHeaders === undefined + ? record + : firstLineToHeaders.call(null, record); + if (!Array.isArray(headers)) { + return this.__error( + new CsvError( + "CSV_INVALID_COLUMN_MAPPING", + [ + "Invalid Column Mapping:", + "expect an array from column function,", + `got ${JSON.stringify(headers)}`, + ], + this.options, + this.__infoField(), + { + headers: headers, + }, + ), + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; + } catch (err) { + return err; + } + }, + __resetRecord: function () { + if (this.options.raw === true) { + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function () { + const { cast, encoding, rtrim, max_record_size } = this.options; + const { enabled, wasQuoting } = this.state; + // Short circuit for the from_line options + if (enabled === false) { + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if (rtrim === true && wasQuoting === false) { + field = field.trimRight(); + } + if (cast === true) { + const [err, f] = this.__cast(field); + if (err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if (max_record_size !== 0 && typeof field === "string") { + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function () { + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function (record, push) { + const { on_record } = this.options; + if (on_record !== undefined) { + const info = this.__infoRecord(); + try { + record = on_record.call(null, record, info); + } catch (err) { + return err; + } + if (record === undefined || record === null) { + return; + } + } + this.info.bytes_records += this.info.bytes; + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function (field) { + const { columns, relax_column_count } = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if ( + isColumns === true && + relax_column_count && + this.options.columns.length <= this.state.record.length + ) { + return [undefined, undefined]; + } + if (this.state.castField !== null) { + try { + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + } catch (err) { + return [err]; + } + } + if (this.__isFloat(field)) { + return [undefined, Number(field)]; + } else if (this.options.cast_date !== false) { + const info = this.__infoField(); + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + __compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) { + if (sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for (let i = 1; i < sourceLength; i++) { + if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0; + } + return sourceLength; + }, + // Helper to test if a character is trimable + __isCharTrimable: function (buf, pos) { + const { timchars, timcharFirstBytes } = this.state; + // Fast bail-out: non-whitespace bytes (the common case) are rejected + // without scanning the full timchar list. + const first = buf[pos]; + if (first === undefined || timcharFirstBytes[first] === 0) return 0; + loop1: for (let i = 0; i < timchars.length; i++) { + const timchar = timchars[i]; + for (let j = 0; j < timchar.length; j++) { + if (timchar[j] !== buf[pos + j]) continue loop1; + } + return timchar.length; + } + return 0; + }, + __isDelimiter: function (buf, pos, chr) { + const { delimiter, ignore_last_delimiters } = this.options; + if ( + ignore_last_delimiters === true && + this.state.record.length === this.options.columns.length - 1 + ) { + return 0; + } else if ( + ignore_last_delimiters !== false && + typeof ignore_last_delimiters === "number" && + this.state.record.length === ignore_last_delimiters - 1 + ) { + return 0; + } + loop1: for (let i = 0; i < delimiter.length; i++) { + const del = delimiter[i]; + if (del[0] === chr) { + for (let j = 1; j < del.length; j++) { + if (del[j] !== buf[pos + j]) continue loop1; + } + return del.length; + } + } + return 0; + }, + __isEscape: function (buf, pos, chr) { + const { escape } = this.options; + if (escape === null) return false; + const l = escape.length; + if (escape[0] === chr) { + for (let i = 0; i < l; i++) { + if (escape[i] !== buf[pos + i]) { + return false; + } + } + return true; + } + return false; + }, + __isFloat: function (value) { + return value - parseFloat(value) + 1 >= 0; // Borrowed from jquery + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isQuote: function (buf, pos) { + const { quote } = this.options; + if (quote === null) return false; + const l = quote.length; + for (let i = 0; i < l; i++) { + if (quote[i] !== buf[pos + i]) { + return false; + } + } + return true; + }, + __isRecordDelimiter: function (chr, buf, pos) { + const { record_delimiter } = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for (let i = 0; i < recordDelimiterLength; i++) { + const rd = record_delimiter[i]; + const rdLength = rd.length; + if (rd[0] !== chr) { + continue; + } + for (let j = 1; j < rdLength; j++) { + if (rd[j] !== buf[pos + j]) { + continue loop1; + } + } + return rd.length; + } + return 0; + }, + __autoDiscoverRecordDelimiter: function (buf, pos) { + const { encoding } = this.options; + // Note, we don't need to cache this information in state, + // It is only called on the first line until we find out a suitable + // record delimiter. + const rds = [ + // Important, the windows line ending must be before mac os 9 + Buffer.from("\r\n", encoding), + Buffer.from("\n", encoding), + Buffer.from("\r", encoding), + ]; + loop: for (let i = 0; i < rds.length; i++) { + const l = rds[i].length; + for (let j = 0; j < l; j++) { + if (rds[i][j] !== buf[pos + j]) { + continue loop; + } + } + this.options.record_delimiter.push(rds[i]); + this.state.recordDelimiterMaxLength = rds[i].length; + return rds[i].length; + } + return 0; + }, + __error: function (msg) { + const { encoding, raw, skip_records_with_error } = this.options; + const err = typeof msg === "string" ? new Error(msg) : msg; + if (skip_records_with_error) { + this.state.recordHasError = true; + if (this.options.on_skip !== undefined) { + try { + this.options.on_skip( + err, + raw ? this.state.rawBuffer.toString(encoding) : undefined, + ); + } catch (err) { + return err; + } + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + } else { + return err; + } + }, + __infoDataSet: function () { + return { + ...this.info, + columns: this.options.columns, + }; + }, + __infoRecord: function () { + const { columns, raw, encoding } = this.options; + return { + ...this.__infoDataSet(), + bytes_records: this.info.bytes, + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined, + }; + }, + __infoField: function () { + const { columns } = this.options; + const isColumns = Array.isArray(columns); + // Bytes records are only incremented when all records'fields are parsed + const bytes_records = this.info.bytes_records; + return { + ...this.__infoRecord(), + bytes_records: bytes_records, + column: + isColumns === true + ? columns.length > this.state.record.length + ? columns[this.state.record.length].name + : null + : this.state.record.length, + quoting: this.state.wasQuoting, + }; + }, + }; +}; + +// Parse an asynchronous iterable without a stream wrapper. Drain each chunk's +// records before requesting another chunk, including records preceding an error. +async function* parse(input, opts = {}) { + const api = transform(opts); + let records = []; + let stopped = false; + const push = (record) => records.push(record); + const close = () => { + stopped = true; + }; + for await (const chunk of input) { + const buffer = chunk instanceof Buffer ? chunk : Buffer.from(chunk); + const error = api.parse(buffer, false, push, close); + for (const record of records) yield record; + records = []; + if (error !== undefined) throw error; + if (stopped) return; + } + const error = api.parse(undefined, true, push, close); + for (const record of records) yield record; + if (error !== undefined) throw error; +} + +exports.CsvError = CsvError; +exports.normalize_options = normalize_options; +exports.parse = parse; diff --git a/packages/csv-parse/dist/cjs/iterator.d.cts b/packages/csv-parse/dist/cjs/iterator.d.cts new file mode 100644 index 000000000..04dd0bc69 --- /dev/null +++ b/packages/csv-parse/dist/cjs/iterator.d.cts @@ -0,0 +1,20 @@ +import type { + Options, + OptionsWithColumns, + OptionsNormalized, +} from "./options.cjs"; + +declare function parse( + input: AsyncIterable, + options: OptionsWithColumns, +): AsyncGenerator; +declare function parse( + input: AsyncIterable, + options?: Options, +): AsyncGenerator; + +declare function normalize_options(options: Options): OptionsNormalized; + +export { parse, normalize_options }; +export * from "./api/CsvError.cjs"; +export type * from "./options.cjs"; diff --git a/packages/csv-parse/dist/esm/iterator.d.ts b/packages/csv-parse/dist/esm/iterator.d.ts new file mode 100644 index 000000000..8c5de6241 --- /dev/null +++ b/packages/csv-parse/dist/esm/iterator.d.ts @@ -0,0 +1,20 @@ +import type { + Options, + OptionsWithColumns, + OptionsNormalized, +} from "./options.js"; + +declare function parse( + input: AsyncIterable, + options: OptionsWithColumns, +): AsyncGenerator; +declare function parse( + input: AsyncIterable, + options?: Options, +): AsyncGenerator; + +declare function normalize_options(options: Options): OptionsNormalized; + +export { parse, normalize_options }; +export * from "./api/CsvError.js"; +export type * from "./options.js"; diff --git a/packages/csv-parse/dist/esm/iterator.js b/packages/csv-parse/dist/esm/iterator.js new file mode 100644 index 000000000..bbf06562c --- /dev/null +++ b/packages/csv-parse/dist/esm/iterator.js @@ -0,0 +1,3983 @@ +var global$1 = (typeof global !== "undefined" ? global : + typeof self !== "undefined" ? self : + typeof window !== "undefined" ? window : {}); + +var lookup = []; +var revLookup = []; +var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; +var inited = false; +function init () { + inited = true; + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; + } + + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; +} + +function toByteArray (b64) { + if (!inited) { + init(); + } + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; + + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') + } + + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; + + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); + + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; + + var L = 0; + + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + return arr +} + +function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] +} + +function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); + } + return output.join('') +} + +function fromByteArray (uint8) { + if (!inited) { + init(); + } + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 + + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); + } + + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; + } + + parts.push(output); + + return parts.join('') +} + +function read (buffer, offset, isLE, mLen, nBytes) { + var e, m; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var nBits = -7; + var i = isLE ? (nBytes - 1) : 0; + var d = isLE ? -1 : 1; + var s = buffer[offset + i]; + + i += d; + + e = s & ((1 << (-nBits)) - 1); + s >>= (-nBits); + nBits += eLen; + for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + m = e & ((1 << (-nBits)) - 1); + e >>= (-nBits); + nBits += mLen; + for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + if (e === 0) { + e = 1 - eBias; + } else if (e === eMax) { + return m ? NaN : ((s ? -1 : 1) * Infinity) + } else { + m = m + Math.pow(2, mLen); + e = e - eBias; + } + return (s ? -1 : 1) * m * Math.pow(2, e - mLen) +} + +function write (buffer, value, offset, isLE, mLen, nBytes) { + var e, m, c; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); + var i = isLE ? 0 : (nBytes - 1); + var d = isLE ? 1 : -1; + var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; + + value = Math.abs(value); + + if (isNaN(value) || value === Infinity) { + m = isNaN(value) ? 1 : 0; + e = eMax; + } else { + e = Math.floor(Math.log(value) / Math.LN2); + if (value * (c = Math.pow(2, -e)) < 1) { + e--; + c *= 2; + } + if (e + eBias >= 1) { + value += rt / c; + } else { + value += rt * Math.pow(2, 1 - eBias); + } + if (value * c >= 2) { + e++; + c /= 2; + } + + if (e + eBias >= eMax) { + m = 0; + e = eMax; + } else if (e + eBias >= 1) { + m = (value * c - 1) * Math.pow(2, mLen); + e = e + eBias; + } else { + m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); + e = 0; + } + } + + for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} + + e = (e << mLen) | m; + eLen += mLen; + for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} + + buffer[offset + i - d] |= s * 128; +} + +var toString = {}.toString; + +var isArray = Array.isArray || function (arr) { + return toString.call(arr) == '[object Array]'; +}; + +var INSPECT_MAX_BYTES = 50; + +/** + * If `Buffer.TYPED_ARRAY_SUPPORT`: + * === true Use Uint8Array implementation (fastest) + * === false Use Object implementation (most compatible, even IE6) + * + * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, + * Opera 11.6+, iOS 4.2+. + * + * Due to various browser bugs, sometimes the Object implementation will be used even + * when the browser supports typed arrays. + * + * Note: + * + * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, + * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. + * + * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. + * + * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of + * incorrect length in some situations. + + * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they + * get the Object implementation, which is slower but behaves correctly. + */ +Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined + ? global$1.TYPED_ARRAY_SUPPORT + : true; + +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + +function kMaxLength () { + return Buffer.TYPED_ARRAY_SUPPORT + ? 0x7fffffff + : 0x3fffffff +} + +function createBuffer (that, length) { + if (kMaxLength() < length) { + throw new RangeError('Invalid typed array length') + } + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = new Uint8Array(length); + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + if (that === null) { + that = new Buffer(length); + } + that.length = length; + } + + return that +} + +/** + * The Buffer constructor returns instances of `Uint8Array` that have their + * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of + * `Uint8Array`, so the returned instances will have all the node `Buffer` methods + * and the `Uint8Array` methods. Square bracket notation works as expected -- it + * returns a single octet. + * + * The `Uint8Array` prototype remains unmodified. + */ + +function Buffer (arg, encodingOrOffset, length) { + if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { + return new Buffer(arg, encodingOrOffset, length) + } + + // Common case. + if (typeof arg === 'number') { + if (typeof encodingOrOffset === 'string') { + throw new Error( + 'If encoding is specified then the first argument must be a string' + ) + } + return allocUnsafe(this, arg) + } + return from(this, arg, encodingOrOffset, length) +} + +Buffer.poolSize = 8192; // not used by this implementation + +// TODO: Legacy, not needed anymore. Remove in next major version. +Buffer._augment = function (arr) { + arr.__proto__ = Buffer.prototype; + return arr +}; + +function from (that, value, encodingOrOffset, length) { + if (typeof value === 'number') { + throw new TypeError('"value" argument must not be a number') + } + + if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { + return fromArrayBuffer(that, value, encodingOrOffset, length) + } + + if (typeof value === 'string') { + return fromString(that, value, encodingOrOffset) + } + + return fromObject(that, value) +} + +/** + * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError + * if value is a number. + * Buffer.from(str[, encoding]) + * Buffer.from(array) + * Buffer.from(buffer) + * Buffer.from(arrayBuffer[, byteOffset[, length]]) + **/ +Buffer.from = function (value, encodingOrOffset, length) { + return from(null, value, encodingOrOffset, length) +}; + +if (Buffer.TYPED_ARRAY_SUPPORT) { + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; +} + +function assertSize (size) { + if (typeof size !== 'number') { + throw new TypeError('"size" argument must be a number') + } else if (size < 0) { + throw new RangeError('"size" argument must not be negative') + } +} + +function alloc (that, size, fill, encoding) { + assertSize(size); + if (size <= 0) { + return createBuffer(that, size) + } + if (fill !== undefined) { + // Only pay attention to encoding if it's a string. This + // prevents accidentally sending in a number that would + // be interpretted as a start offset. + return typeof encoding === 'string' + ? createBuffer(that, size).fill(fill, encoding) + : createBuffer(that, size).fill(fill) + } + return createBuffer(that, size) +} + +/** + * Creates a new filled Buffer instance. + * alloc(size[, fill[, encoding]]) + **/ +Buffer.alloc = function (size, fill, encoding) { + return alloc(null, size, fill, encoding) +}; + +function allocUnsafe (that, size) { + assertSize(size); + that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) { + for (var i = 0; i < size; ++i) { + that[i] = 0; + } + } + return that +} + +/** + * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. + * */ +Buffer.allocUnsafe = function (size) { + return allocUnsafe(null, size) +}; +/** + * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. + */ +Buffer.allocUnsafeSlow = function (size) { + return allocUnsafe(null, size) +}; + +function fromString (that, string, encoding) { + if (typeof encoding !== 'string' || encoding === '') { + encoding = 'utf8'; + } + + if (!Buffer.isEncoding(encoding)) { + throw new TypeError('"encoding" must be a valid string encoding') + } + + var length = byteLength(string, encoding) | 0; + that = createBuffer(that, length); + + var actual = that.write(string, encoding); + + if (actual !== length) { + // Writing a hex string, for example, that contains invalid characters will + // cause everything after the first invalid character to be ignored. (e.g. + // 'abxxcd' will be treated as 'ab') + that = that.slice(0, actual); + } + + return that +} + +function fromArrayLike (that, array) { + var length = array.length < 0 ? 0 : checked(array.length) | 0; + that = createBuffer(that, length); + for (var i = 0; i < length; i += 1) { + that[i] = array[i] & 255; + } + return that +} + +function fromArrayBuffer (that, array, byteOffset, length) { + array.byteLength; // this throws if `array` is not a valid ArrayBuffer + + if (byteOffset < 0 || array.byteLength < byteOffset) { + throw new RangeError('\'offset\' is out of bounds') + } + + if (array.byteLength < byteOffset + (length || 0)) { + throw new RangeError('\'length\' is out of bounds') + } + + if (byteOffset === undefined && length === undefined) { + array = new Uint8Array(array); + } else if (length === undefined) { + array = new Uint8Array(array, byteOffset); + } else { + array = new Uint8Array(array, byteOffset, length); + } + + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = array; + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + that = fromArrayLike(that, array); + } + return that +} + +function fromObject (that, obj) { + if (internalIsBuffer(obj)) { + var len = checked(obj.length) | 0; + that = createBuffer(that, len); + + if (that.length === 0) { + return that + } + + obj.copy(that, 0, 0, len); + return that + } + + if (obj) { + if ((typeof ArrayBuffer !== 'undefined' && + obj.buffer instanceof ArrayBuffer) || 'length' in obj) { + if (typeof obj.length !== 'number' || isnan(obj.length)) { + return createBuffer(that, 0) + } + return fromArrayLike(that, obj) + } + + if (obj.type === 'Buffer' && isArray(obj.data)) { + return fromArrayLike(that, obj.data) + } + } + + throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') +} + +function checked (length) { + // Note: cannot use `length < kMaxLength()` here because that fails when + // length is NaN (which is otherwise coerced to zero.) + if (length >= kMaxLength()) { + throw new RangeError('Attempt to allocate Buffer larger than maximum ' + + 'size: 0x' + kMaxLength().toString(16) + ' bytes') + } + return length | 0 +} +Buffer.isBuffer = isBuffer; +function internalIsBuffer (b) { + return !!(b != null && b._isBuffer) +} + +Buffer.compare = function compare (a, b) { + if (!internalIsBuffer(a) || !internalIsBuffer(b)) { + throw new TypeError('Arguments must be Buffers') + } + + if (a === b) return 0 + + var x = a.length; + var y = b.length; + + for (var i = 0, len = Math.min(x, y); i < len; ++i) { + if (a[i] !== b[i]) { + x = a[i]; + y = b[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 +}; + +Buffer.isEncoding = function isEncoding (encoding) { + switch (String(encoding).toLowerCase()) { + case 'hex': + case 'utf8': + case 'utf-8': + case 'ascii': + case 'latin1': + case 'binary': + case 'base64': + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return true + default: + return false + } +}; + +Buffer.concat = function concat (list, length) { + if (!isArray(list)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + + if (list.length === 0) { + return Buffer.alloc(0) + } + + var i; + if (length === undefined) { + length = 0; + for (i = 0; i < list.length; ++i) { + length += list[i].length; + } + } + + var buffer = Buffer.allocUnsafe(length); + var pos = 0; + for (i = 0; i < list.length; ++i) { + var buf = list[i]; + if (!internalIsBuffer(buf)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + buf.copy(buffer, pos); + pos += buf.length; + } + return buffer +}; + +function byteLength (string, encoding) { + if (internalIsBuffer(string)) { + return string.length + } + if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && + (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { + return string.byteLength + } + if (typeof string !== 'string') { + string = '' + string; + } + + var len = string.length; + if (len === 0) return 0 + + // Use a for loop to avoid recursion + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'ascii': + case 'latin1': + case 'binary': + return len + case 'utf8': + case 'utf-8': + case undefined: + return utf8ToBytes(string).length + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return len * 2 + case 'hex': + return len >>> 1 + case 'base64': + return base64ToBytes(string).length + default: + if (loweredCase) return utf8ToBytes(string).length // assume utf8 + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } +} +Buffer.byteLength = byteLength; + +function slowToString (encoding, start, end) { + var loweredCase = false; + + // No need to verify that "this.length <= MAX_UINT32" since it's a read-only + // property of a typed array. + + // This behaves neither like String nor Uint8Array in that we set start/end + // to their upper/lower bounds if the value passed is out of range. + // undefined is handled specially as per ECMA-262 6th Edition, + // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. + if (start === undefined || start < 0) { + start = 0; + } + // Return early if start > this.length. Done here to prevent potential uint32 + // coercion fail below. + if (start > this.length) { + return '' + } + + if (end === undefined || end > this.length) { + end = this.length; + } + + if (end <= 0) { + return '' + } + + // Force coersion to uint32. This will also coerce falsey/NaN values to 0. + end >>>= 0; + start >>>= 0; + + if (end <= start) { + return '' + } + + if (!encoding) encoding = 'utf8'; + + while (true) { + switch (encoding) { + case 'hex': + return hexSlice(this, start, end) + + case 'utf8': + case 'utf-8': + return utf8Slice(this, start, end) + + case 'ascii': + return asciiSlice(this, start, end) + + case 'latin1': + case 'binary': + return latin1Slice(this, start, end) + + case 'base64': + return base64Slice(this, start, end) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return utf16leSlice(this, start, end) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = (encoding + '').toLowerCase(); + loweredCase = true; + } + } +} + +// The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect +// Buffer instances. +Buffer.prototype._isBuffer = true; + +function swap (b, n, m) { + var i = b[n]; + b[n] = b[m]; + b[m] = i; +} + +Buffer.prototype.swap16 = function swap16 () { + var len = this.length; + if (len % 2 !== 0) { + throw new RangeError('Buffer size must be a multiple of 16-bits') + } + for (var i = 0; i < len; i += 2) { + swap(this, i, i + 1); + } + return this +}; + +Buffer.prototype.swap32 = function swap32 () { + var len = this.length; + if (len % 4 !== 0) { + throw new RangeError('Buffer size must be a multiple of 32-bits') + } + for (var i = 0; i < len; i += 4) { + swap(this, i, i + 3); + swap(this, i + 1, i + 2); + } + return this +}; + +Buffer.prototype.swap64 = function swap64 () { + var len = this.length; + if (len % 8 !== 0) { + throw new RangeError('Buffer size must be a multiple of 64-bits') + } + for (var i = 0; i < len; i += 8) { + swap(this, i, i + 7); + swap(this, i + 1, i + 6); + swap(this, i + 2, i + 5); + swap(this, i + 3, i + 4); + } + return this +}; + +Buffer.prototype.toString = function toString () { + var length = this.length | 0; + if (length === 0) return '' + if (arguments.length === 0) return utf8Slice(this, 0, length) + return slowToString.apply(this, arguments) +}; + +Buffer.prototype.equals = function equals (b) { + if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') + if (this === b) return true + return Buffer.compare(this, b) === 0 +}; + +Buffer.prototype.inspect = function inspect () { + var str = ''; + var max = INSPECT_MAX_BYTES; + if (this.length > 0) { + str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); + if (this.length > max) str += ' ... '; + } + return '' +}; + +Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { + if (!internalIsBuffer(target)) { + throw new TypeError('Argument must be a Buffer') + } + + if (start === undefined) { + start = 0; + } + if (end === undefined) { + end = target ? target.length : 0; + } + if (thisStart === undefined) { + thisStart = 0; + } + if (thisEnd === undefined) { + thisEnd = this.length; + } + + if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { + throw new RangeError('out of range index') + } + + if (thisStart >= thisEnd && start >= end) { + return 0 + } + if (thisStart >= thisEnd) { + return -1 + } + if (start >= end) { + return 1 + } + + start >>>= 0; + end >>>= 0; + thisStart >>>= 0; + thisEnd >>>= 0; + + if (this === target) return 0 + + var x = thisEnd - thisStart; + var y = end - start; + var len = Math.min(x, y); + + var thisCopy = this.slice(thisStart, thisEnd); + var targetCopy = target.slice(start, end); + + for (var i = 0; i < len; ++i) { + if (thisCopy[i] !== targetCopy[i]) { + x = thisCopy[i]; + y = targetCopy[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 +}; + +// Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, +// OR the last index of `val` in `buffer` at offset <= `byteOffset`. +// +// Arguments: +// - buffer - a Buffer to search +// - val - a string, Buffer, or number +// - byteOffset - an index into `buffer`; will be clamped to an int32 +// - encoding - an optional encoding, relevant is val is a string +// - dir - true for indexOf, false for lastIndexOf +function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { + // Empty buffer means no match + if (buffer.length === 0) return -1 + + // Normalize byteOffset + if (typeof byteOffset === 'string') { + encoding = byteOffset; + byteOffset = 0; + } else if (byteOffset > 0x7fffffff) { + byteOffset = 0x7fffffff; + } else if (byteOffset < -2147483648) { + byteOffset = -2147483648; + } + byteOffset = +byteOffset; // Coerce to Number. + if (isNaN(byteOffset)) { + // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer + byteOffset = dir ? 0 : (buffer.length - 1); + } + + // Normalize byteOffset: negative offsets start from the end of the buffer + if (byteOffset < 0) byteOffset = buffer.length + byteOffset; + if (byteOffset >= buffer.length) { + if (dir) return -1 + else byteOffset = buffer.length - 1; + } else if (byteOffset < 0) { + if (dir) byteOffset = 0; + else return -1 + } + + // Normalize val + if (typeof val === 'string') { + val = Buffer.from(val, encoding); + } + + // Finally, search either indexOf (if dir is true) or lastIndexOf + if (internalIsBuffer(val)) { + // Special case: looking for empty string/buffer always fails + if (val.length === 0) { + return -1 + } + return arrayIndexOf(buffer, val, byteOffset, encoding, dir) + } else if (typeof val === 'number') { + val = val & 0xFF; // Search for a byte value [0-255] + if (Buffer.TYPED_ARRAY_SUPPORT && + typeof Uint8Array.prototype.indexOf === 'function') { + if (dir) { + return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) + } else { + return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) + } + } + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + } + + throw new TypeError('val must be string, number or Buffer') +} + +function arrayIndexOf (arr, val, byteOffset, encoding, dir) { + var indexSize = 1; + var arrLength = arr.length; + var valLength = val.length; + + if (encoding !== undefined) { + encoding = String(encoding).toLowerCase(); + if (encoding === 'ucs2' || encoding === 'ucs-2' || + encoding === 'utf16le' || encoding === 'utf-16le') { + if (arr.length < 2 || val.length < 2) { + return -1 + } + indexSize = 2; + arrLength /= 2; + valLength /= 2; + byteOffset /= 2; + } + } + + function read (buf, i) { + if (indexSize === 1) { + return buf[i] + } else { + return buf.readUInt16BE(i * indexSize) + } + } + + var i; + if (dir) { + var foundIndex = -1; + for (i = byteOffset; i < arrLength; i++) { + if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { + if (foundIndex === -1) foundIndex = i; + if (i - foundIndex + 1 === valLength) return foundIndex * indexSize + } else { + if (foundIndex !== -1) i -= i - foundIndex; + foundIndex = -1; + } + } + } else { + if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; + for (i = byteOffset; i >= 0; i--) { + var found = true; + for (var j = 0; j < valLength; j++) { + if (read(arr, i + j) !== read(val, j)) { + found = false; + break + } + } + if (found) return i + } + } + + return -1 +} + +Buffer.prototype.includes = function includes (val, byteOffset, encoding) { + return this.indexOf(val, byteOffset, encoding) !== -1 +}; + +Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, true) +}; + +Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, false) +}; + +function hexWrite (buf, string, offset, length) { + offset = Number(offset) || 0; + var remaining = buf.length - offset; + if (!length) { + length = remaining; + } else { + length = Number(length); + if (length > remaining) { + length = remaining; + } + } + + // must be an even number of digits + var strLen = string.length; + if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') + + if (length > strLen / 2) { + length = strLen / 2; + } + for (var i = 0; i < length; ++i) { + var parsed = parseInt(string.substr(i * 2, 2), 16); + if (isNaN(parsed)) return i + buf[offset + i] = parsed; + } + return i +} + +function utf8Write (buf, string, offset, length) { + return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) +} + +function asciiWrite (buf, string, offset, length) { + return blitBuffer(asciiToBytes(string), buf, offset, length) +} + +function latin1Write (buf, string, offset, length) { + return asciiWrite(buf, string, offset, length) +} + +function base64Write (buf, string, offset, length) { + return blitBuffer(base64ToBytes(string), buf, offset, length) +} + +function ucs2Write (buf, string, offset, length) { + return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) +} + +Buffer.prototype.write = function write (string, offset, length, encoding) { + // Buffer#write(string) + if (offset === undefined) { + encoding = 'utf8'; + length = this.length; + offset = 0; + // Buffer#write(string, encoding) + } else if (length === undefined && typeof offset === 'string') { + encoding = offset; + length = this.length; + offset = 0; + // Buffer#write(string, offset[, length][, encoding]) + } else if (isFinite(offset)) { + offset = offset | 0; + if (isFinite(length)) { + length = length | 0; + if (encoding === undefined) encoding = 'utf8'; + } else { + encoding = length; + length = undefined; + } + // legacy write(string, encoding, offset, length) - remove in v0.13 + } else { + throw new Error( + 'Buffer.write(string, encoding, offset[, length]) is no longer supported' + ) + } + + var remaining = this.length - offset; + if (length === undefined || length > remaining) length = remaining; + + if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { + throw new RangeError('Attempt to write outside buffer bounds') + } + + if (!encoding) encoding = 'utf8'; + + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'hex': + return hexWrite(this, string, offset, length) + + case 'utf8': + case 'utf-8': + return utf8Write(this, string, offset, length) + + case 'ascii': + return asciiWrite(this, string, offset, length) + + case 'latin1': + case 'binary': + return latin1Write(this, string, offset, length) + + case 'base64': + // Warning: maxLength not taken into account in base64Write + return base64Write(this, string, offset, length) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return ucs2Write(this, string, offset, length) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } +}; + +Buffer.prototype.toJSON = function toJSON () { + return { + type: 'Buffer', + data: Array.prototype.slice.call(this._arr || this, 0) + } +}; + +function base64Slice (buf, start, end) { + if (start === 0 && end === buf.length) { + return fromByteArray(buf) + } else { + return fromByteArray(buf.slice(start, end)) + } +} + +function utf8Slice (buf, start, end) { + end = Math.min(buf.length, end); + var res = []; + + var i = start; + while (i < end) { + var firstByte = buf[i]; + var codePoint = null; + var bytesPerSequence = (firstByte > 0xEF) ? 4 + : (firstByte > 0xDF) ? 3 + : (firstByte > 0xBF) ? 2 + : 1; + + if (i + bytesPerSequence <= end) { + var secondByte, thirdByte, fourthByte, tempCodePoint; + + switch (bytesPerSequence) { + case 1: + if (firstByte < 0x80) { + codePoint = firstByte; + } + break + case 2: + secondByte = buf[i + 1]; + if ((secondByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); + if (tempCodePoint > 0x7F) { + codePoint = tempCodePoint; + } + } + break + case 3: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); + if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { + codePoint = tempCodePoint; + } + } + break + case 4: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + fourthByte = buf[i + 3]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); + if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { + codePoint = tempCodePoint; + } + } + } + } + + if (codePoint === null) { + // we did not generate a valid codePoint so insert a + // replacement char (U+FFFD) and advance only 1 byte + codePoint = 0xFFFD; + bytesPerSequence = 1; + } else if (codePoint > 0xFFFF) { + // encode to utf16 (surrogate pair dance) + codePoint -= 0x10000; + res.push(codePoint >>> 10 & 0x3FF | 0xD800); + codePoint = 0xDC00 | codePoint & 0x3FF; + } + + res.push(codePoint); + i += bytesPerSequence; + } + + return decodeCodePointsArray(res) +} + +// Based on http://stackoverflow.com/a/22747272/680742, the browser with +// the lowest limit is Chrome, with 0x10000 args. +// We go 1 magnitude less, for safety +var MAX_ARGUMENTS_LENGTH = 0x1000; + +function decodeCodePointsArray (codePoints) { + var len = codePoints.length; + if (len <= MAX_ARGUMENTS_LENGTH) { + return String.fromCharCode.apply(String, codePoints) // avoid extra slice() + } + + // Decode in chunks to avoid "call stack size exceeded". + var res = ''; + var i = 0; + while (i < len) { + res += String.fromCharCode.apply( + String, + codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) + ); + } + return res +} + +function asciiSlice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i] & 0x7F); + } + return ret +} + +function latin1Slice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i]); + } + return ret +} + +function hexSlice (buf, start, end) { + var len = buf.length; + + if (!start || start < 0) start = 0; + if (!end || end < 0 || end > len) end = len; + + var out = ''; + for (var i = start; i < end; ++i) { + out += toHex(buf[i]); + } + return out +} + +function utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end); + var res = ''; + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + } + return res +} + +Buffer.prototype.slice = function slice (start, end) { + var len = this.length; + start = ~~start; + end = end === undefined ? len : ~~end; + + if (start < 0) { + start += len; + if (start < 0) start = 0; + } else if (start > len) { + start = len; + } + + if (end < 0) { + end += len; + if (end < 0) end = 0; + } else if (end > len) { + end = len; + } + + if (end < start) end = start; + + var newBuf; + if (Buffer.TYPED_ARRAY_SUPPORT) { + newBuf = this.subarray(start, end); + newBuf.__proto__ = Buffer.prototype; + } else { + var sliceLen = end - start; + newBuf = new Buffer(sliceLen, undefined); + for (var i = 0; i < sliceLen; ++i) { + newBuf[i] = this[i + start]; + } + } + + return newBuf +}; + +/* + * Need to make sure that buffer isn't trying to write out of bounds. + */ +function checkOffset (offset, ext, length) { + if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') + if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') +} + +Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + + return val +}; + +Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + checkOffset(offset, byteLength, this.length); + } + + var val = this[offset + --byteLength]; + var mul = 1; + while (byteLength > 0 && (mul *= 0x100)) { + val += this[offset + --byteLength] * mul; + } + + return val +}; + +Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + return this[offset] +}; + +Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return this[offset] | (this[offset + 1] << 8) +}; + +Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return (this[offset] << 8) | this[offset + 1] +}; + +Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return ((this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16)) + + (this[offset + 3] * 0x1000000) +}; + +Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] * 0x1000000) + + ((this[offset + 1] << 16) | + (this[offset + 2] << 8) | + this[offset + 3]) +}; + +Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val +}; + +Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var i = byteLength; + var mul = 1; + var val = this[offset + --i]; + while (i > 0 && (mul *= 0x100)) { + val += this[offset + --i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val +}; + +Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + if (!(this[offset] & 0x80)) return (this[offset]) + return ((0xff - this[offset] + 1) * -1) +}; + +Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset] | (this[offset + 1] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val +}; + +Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset + 1] | (this[offset] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val +}; + +Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16) | + (this[offset + 3] << 24) +}; + +Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] << 24) | + (this[offset + 1] << 16) | + (this[offset + 2] << 8) | + (this[offset + 3]) +}; + +Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, true, 23, 4) +}; + +Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, false, 23, 4) +}; + +Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, true, 52, 8) +}; + +Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, false, 52, 8) +}; + +function checkInt (buf, value, offset, ext, max, min) { + if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') + if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') + if (offset + ext > buf.length) throw new RangeError('Index out of range') +} + +Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var mul = 1; + var i = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength +}; + +Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var i = byteLength - 1; + var mul = 1; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength +}; + +Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + this[offset] = (value & 0xff); + return offset + 1 +}; + +function objectWriteUInt16 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { + buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> + (littleEndian ? i : 1 - i) * 8; + } +} + +Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 +}; + +Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 +}; + +function objectWriteUInt32 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffffffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { + buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; + } +} + +Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset + 3] = (value >>> 24); + this[offset + 2] = (value >>> 16); + this[offset + 1] = (value >>> 8); + this[offset] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 +}; + +Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 +}; + +Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = 0; + var mul = 1; + var sub = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength +}; + +Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = byteLength - 1; + var mul = 1; + var sub = 0; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength +}; + +Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -128); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + if (value < 0) value = 0xff + value + 1; + this[offset] = (value & 0xff); + return offset + 1 +}; + +Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -32768); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 +}; + +Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -32768); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 +}; + +Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -2147483648); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + this[offset + 2] = (value >>> 16); + this[offset + 3] = (value >>> 24); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 +}; + +Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -2147483648); + if (value < 0) value = 0xffffffff + value + 1; + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 +}; + +function checkIEEE754 (buf, value, offset, ext, max, min) { + if (offset + ext > buf.length) throw new RangeError('Index out of range') + if (offset < 0) throw new RangeError('Index out of range') +} + +function writeFloat (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 4); + } + write(buf, value, offset, littleEndian, 23, 4); + return offset + 4 +} + +Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { + return writeFloat(this, value, offset, true, noAssert) +}; + +Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { + return writeFloat(this, value, offset, false, noAssert) +}; + +function writeDouble (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 8); + } + write(buf, value, offset, littleEndian, 52, 8); + return offset + 8 +} + +Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { + return writeDouble(this, value, offset, true, noAssert) +}; + +Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { + return writeDouble(this, value, offset, false, noAssert) +}; + +// copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) +Buffer.prototype.copy = function copy (target, targetStart, start, end) { + if (!start) start = 0; + if (!end && end !== 0) end = this.length; + if (targetStart >= target.length) targetStart = target.length; + if (!targetStart) targetStart = 0; + if (end > 0 && end < start) end = start; + + // Copy 0 bytes; we're done + if (end === start) return 0 + if (target.length === 0 || this.length === 0) return 0 + + // Fatal error conditions + if (targetStart < 0) { + throw new RangeError('targetStart out of bounds') + } + if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') + if (end < 0) throw new RangeError('sourceEnd out of bounds') + + // Are we oob? + if (end > this.length) end = this.length; + if (target.length - targetStart < end - start) { + end = target.length - targetStart + start; + } + + var len = end - start; + var i; + + if (this === target && start < targetStart && targetStart < end) { + // descending copy from end + for (i = len - 1; i >= 0; --i) { + target[i + targetStart] = this[i + start]; + } + } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { + // ascending copy from start + for (i = 0; i < len; ++i) { + target[i + targetStart] = this[i + start]; + } + } else { + Uint8Array.prototype.set.call( + target, + this.subarray(start, start + len), + targetStart + ); + } + + return len +}; + +// Usage: +// buffer.fill(number[, offset[, end]]) +// buffer.fill(buffer[, offset[, end]]) +// buffer.fill(string[, offset[, end]][, encoding]) +Buffer.prototype.fill = function fill (val, start, end, encoding) { + // Handle string cases: + if (typeof val === 'string') { + if (typeof start === 'string') { + encoding = start; + start = 0; + end = this.length; + } else if (typeof end === 'string') { + encoding = end; + end = this.length; + } + if (val.length === 1) { + var code = val.charCodeAt(0); + if (code < 256) { + val = code; + } + } + if (encoding !== undefined && typeof encoding !== 'string') { + throw new TypeError('encoding must be a string') + } + if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { + throw new TypeError('Unknown encoding: ' + encoding) + } + } else if (typeof val === 'number') { + val = val & 255; + } + + // Invalid ranges are not set to a default, so can range check early. + if (start < 0 || this.length < start || this.length < end) { + throw new RangeError('Out of range index') + } + + if (end <= start) { + return this + } + + start = start >>> 0; + end = end === undefined ? this.length : end >>> 0; + + if (!val) val = 0; + + var i; + if (typeof val === 'number') { + for (i = start; i < end; ++i) { + this[i] = val; + } + } else { + var bytes = internalIsBuffer(val) + ? val + : utf8ToBytes(new Buffer(val, encoding).toString()); + var len = bytes.length; + for (i = 0; i < end - start; ++i) { + this[i + start] = bytes[i % len]; + } + } + + return this +}; + +// HELPER FUNCTIONS +// ================ + +var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; + +function base64clean (str) { + // Node strips out invalid characters like \n and \t from the string, base64-js does not + str = stringtrim(str).replace(INVALID_BASE64_RE, ''); + // Node converts strings with length < 2 to '' + if (str.length < 2) return '' + // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not + while (str.length % 4 !== 0) { + str = str + '='; + } + return str +} + +function stringtrim (str) { + if (str.trim) return str.trim() + return str.replace(/^\s+|\s+$/g, '') +} + +function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) +} + +function utf8ToBytes (string, units) { + units = units || Infinity; + var codePoint; + var length = string.length; + var leadSurrogate = null; + var bytes = []; + + for (var i = 0; i < length; ++i) { + codePoint = string.charCodeAt(i); + + // is surrogate component + if (codePoint > 0xD7FF && codePoint < 0xE000) { + // last char was a lead + if (!leadSurrogate) { + // no lead yet + if (codePoint > 0xDBFF) { + // unexpected trail + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } else if (i + 1 === length) { + // unpaired lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } + + // valid lead + leadSurrogate = codePoint; + + continue + } + + // 2 leads in a row + if (codePoint < 0xDC00) { + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + leadSurrogate = codePoint; + continue + } + + // valid surrogate pair + codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; + } else if (leadSurrogate) { + // valid bmp char, but last char was a lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + } + + leadSurrogate = null; + + // encode utf8 + if (codePoint < 0x80) { + if ((units -= 1) < 0) break + bytes.push(codePoint); + } else if (codePoint < 0x800) { + if ((units -= 2) < 0) break + bytes.push( + codePoint >> 0x6 | 0xC0, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x10000) { + if ((units -= 3) < 0) break + bytes.push( + codePoint >> 0xC | 0xE0, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x110000) { + if ((units -= 4) < 0) break + bytes.push( + codePoint >> 0x12 | 0xF0, + codePoint >> 0xC & 0x3F | 0x80, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else { + throw new Error('Invalid code point') + } + } + + return bytes +} + +function asciiToBytes (str) { + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + // Node's code seems to be doing this and not & 0x7F.. + byteArray.push(str.charCodeAt(i) & 0xFF); + } + return byteArray +} + +function utf16leToBytes (str, units) { + var c, hi, lo; + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + if ((units -= 2) < 0) break + + c = str.charCodeAt(i); + hi = c >> 8; + lo = c % 256; + byteArray.push(lo); + byteArray.push(hi); + } + + return byteArray +} + + +function base64ToBytes (str) { + return toByteArray(base64clean(str)) +} + +function blitBuffer (src, dst, offset, length) { + for (var i = 0; i < length; ++i) { + if ((i + offset >= dst.length) || (i >= src.length)) break + dst[i + offset] = src[i]; + } + return i +} + +function isnan (val) { + return val !== val // eslint-disable-line no-self-compare +} + + +// the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence +// The _isBuffer check is for Safari 5-7 support, because it's missing +// Object.prototype.constructor. Remove this eventually +function isBuffer(obj) { + return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) +} + +function isFastBuffer (obj) { + return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) +} + +// For Node v0.10 support. Remove this eventually. +function isSlowBuffer (obj) { + return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) +} + +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if (Array.isArray(message)) message = message.join(" ").trim(); + super(message); + if (Error.captureStackTrace !== undefined) { + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for (const context of contexts) { + for (const key in context) { + const value = context[key]; + this[key] = isBuffer(value) + ? value.toString(options.encoding) + : value == null + ? value + : JSON.parse(JSON.stringify(value)); + } + } + } +} + +const is_object = function (obj) { + return typeof obj === "object" && obj !== null && !Array.isArray(obj); +}; + +const normalize_columns_array = function (columns) { + const normalizedColumns = []; + for (let i = 0, l = columns.length; i < l; i++) { + const column = columns[i]; + if (column === undefined || column === null || column === false) { + normalizedColumns[i] = { disabled: true }; + } else if (typeof column === "string" || typeof column === "number") { + normalizedColumns[i] = { name: `${column}` }; + } else if (is_object(column)) { + if (typeof column.name !== "string") { + throw new CsvError("CSV_OPTION_COLUMNS_MISSING_NAME", [ + "Option columns missing name:", + `property "name" is required at position ${i}`, + "when column is an object literal", + ]); + } + normalizedColumns[i] = column; + } else { + throw new CsvError("CSV_INVALID_COLUMN_DEFINITION", [ + "Invalid column definition:", + "expect a string or a literal object,", + `got ${JSON.stringify(column)} at position ${i}`, + ]); + } + } + return normalizedColumns; +}; + +class ResizeableBuffer { + constructor(size = 100) { + this.size = size; + this.length = 0; + this.buf = Buffer.allocUnsafe(size); + } + prepend(val) { + if (isBuffer(val)) { + const length = this.length + val.length; + if (length >= this.size) { + this.resize(); + if (length >= this.size) { + throw Error("INVALID_BUFFER_STATE"); + } + } + const buf = this.buf; + this.buf = Buffer.allocUnsafe(this.size); + val.copy(this.buf, 0); + buf.copy(this.buf, val.length); + this.length += val.length; + } else { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + const buf = this.clone(); + this.buf[0] = val; + buf.copy(this.buf, 1, 0, length); + } + } + append(val) { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + this.buf[length] = val; + } + clone() { + return Buffer.from(this.buf.slice(0, this.length)); + } + resize() { + const length = this.length; + this.size = this.size * 2; + const buf = Buffer.allocUnsafe(this.size); + this.buf.copy(buf, 0, 0, length); + this.buf = buf; + } + toString(encoding) { + if (encoding) { + return this.buf.toString(encoding, 0, this.length); + } else { + return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)); + } + } + toJSON() { + return this.toString("utf8"); + } + reset() { + this.length = 0; + } +} + +const init_state = function (options) { + // ECMAScript WhiteSpace + LineTerminator codepoints, encoded under + // `options.encoding`. Aligns trimming with `String.prototype.trim()`. + // https://tc39.es/ecma262/#sec-white-space + // https://tc39.es/ecma262/#sec-line-terminators + // + // Codepoints unrepresentable in the target encoding are dropped: Node's + // Buffer substitutes them with `?` (0x3F), and including those would cause + // literal `?` bytes in the input to be trimmed under `latin1`/`ascii`. + const timchars = [ + // Basic Latin + 0x0020, // [Space](https://www.fileformat.info/info/unicode/char/0020/index.htm) + 0x0009, // [CHARACTER TABULATION (HT)](https://www.fileformat.info/info/unicode/char/0009/index.htm) + 0x000a, // [LINE FEED (LF)](https://www.fileformat.info/info/unicode/char/000a/index.htm) + 0x000d, // [CARRIAGE RETURN (CR)](https://www.fileformat.info/info/unicode/char/000d/index.htm) + 0x000c, // [FORM FEED (FF)](https://www.fileformat.info/info/unicode/char/000c/index.htm) + 0x000b, // [LINE TABULATION (VT)](https://www.fileformat.info/info/unicode/char/000b/index.htm) + // Latin-1 Supplement + 0x00a0, // [NO-BREAK SPACE (NBSP)](https://www.fileformat.info/info/unicode/char/00a0/index.htm) + // Ogham + 0x1680, // [OGHAM SPACE MARK](https://www.fileformat.info/info/unicode/char/1680/index.htm) + // General Punctuation + 0x2000, // [EN QUAD](https://www.fileformat.info/info/unicode/char/2000/index.htm) + 0x2001, // [EM QUAD](https://www.fileformat.info/info/unicode/char/2001/index.htm) + 0x2002, // [EN SPACE](https://www.fileformat.info/info/unicode/char/2002/index.htm) + 0x2003, // [EM SPACE](https://www.fileformat.info/info/unicode/char/2003/index.htm) + 0x2004, // [THREE-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2004/index.htm) + 0x2005, // [FOUR-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2005/index.htm) + 0x2006, // [SIX-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2006/index.htm) + 0x2007, // [FIGURE SPACE](https://www.fileformat.info/info/unicode/char/2007/index.htm) + 0x2008, // [PUNCTUATION SPACE](https://www.fileformat.info/info/unicode/char/2008/index.htm) + 0x2009, // [THIN SPACE](https://www.fileformat.info/info/unicode/char/2009/index.htm) + 0x200a, // [HAIR SPACE](https://www.fileformat.info/info/unicode/char/200a/index.htm) + 0x2028, // [LINE SEPARATOR](https://www.fileformat.info/info/unicode/char/2028/index.htm) + 0x2029, // [PARAGRAPH SEPARATOR](https://www.fileformat.info/info/unicode/char/2029/index.htm) + 0x202f, // [NARROW NO-BREAK SPACE (NNBSP)](https://www.fileformat.info/info/unicode/char/202f/index.htm) + 0x205f, // [MEDIUM MATHEMATICAL SPACE (MMSP)](https://www.fileformat.info/info/unicode/char/205f/index.htm) + 0x3000, // [IDEOGRAPHIC SPACE](https://www.fileformat.info/info/unicode/char/3000/index.htm) + 0xfeff, // [ZERO WIDTH NO-BREAK SPACE (BOM)](https://www.fileformat.info/info/unicode/char/feff/index.htm) + ].reduce((acc, codepoint) => { + const encoded = Buffer.from( + String.fromCharCode(codepoint), + options.encoding, + ); + if (codepoint !== 0x3f && encoded.length === 1 && encoded[0] === 0x3f) { + return acc; + } + acc.push(encoded); + return acc; + }, []); + // First-byte lookup table for `__isCharTrimable`. Non-whitespace bytes + // (the common case) bail out in O(1) without scanning every timchar. + const timcharFirstBytes = new Uint8Array(256); + for (const t of timchars) timcharFirstBytes[t[0]] = 1; + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + delimiterBufPrevious: undefined, + delimiterDiscovered: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: + isBuffer(options.escape) && + isBuffer(options.quote) && + Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) + ? options.columns.length + : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...(options.delimiter + ? options.delimiter.map((delimiter) => delimiter.length) + : []), + // Auto discovery of delimiter is limited to 1 character + options.delimiter_auto ? 1 : 0, + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + // Skip if the remaining buffer can be a multi-byte trim character + ...timchars.map((t) => t.length), + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: + options.record_delimiter.length === 0 + ? 0 + : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [ + Buffer.from(" ", options.encoding)[0], + Buffer.from("\t", options.encoding)[0], + ], + wasQuoting: false, + wasRowDelimiter: false, + timchars: timchars, + timcharFirstBytes: timcharFirstBytes, + }; +}; + +const underscore = function (str) { + return str.replace(/([A-Z])/g, function (_, match) { + return "_" + match.toLowerCase(); + }); +}; + +const normalize_options = function (opts) { + const options = {}; + // Merge with user options + for (const opt in opts) { + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if (options.encoding === undefined || options.encoding === true) { + options.encoding = "utf8"; + } else if (options.encoding === null || options.encoding === false) { + options.encoding = null; + } else if ( + typeof options.encoding !== "string" && + options.encoding !== null + ) { + throw new CsvError( + "CSV_INVALID_OPTION_ENCODING", + [ + "Invalid option encoding:", + "encoding must be a string or null to return a buffer,", + `got ${JSON.stringify(options.encoding)}`, + ], + options, + ); + } + // Normalize option `bom` + if ( + options.bom === undefined || + options.bom === null || + options.bom === false + ) { + options.bom = false; + } else if (options.bom !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_BOM", + [ + "Invalid option bom:", + "bom must be true,", + `got ${JSON.stringify(options.bom)}`, + ], + options, + ); + } + // Normalize option `cast` + options.cast_function = null; + if ( + options.cast === undefined || + options.cast === null || + options.cast === false || + options.cast === "" + ) { + options.cast = undefined; + } else if (typeof options.cast === "function") { + options.cast_function = options.cast; + options.cast = true; + } else if (options.cast !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_CAST", + [ + "Invalid option cast:", + "cast must be true or a function,", + `got ${JSON.stringify(options.cast)}`, + ], + options, + ); + } + // Normalize option `cast_date` + if ( + options.cast_date === undefined || + options.cast_date === null || + options.cast_date === false || + options.cast_date === "" + ) { + options.cast_date = false; + } else if (options.cast_date === true) { + options.cast_date = function (value) { + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + } else if (typeof options.cast_date !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_CAST_DATE", + [ + "Invalid option cast_date:", + "cast_date must be true or a function,", + `got ${JSON.stringify(options.cast_date)}`, + ], + options, + ); + } + // Normalize option `columns` + options.cast_first_line_to_header = undefined; + if (options.columns === true) { + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + } else if (typeof options.columns === "function") { + options.cast_first_line_to_header = options.columns; + options.columns = true; + } else if (Array.isArray(options.columns)) { + options.columns = normalize_columns_array(options.columns); + } else if ( + options.columns === undefined || + options.columns === null || + options.columns === false + ) { + options.columns = false; + } else { + throw new CsvError( + "CSV_INVALID_OPTION_COLUMNS", + [ + "Invalid option columns:", + "expect an array, a function or true,", + `got ${JSON.stringify(options.columns)}`, + ], + options, + ); + } + // Normalize option `group_columns_by_name` + if ( + options.group_columns_by_name === undefined || + options.group_columns_by_name === null || + options.group_columns_by_name === false + ) { + options.group_columns_by_name = false; + } else if (options.group_columns_by_name !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "expect a boolean,", + `got ${JSON.stringify(options.group_columns_by_name)}`, + ], + options, + ); + } else if (options.columns === false) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "the `columns` mode must be activated.", + ], + options, + ); + } + // Normalize option `comment` + if ( + options.comment === undefined || + options.comment === null || + options.comment === false || + options.comment === "" + ) { + options.comment = null; + } else { + if (typeof options.comment === "string") { + options.comment = Buffer.from(options.comment, options.encoding); + } + if (!isBuffer(options.comment)) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment:", + "comment must be a buffer or a string,", + `got ${JSON.stringify(options.comment)}`, + ], + options, + ); + } + } + // Normalize option `comment_no_infix` + if ( + options.comment_no_infix === undefined || + options.comment_no_infix === null || + options.comment_no_infix === false + ) { + options.comment_no_infix = false; + } else if (options.comment_no_infix !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment_no_infix:", + "value must be a boolean,", + `got ${JSON.stringify(options.comment_no_infix)}`, + ], + options, + ); + } + // Normalize option `delimiter_auto` + if ( + options.delimiter_auto === undefined || + options.delimiter_auto === null || + options.delimiter_auto === false + ) { + options.delimiter_auto = false; + } else if (options.delimiter_auto === true) { + options.delimiter_auto = {}; + } else if (!is_object(options.delimiter_auto)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "delimiter_auto must be a boolean or a configuration object,", + `got ${JSON.stringify(options.delimiter_auto)}`, + ], + options, + ); + } + if (options.delimiter_auto) { + if (options.delimiter_auto.preferred === undefined) + options.delimiter_auto.preferred = { + [",".charCodeAt(0)]: 1.8, + ["\t".charCodeAt(0)]: 1.8, + [";".charCodeAt(0)]: 1.6, + [" ".charCodeAt(0)]: 1.6, + [":".charCodeAt(0)]: 1.5, + [".".charCodeAt(0)]: 1.4, + ["/".charCodeAt(0)]: 1.4, + }; + else if (!is_object(options.delimiter_auto.preferred)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "preferred must be an object,", + `got ${JSON.stringify(options.delimiter_auto.preferred)}`, + ], + options, + ); + } + if (options.delimiter_auto.score === undefined) + options.delimiter_auto.score = (info, options) => { + return ( + (info.total - info.std) * (options.preferred[info.char_code] || 1) + ); + }; + else if (typeof options.delimiter_auto.score !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "score must be a function,", + `got ${JSON.stringify(options.delimiter_auto.score)}`, + ], + options, + ); + } + if (options.delimiter_auto.size === undefined) + options.delimiter_auto.size = 2048; + else if (typeof options.delimiter_auto.size !== "number") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "size must be a number,", + `got ${JSON.stringify(options.delimiter_auto.size)}`, + ], + options, + ); + } + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if (options.delimiter_auto !== false) { + options.delimiter = []; + } + if (!Array.isArray(options.delimiter)) { + if ( + options.delimiter === undefined || + options.delimiter === null || + options.delimiter === false + ) { + options.delimiter = Buffer.from(",", options.encoding); + } + options.delimiter = [options.delimiter]; + } + options.delimiter = options.delimiter.map(function (delimiter) { + if (typeof delimiter === "string") { + delimiter = Buffer.from(delimiter, options.encoding); + } + if (!isBuffer(delimiter) || delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER", + [ + "Invalid option delimiter:", + "delimiter must be a non empty string or buffer or array of string|buffer,", + `got ${delimiter_json}`, + ], + options, + ); + } + return delimiter; + }); + // Normalize option `escape` + if (options.escape === undefined || options.escape === true) { + options.escape = Buffer.from('"', options.encoding); + } else if (typeof options.escape === "string") { + options.escape = Buffer.from(options.escape, options.encoding); + } else if (options.escape === null || options.escape === false) { + options.escape = null; + } + if (options.escape !== null) { + if (!isBuffer(options.escape)) { + throw new Error( + `Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`, + ); + } + } + // Normalize option `from` + if (options.from === undefined || options.from === null) { + options.from = 1; + } else { + if (typeof options.from === "string" && /\d+/.test(options.from)) { + options.from = parseInt(options.from); + } + if (Number.isInteger(options.from)) { + if (options.from < 0) { + throw new Error( + `Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`, + ); + } + } + // Normalize option `from_line` + if (options.from_line === undefined || options.from_line === null) { + options.from_line = 1; + } else { + if ( + typeof options.from_line === "string" && + /\d+/.test(options.from_line) + ) { + options.from_line = parseInt(options.from_line); + } + if (Number.isInteger(options.from_line)) { + if (options.from_line <= 0) { + throw new Error( + `Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`, + ); + } + } + // Normalize options `ignore_last_delimiters` + if ( + options.ignore_last_delimiters === undefined || + options.ignore_last_delimiters === null + ) { + options.ignore_last_delimiters = false; + } else if (typeof options.ignore_last_delimiters === "number") { + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if (options.ignore_last_delimiters === 0) { + options.ignore_last_delimiters = false; + } + } else if (typeof options.ignore_last_delimiters !== "boolean") { + throw new CsvError( + "CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS", + [ + "Invalid option `ignore_last_delimiters`:", + "the value must be a boolean value or an integer,", + `got ${JSON.stringify(options.ignore_last_delimiters)}`, + ], + options, + ); + } + if (options.ignore_last_delimiters === true && options.columns === false) { + throw new CsvError( + "CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS", + [ + "The option `ignore_last_delimiters`", + "requires the activation of the `columns` option", + ], + options, + ); + } + // Normalize option `info` + if ( + options.info === undefined || + options.info === null || + options.info === false + ) { + options.info = false; + } else if (options.info !== true) { + throw new Error( + `Invalid Option: info must be true, got ${JSON.stringify(options.info)}`, + ); + } + // Normalize option `max_record_size` + if ( + options.max_record_size === undefined || + options.max_record_size === null || + options.max_record_size === false + ) { + options.max_record_size = 0; + } else if ( + Number.isInteger(options.max_record_size) && + options.max_record_size >= 0 + ) ; else if ( + typeof options.max_record_size === "string" && + /\d+/.test(options.max_record_size) + ) { + options.max_record_size = parseInt(options.max_record_size); + } else { + throw new Error( + `Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`, + ); + } + // Normalize option `objname` + if ( + options.objname === undefined || + options.objname === null || + options.objname === false + ) { + options.objname = undefined; + } else if (isBuffer(options.objname)) { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if (options.encoding === null) ; else { + options.objname = options.objname.toString(options.encoding); + } + } else if (typeof options.objname === "string") { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + } else if (typeof options.objname === "number") ; else { + throw new Error( + `Invalid Option: objname must be a string or a buffer, got ${options.objname}`, + ); + } + if (options.objname !== undefined) { + if (typeof options.objname === "number") { + if (options.columns !== false) { + throw Error( + "Invalid Option: objname index cannot be combined with columns or be defined as a field", + ); + } + } else { + // A string or a buffer + if (options.columns === false) { + throw Error( + "Invalid Option: objname field must be combined with columns or be defined as an index", + ); + } + } + } + // Normalize option `on_record` + if (options.on_record === undefined || options.on_record === null) { + options.on_record = undefined; + } else if (typeof options.on_record !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_ON_RECORD", + [ + "Invalid option `on_record`:", + "expect a function,", + `got ${JSON.stringify(options.on_record)}`, + ], + options, + ); + } + // Normalize option `on_skip` + // options.on_skip ??= (err, chunk) => { + // this.emit('skip', err, chunk); + // }; + if ( + options.on_skip !== undefined && + options.on_skip !== null && + typeof options.on_skip !== "function" + ) { + throw new Error( + `Invalid Option: on_skip must be a function, got ${JSON.stringify(options.on_skip)}`, + ); + } + // Normalize option `quote` + if ( + options.quote === null || + options.quote === false || + options.quote === "" + ) { + options.quote = null; + } else { + if (options.quote === undefined || options.quote === true) { + options.quote = Buffer.from('"', options.encoding); + } else if (typeof options.quote === "string") { + options.quote = Buffer.from(options.quote, options.encoding); + } + if (!isBuffer(options.quote)) { + throw new Error( + `Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`, + ); + } + } + // Normalize option `raw` + if ( + options.raw === undefined || + options.raw === null || + options.raw === false + ) { + options.raw = false; + } else if (options.raw !== true) { + throw new Error( + `Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`, + ); + } + // Normalize option `record_delimiter` + if (options.record_delimiter === undefined) { + options.record_delimiter = []; + } else if ( + typeof options.record_delimiter === "string" || + isBuffer(options.record_delimiter) + ) { + if (options.record_delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = [options.record_delimiter]; + } else if (!Array.isArray(options.record_delimiter)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = options.record_delimiter.map(function (rd, i) { + if (typeof rd !== "string" && !isBuffer(rd)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } else if (rd.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } + if (typeof rd === "string") { + rd = Buffer.from(rd, options.encoding); + } + return rd; + }); + // Normalize option `relax_column_count` + if (typeof options.relax_column_count === "boolean") ; else if ( + options.relax_column_count === undefined || + options.relax_column_count === null + ) { + options.relax_column_count = false; + } else { + throw new Error( + `Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`, + ); + } + if (typeof options.relax_column_count_less === "boolean") ; else if ( + options.relax_column_count_less === undefined || + options.relax_column_count_less === null + ) { + options.relax_column_count_less = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`, + ); + } + if (typeof options.relax_column_count_more === "boolean") ; else if ( + options.relax_column_count_more === undefined || + options.relax_column_count_more === null + ) { + options.relax_column_count_more = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`, + ); + } + // Normalize option `relax_quotes` + if (typeof options.relax_quotes === "boolean") ; else if ( + options.relax_quotes === undefined || + options.relax_quotes === null + ) { + options.relax_quotes = false; + } else { + throw new Error( + `Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`, + ); + } + // Normalize option `skip_empty_lines` + if (typeof options.skip_empty_lines === "boolean") ; else if ( + options.skip_empty_lines === undefined || + options.skip_empty_lines === null + ) { + options.skip_empty_lines = false; + } else { + throw new Error( + `Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`, + ); + } + // Normalize option `skip_records_with_empty_values` + if (typeof options.skip_records_with_empty_values === "boolean") ; else if ( + options.skip_records_with_empty_values === undefined || + options.skip_records_with_empty_values === null + ) { + options.skip_records_with_empty_values = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`, + ); + } + // Normalize option `skip_records_with_error` + if (typeof options.skip_records_with_error === "boolean") ; else if ( + options.skip_records_with_error === undefined || + options.skip_records_with_error === null + ) { + options.skip_records_with_error = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`, + ); + } + // Normalize option `rtrim` + if ( + options.rtrim === undefined || + options.rtrim === null || + options.rtrim === false + ) { + options.rtrim = false; + } else if (options.rtrim !== true) { + throw new Error( + `Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`, + ); + } + // Normalize option `ltrim` + if ( + options.ltrim === undefined || + options.ltrim === null || + options.ltrim === false + ) { + options.ltrim = false; + } else if (options.ltrim !== true) { + throw new Error( + `Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`, + ); + } + // Normalize option `trim` + if ( + options.trim === undefined || + options.trim === null || + options.trim === false + ) { + options.trim = false; + } else if (options.trim !== true) { + throw new Error( + `Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`, + ); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if (options.trim === true && opts.ltrim !== false) { + options.ltrim = true; + } else if (options.ltrim !== true) { + options.ltrim = false; + } + if (options.trim === true && opts.rtrim !== false) { + options.rtrim = true; + } else if (options.rtrim !== true) { + options.rtrim = false; + } + // Normalize option `to` + if (options.to === undefined || options.to === null) { + options.to = -1; + } else if (options.to !== -1) { + if (typeof options.to === "string" && /\d+/.test(options.to)) { + options.to = parseInt(options.to); + } + if (Number.isInteger(options.to)) { + if (options.to <= 0) { + throw new Error( + `Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`, + ); + } + } + // Normalize option `to_line` + if (options.to_line === undefined || options.to_line === null) { + options.to_line = -1; + } else if (options.to_line !== -1) { + if (typeof options.to_line === "string" && /\d+/.test(options.to_line)) { + options.to_line = parseInt(options.to_line); + } + if (Number.isInteger(options.to_line)) { + if (options.to_line <= 0) { + throw new Error( + `Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`, + ); + } + } + return options; +}; + +// Discussed in [issue #400](https://github.com/adaltas/node-csv/issues/400) +// See https://github.com/python/cpython/blob/ea1b1c579f600cc85d145c60862b2e6b98701b24/Lib/csv.py#L349 +const delimiter_discover = function (records, options) { + // Normalize the configuration + if (!options) { + ({ delimiter_auto: options } = normalize_options({ delimiter_auto: true })); + } + // Convert String to Buffer + if (typeof records === "string") { + records = Buffer.from(records); + } + // Convert Buffer to an array of records + if (isBuffer(records)) { + records = ((data) => { + const records = []; + const parser = transform({ delimiter: [] }); + const push = (record) => records.push(record); + const close = () => {}; + const error = parser.parse(data, true, push, close); + if (error !== undefined) throw error; + return records; + })(records); + } + // Info array initialization, 127 entries, one per char code + const info = Array(127) + .fill() + .map(() => ({ lines: [] })); + // Traverse each records, count occurences per char code + records.map(([record], line) => { + for (let i = 0, l = record.length; i < l; i++) { + // Count the character frequency + const code = record.charCodeAt(i); + if (info[code].lines[line] === undefined) info[code].lines[line] = 0; + info[code].lines[line]++; + } + }); + // Traverse each char code, compute the score + info.map((info, i) => { + info.char_code = i; + info.std = std(info.lines); + info.total = info.lines.reduce((acc, val) => acc + val, 0); + info.preferred = !!options.preferred[i]; + info.score = options.score(info, options); + }); + // Extract the dominant character + const result = info.reduce( + (acc, info) => (acc.score > info.score ? acc : info), + {}, + ); + return String.fromCharCode(result.char_code); +}; + +const std = function (array) { + const n = array.length; + if (n === 0) return 0; + const mean = array.reduce((a, b) => a + b) / n; + return Math.sqrt( + array.map((x) => Math.pow(x - mean, 2)).reduce((a, b) => a + b) / n, + ); +}; + +const isRecordEmpty = function (record) { + return record.every( + (field) => + field == null || (field.toString && field.toString().trim() === ""), + ); +}; + +const cr = 13; // `\r`, carriage return, 0x0D in hexadécimal, 13 in decimal +const nl = 10; // `\n`, newline, 0x0A in hexadecimal, 10 in decimal + +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + utf8: Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + utf16le: Buffer.from([255, 254]), +}; + +const transform = function (original_options = {}) { + const info = { + bytes: 0, + bytes_records: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0, + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function (i, bufLen, end) { + if (end) return false; + const { encoding, escape, quote } = this.options; + const { quoting, needMoreDataSize, recordDelimiterMaxLength } = + this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + // If "record_delimiter" is yet to be discovered: + // 1. It is equals to `[]` and "recordDelimiterMaxLength" equals `0` + // 2. We set the length to windows line ending in the current encoding + // Note, that encoding is known from user or bom discovery at that point + // recordDelimiterMaxLength, + recordDelimiterMaxLength === 0 + ? Buffer.from("\r\n", encoding).length + : recordDelimiterMaxLength, + // Skip if remaining buffer can be an escaped quote + quoting ? (escape === null ? 0 : escape.length) + quote.length : 0, + // Skip if remaining buffer can be record delimiter following the closing quote + quoting ? quote.length + recordDelimiterMaxLength : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function (nextBuf, end, push, close) { + const { + bom, + comment_no_infix, + delimiter_auto, + encoding, + from_line, + ltrim, + max_record_size, + raw, + relax_quotes, + rtrim, + skip_empty_lines, + to, + to_line, + } = this.options; + let { comment, escape, quote, record_delimiter } = this.options; + const { + bomSkipped, + delimiterDiscovered, + delimiterBufPrevious, + rawBuffer, + escapeIsQuote, + } = this.state; + // Automatic delimiter discovery + if (!delimiterDiscovered && delimiter_auto) { + let delimiterBuf; + if (delimiterBufPrevious === undefined) { + delimiterBuf = nextBuf; + } else if ( + delimiterBufPrevious !== undefined && + nextBuf === undefined + ) { + delimiterBuf = delimiterBufPrevious; + } else { + delimiterBuf = Buffer.concat([delimiterBufPrevious, nextBuf]); + } + // Ensure that nextBuf is not concatenated a second time during buffer reconciliation + nextBuf = undefined; + // this.delimiterBufPrevious = delimiterBuf; + if (end || delimiterBuf.length > delimiter_auto.size) { + this.options.delimiter = [ + Buffer.from( + delimiter_discover(delimiterBuf, this.options.delimiter_auto), + ), + ]; + this.state.previousBuf = delimiterBuf; + this.state.delimiterBufPrevious = undefined; + this.state.delimiterDiscovered = true; + } else { + this.state.delimiterBufPrevious = delimiterBuf; + return; + } + } + // Previous buffers reconciliation + const { previousBuf } = this.state; + let buf; + if (previousBuf === undefined) { + if (nextBuf === undefined) { + // Handle empty string + close(); + return; + } else { + buf = nextBuf; + } + } else if (previousBuf !== undefined && nextBuf === undefined) { + buf = previousBuf; + } else { + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if (bomSkipped === false) { + if (bom === false) { + this.state.bomSkipped = true; + } else if (buf.length < 3) { + // No enough data + if (end === false) { + // Wait for more data + this.state.previousBuf = buf; + return; + } + } else { + for (const encoding in boms) { + if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) { + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + const options = normalize_options({ + ...this.original_options, + encoding: encoding, + }); + // Properties are merged with the existing options instance + for (const key in options) { + this.options[key] = options[key]; + } + // Options will re-evaluate the Buffer with the new encoding + ({ comment, escape, quote } = this.options); + break; + } + } + this.state.bomSkipped = true; + } + } + const bufLen = buf.length; + let pos; + for (pos = 0; pos < bufLen; pos++) { + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if (this.__needMoreData(pos, bufLen, end)) { + break; + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if (to_line !== -1 && this.info.lines > to_line) { + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if (this.state.quoting === false && record_delimiter.length === 0) { + const record_delimiterCount = this.__autoDiscoverRecordDelimiter( + buf, + pos, + ); + if (record_delimiterCount) { + record_delimiter = this.options.record_delimiter; + } + } + const chr = buf[pos]; + if (raw === true) { + rawBuffer.append(chr); + } + if ( + (chr === cr || chr === nl) && + this.state.wasRowDelimiter === false + ) { + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if (this.state.escaping === true) { + this.state.escaping = false; + } else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if ( + escape !== null && + this.state.quoting === true && + this.__isEscape(buf, pos, chr) && + pos + escape.length < bufLen + ) { + if (escapeIsQuote) { + if (this.__isQuote(buf, pos + escape.length)) { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } else { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if (this.state.commenting === false && this.__isQuote(buf, pos)) { + if (this.state.quoting === true) { + const nextChr = buf[pos + quote.length]; + const isNextChrTrimable = + rtrim && this.__isCharTrimable(buf, pos + quote.length); + const isNextChrComment = + comment !== null && + this.__compareBytes(comment, buf, pos + quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter( + buf, + pos + quote.length, + nextChr, + ); + const isNextChrRecordDelimiter = + record_delimiter.length === 0 + ? this.__autoDiscoverRecordDelimiter(buf, pos + quote.length) + : this.__isRecordDelimiter(nextChr, buf, pos + quote.length); + // Escape a quote + // Treat next char as a regular character + if ( + escape !== null && + this.__isEscape(buf, pos, chr) && + this.__isQuote(buf, pos + escape.length) + ) { + pos += escape.length - 1; + } else if ( + !nextChr || + isNextChrDelimiter || + isNextChrRecordDelimiter || + isNextChrComment || + isNextChrTrimable + ) { + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + } else if (relax_quotes === false) { + const err = this.__error( + new CsvError( + "CSV_INVALID_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + "instead of delimiter, record delimiter, trimable character", + "(if activated) or comment", + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; + } + } else { + if (this.state.field.length !== 0) { + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if (relax_quotes === false) { + const info = this.__infoField(); + const bom = Object.keys(boms) + .map((b) => + boms[b].equals(this.state.field.toString()) ? b : false, + ) + .filter(Boolean)[0]; + const err = this.__error( + new CsvError( + "INVALID_OPENING_QUOTE", + [ + "Invalid Opening Quote:", + `a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`, + bom ? `(${bom} bom)` : undefined, + ], + this.options, + info, + { + field: this.state.field, + }, + ), + ); + if (err !== undefined) return err; + } + } else { + this.state.quoting = true; + pos += quote.length - 1; + continue; + } + } + } + if (this.state.quoting === false) { + const recordDelimiterLength = this.__isRecordDelimiter( + chr, + buf, + pos, + ); + if (recordDelimiterLength !== 0) { + // Do not emit comments which take a full line + const skipCommentLine = + this.state.commenting && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0; + if (skipCommentLine) { + this.info.comment_lines++; + // Skip full comment line + } else { + // Activate records emission if above from_line + if ( + this.state.enabled === false && + this.info.lines + + (this.state.wasRowDelimiter === true ? 1 : 0) >= + from_line + ) { + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if ( + skip_empty_lines === true && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0 + ) { + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + this.info.bytes = + this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + if (to !== -1 && this.info.records >= to) { + this.state.stop = true; + close(); + return; + } + } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if (this.state.commenting) { + continue; + } + if ( + comment !== null && + (comment_no_infix === false || + (this.state.record.length === 0 && + this.state.field.length === 0)) + ) { + const commentCount = this.__compareBytes(comment, buf, pos, chr); + if (commentCount !== 0) { + this.state.commenting = true; + continue; + } + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if (delimiterLength !== 0) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + pos += delimiterLength - 1; + continue; + } + } + } + if (this.state.commenting === false) { + if ( + max_record_size !== 0 && + this.state.record_length + this.state.field.length > max_record_size + ) { + return this.__error( + new CsvError( + "CSV_MAX_RECORD_SIZE", + [ + "Max Record Size:", + "record exceed the maximum number of tolerated bytes", + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } + } + const lappend = + ltrim === false || + this.state.quoting === true || + this.state.field.length !== 0 || + !this.__isCharTrimable(buf, pos); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if (lappend === true && rappend === true) { + this.state.field.append(chr); + } else if (rtrim === true && !this.__isCharTrimable(buf, pos)) { + return this.__error( + new CsvError( + "CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + "found non trimable byte after quote", + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } else { + if (lappend === false) { + pos += this.__isCharTrimable(buf, pos) - 1; + } + continue; + } + } + if (end === true) { + // Ensure we are not ending in a quoting state + if (this.state.quoting === true) { + const err = this.__error( + new CsvError( + "CSV_QUOTE_NOT_CLOSED", + [ + "Quote Not Closed:", + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + // Skip last line if it has no characters + if ( + this.state.wasQuoting === true || + this.state.record.length !== 0 || + this.state.field.length !== 0 + ) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + } else if (this.state.wasRowDelimiter === true) { + this.info.empty_lines++; + } else if (this.state.commenting === true) { + this.info.comment_lines++; + } + } + } else { + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function (push) { + const { + columns, + group_columns_by_name, + encoding, + info, + from, + relax_column_count, + relax_column_count_less, + relax_column_count_more, + raw, + skip_records_with_empty_values, + } = this.options; + const { enabled, record } = this.state; + if (enabled === false) { + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if (columns === true) { + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if (columns === false && this.info.records === 0) { + this.state.expectedRecordLength = recordLength; + } + if (recordLength !== this.state.expectedRecordLength) { + const err = + columns === false + ? new CsvError( + "CSV_RECORD_INCONSISTENT_FIELDS_LENGTH", + [ + "Invalid Record Length:", + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ) + : new CsvError( + "CSV_RECORD_INCONSISTENT_COLUMNS", + [ + "Invalid Record Length:", + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ); + if ( + relax_column_count === true || + (relax_column_count_less === true && + recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && + recordLength > this.state.expectedRecordLength) + ) { + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + } else { + const finalErr = this.__error(err); + if (finalErr) return finalErr; + } + } + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + if (this.state.recordHasError === true) { + this.__resetRecord(); + this.state.recordHasError = false; + return; + } + this.info.records++; + if (from === 1 || this.info.records >= from) { + const { objname } = this.options; + // With columns, records are object + if (columns !== false) { + const obj = {}; + // Transform record array to an object + for (let i = 0, l = record.length; i < l; i++) { + if (columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if ( + group_columns_by_name === true && + Object.hasOwn(obj, columns[i].name) + ) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } + } else { + Object.defineProperty(obj, columns[i].name, { + value: record[i], + enumerable: true, + writable: true, + configurable: true, + }); + } + } + // Without objname (default) + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: obj }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj], + push, + ); + if (err) { + return err; + } + } + // Without columns, records are array + } else { + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: record }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? record : [record[objname], record], + push, + ); + if (err) { + return err; + } + } + } + } + this.__resetRecord(); + }, + __firstLineToColumns: function (record) { + const { firstLineToHeaders } = this.state; + try { + const headers = + firstLineToHeaders === undefined + ? record + : firstLineToHeaders.call(null, record); + if (!Array.isArray(headers)) { + return this.__error( + new CsvError( + "CSV_INVALID_COLUMN_MAPPING", + [ + "Invalid Column Mapping:", + "expect an array from column function,", + `got ${JSON.stringify(headers)}`, + ], + this.options, + this.__infoField(), + { + headers: headers, + }, + ), + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; + } catch (err) { + return err; + } + }, + __resetRecord: function () { + if (this.options.raw === true) { + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function () { + const { cast, encoding, rtrim, max_record_size } = this.options; + const { enabled, wasQuoting } = this.state; + // Short circuit for the from_line options + if (enabled === false) { + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if (rtrim === true && wasQuoting === false) { + field = field.trimRight(); + } + if (cast === true) { + const [err, f] = this.__cast(field); + if (err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if (max_record_size !== 0 && typeof field === "string") { + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function () { + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function (record, push) { + const { on_record } = this.options; + if (on_record !== undefined) { + const info = this.__infoRecord(); + try { + record = on_record.call(null, record, info); + } catch (err) { + return err; + } + if (record === undefined || record === null) { + return; + } + } + this.info.bytes_records += this.info.bytes; + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function (field) { + const { columns, relax_column_count } = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if ( + isColumns === true && + relax_column_count && + this.options.columns.length <= this.state.record.length + ) { + return [undefined, undefined]; + } + if (this.state.castField !== null) { + try { + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + } catch (err) { + return [err]; + } + } + if (this.__isFloat(field)) { + return [undefined, Number(field)]; + } else if (this.options.cast_date !== false) { + const info = this.__infoField(); + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + __compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) { + if (sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for (let i = 1; i < sourceLength; i++) { + if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0; + } + return sourceLength; + }, + // Helper to test if a character is trimable + __isCharTrimable: function (buf, pos) { + const { timchars, timcharFirstBytes } = this.state; + // Fast bail-out: non-whitespace bytes (the common case) are rejected + // without scanning the full timchar list. + const first = buf[pos]; + if (first === undefined || timcharFirstBytes[first] === 0) return 0; + loop1: for (let i = 0; i < timchars.length; i++) { + const timchar = timchars[i]; + for (let j = 0; j < timchar.length; j++) { + if (timchar[j] !== buf[pos + j]) continue loop1; + } + return timchar.length; + } + return 0; + }, + __isDelimiter: function (buf, pos, chr) { + const { delimiter, ignore_last_delimiters } = this.options; + if ( + ignore_last_delimiters === true && + this.state.record.length === this.options.columns.length - 1 + ) { + return 0; + } else if ( + ignore_last_delimiters !== false && + typeof ignore_last_delimiters === "number" && + this.state.record.length === ignore_last_delimiters - 1 + ) { + return 0; + } + loop1: for (let i = 0; i < delimiter.length; i++) { + const del = delimiter[i]; + if (del[0] === chr) { + for (let j = 1; j < del.length; j++) { + if (del[j] !== buf[pos + j]) continue loop1; + } + return del.length; + } + } + return 0; + }, + __isEscape: function (buf, pos, chr) { + const { escape } = this.options; + if (escape === null) return false; + const l = escape.length; + if (escape[0] === chr) { + for (let i = 0; i < l; i++) { + if (escape[i] !== buf[pos + i]) { + return false; + } + } + return true; + } + return false; + }, + __isFloat: function (value) { + return value - parseFloat(value) + 1 >= 0; // Borrowed from jquery + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isQuote: function (buf, pos) { + const { quote } = this.options; + if (quote === null) return false; + const l = quote.length; + for (let i = 0; i < l; i++) { + if (quote[i] !== buf[pos + i]) { + return false; + } + } + return true; + }, + __isRecordDelimiter: function (chr, buf, pos) { + const { record_delimiter } = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for (let i = 0; i < recordDelimiterLength; i++) { + const rd = record_delimiter[i]; + const rdLength = rd.length; + if (rd[0] !== chr) { + continue; + } + for (let j = 1; j < rdLength; j++) { + if (rd[j] !== buf[pos + j]) { + continue loop1; + } + } + return rd.length; + } + return 0; + }, + __autoDiscoverRecordDelimiter: function (buf, pos) { + const { encoding } = this.options; + // Note, we don't need to cache this information in state, + // It is only called on the first line until we find out a suitable + // record delimiter. + const rds = [ + // Important, the windows line ending must be before mac os 9 + Buffer.from("\r\n", encoding), + Buffer.from("\n", encoding), + Buffer.from("\r", encoding), + ]; + loop: for (let i = 0; i < rds.length; i++) { + const l = rds[i].length; + for (let j = 0; j < l; j++) { + if (rds[i][j] !== buf[pos + j]) { + continue loop; + } + } + this.options.record_delimiter.push(rds[i]); + this.state.recordDelimiterMaxLength = rds[i].length; + return rds[i].length; + } + return 0; + }, + __error: function (msg) { + const { encoding, raw, skip_records_with_error } = this.options; + const err = typeof msg === "string" ? new Error(msg) : msg; + if (skip_records_with_error) { + this.state.recordHasError = true; + if (this.options.on_skip !== undefined) { + try { + this.options.on_skip( + err, + raw ? this.state.rawBuffer.toString(encoding) : undefined, + ); + } catch (err) { + return err; + } + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + } else { + return err; + } + }, + __infoDataSet: function () { + return { + ...this.info, + columns: this.options.columns, + }; + }, + __infoRecord: function () { + const { columns, raw, encoding } = this.options; + return { + ...this.__infoDataSet(), + bytes_records: this.info.bytes, + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined, + }; + }, + __infoField: function () { + const { columns } = this.options; + const isColumns = Array.isArray(columns); + // Bytes records are only incremented when all records'fields are parsed + const bytes_records = this.info.bytes_records; + return { + ...this.__infoRecord(), + bytes_records: bytes_records, + column: + isColumns === true + ? columns.length > this.state.record.length + ? columns[this.state.record.length].name + : null + : this.state.record.length, + quoting: this.state.wasQuoting, + }; + }, + }; +}; + +// Parse an asynchronous iterable without a stream wrapper. Drain each chunk's +// records before requesting another chunk, including records preceding an error. +async function* parse(input, opts = {}) { + const api = transform(opts); + let records = []; + let stopped = false; + const push = (record) => records.push(record); + const close = () => { + stopped = true; + }; + for await (const chunk of input) { + const buffer = chunk instanceof Buffer ? chunk : Buffer.from(chunk); + const error = api.parse(buffer, false, push, close); + for (const record of records) yield record; + records = []; + if (error !== undefined) throw error; + if (stopped) return; + } + const error = api.parse(undefined, true, push, close); + for (const record of records) yield record; + if (error !== undefined) throw error; +} + +export { CsvError, normalize_options, parse }; diff --git a/packages/csv-parse/dist/iife/iterator.js b/packages/csv-parse/dist/iife/iterator.js new file mode 100644 index 000000000..1377fc20d --- /dev/null +++ b/packages/csv-parse/dist/iife/iterator.js @@ -0,0 +1,3992 @@ +var csv_parse_iterator = (function (exports) { + 'use strict'; + + var global$1 = (typeof global !== "undefined" ? global : + typeof self !== "undefined" ? self : + typeof window !== "undefined" ? window : {}); + + var lookup = []; + var revLookup = []; + var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; + var inited = false; + function init () { + inited = true; + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; + } + + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; + } + + function toByteArray (b64) { + if (!inited) { + init(); + } + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; + + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') + } + + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; + + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); + + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; + + var L = 0; + + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + return arr + } + + function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] + } + + function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); + } + return output.join('') + } + + function fromByteArray (uint8) { + if (!inited) { + init(); + } + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 + + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); + } + + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; + } + + parts.push(output); + + return parts.join('') + } + + function read (buffer, offset, isLE, mLen, nBytes) { + var e, m; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var nBits = -7; + var i = isLE ? (nBytes - 1) : 0; + var d = isLE ? -1 : 1; + var s = buffer[offset + i]; + + i += d; + + e = s & ((1 << (-nBits)) - 1); + s >>= (-nBits); + nBits += eLen; + for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + m = e & ((1 << (-nBits)) - 1); + e >>= (-nBits); + nBits += mLen; + for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + if (e === 0) { + e = 1 - eBias; + } else if (e === eMax) { + return m ? NaN : ((s ? -1 : 1) * Infinity) + } else { + m = m + Math.pow(2, mLen); + e = e - eBias; + } + return (s ? -1 : 1) * m * Math.pow(2, e - mLen) + } + + function write (buffer, value, offset, isLE, mLen, nBytes) { + var e, m, c; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); + var i = isLE ? 0 : (nBytes - 1); + var d = isLE ? 1 : -1; + var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; + + value = Math.abs(value); + + if (isNaN(value) || value === Infinity) { + m = isNaN(value) ? 1 : 0; + e = eMax; + } else { + e = Math.floor(Math.log(value) / Math.LN2); + if (value * (c = Math.pow(2, -e)) < 1) { + e--; + c *= 2; + } + if (e + eBias >= 1) { + value += rt / c; + } else { + value += rt * Math.pow(2, 1 - eBias); + } + if (value * c >= 2) { + e++; + c /= 2; + } + + if (e + eBias >= eMax) { + m = 0; + e = eMax; + } else if (e + eBias >= 1) { + m = (value * c - 1) * Math.pow(2, mLen); + e = e + eBias; + } else { + m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); + e = 0; + } + } + + for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} + + e = (e << mLen) | m; + eLen += mLen; + for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} + + buffer[offset + i - d] |= s * 128; + } + + var toString = {}.toString; + + var isArray = Array.isArray || function (arr) { + return toString.call(arr) == '[object Array]'; + }; + + var INSPECT_MAX_BYTES = 50; + + /** + * If `Buffer.TYPED_ARRAY_SUPPORT`: + * === true Use Uint8Array implementation (fastest) + * === false Use Object implementation (most compatible, even IE6) + * + * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, + * Opera 11.6+, iOS 4.2+. + * + * Due to various browser bugs, sometimes the Object implementation will be used even + * when the browser supports typed arrays. + * + * Note: + * + * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, + * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. + * + * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. + * + * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of + * incorrect length in some situations. + + * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they + * get the Object implementation, which is slower but behaves correctly. + */ + Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined + ? global$1.TYPED_ARRAY_SUPPORT + : true; + + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + + function kMaxLength () { + return Buffer.TYPED_ARRAY_SUPPORT + ? 0x7fffffff + : 0x3fffffff + } + + function createBuffer (that, length) { + if (kMaxLength() < length) { + throw new RangeError('Invalid typed array length') + } + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = new Uint8Array(length); + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + if (that === null) { + that = new Buffer(length); + } + that.length = length; + } + + return that + } + + /** + * The Buffer constructor returns instances of `Uint8Array` that have their + * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of + * `Uint8Array`, so the returned instances will have all the node `Buffer` methods + * and the `Uint8Array` methods. Square bracket notation works as expected -- it + * returns a single octet. + * + * The `Uint8Array` prototype remains unmodified. + */ + + function Buffer (arg, encodingOrOffset, length) { + if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { + return new Buffer(arg, encodingOrOffset, length) + } + + // Common case. + if (typeof arg === 'number') { + if (typeof encodingOrOffset === 'string') { + throw new Error( + 'If encoding is specified then the first argument must be a string' + ) + } + return allocUnsafe(this, arg) + } + return from(this, arg, encodingOrOffset, length) + } + + Buffer.poolSize = 8192; // not used by this implementation + + // TODO: Legacy, not needed anymore. Remove in next major version. + Buffer._augment = function (arr) { + arr.__proto__ = Buffer.prototype; + return arr + }; + + function from (that, value, encodingOrOffset, length) { + if (typeof value === 'number') { + throw new TypeError('"value" argument must not be a number') + } + + if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { + return fromArrayBuffer(that, value, encodingOrOffset, length) + } + + if (typeof value === 'string') { + return fromString(that, value, encodingOrOffset) + } + + return fromObject(that, value) + } + + /** + * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError + * if value is a number. + * Buffer.from(str[, encoding]) + * Buffer.from(array) + * Buffer.from(buffer) + * Buffer.from(arrayBuffer[, byteOffset[, length]]) + **/ + Buffer.from = function (value, encodingOrOffset, length) { + return from(null, value, encodingOrOffset, length) + }; + + if (Buffer.TYPED_ARRAY_SUPPORT) { + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; + } + + function assertSize (size) { + if (typeof size !== 'number') { + throw new TypeError('"size" argument must be a number') + } else if (size < 0) { + throw new RangeError('"size" argument must not be negative') + } + } + + function alloc (that, size, fill, encoding) { + assertSize(size); + if (size <= 0) { + return createBuffer(that, size) + } + if (fill !== undefined) { + // Only pay attention to encoding if it's a string. This + // prevents accidentally sending in a number that would + // be interpretted as a start offset. + return typeof encoding === 'string' + ? createBuffer(that, size).fill(fill, encoding) + : createBuffer(that, size).fill(fill) + } + return createBuffer(that, size) + } + + /** + * Creates a new filled Buffer instance. + * alloc(size[, fill[, encoding]]) + **/ + Buffer.alloc = function (size, fill, encoding) { + return alloc(null, size, fill, encoding) + }; + + function allocUnsafe (that, size) { + assertSize(size); + that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) { + for (var i = 0; i < size; ++i) { + that[i] = 0; + } + } + return that + } + + /** + * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. + * */ + Buffer.allocUnsafe = function (size) { + return allocUnsafe(null, size) + }; + /** + * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. + */ + Buffer.allocUnsafeSlow = function (size) { + return allocUnsafe(null, size) + }; + + function fromString (that, string, encoding) { + if (typeof encoding !== 'string' || encoding === '') { + encoding = 'utf8'; + } + + if (!Buffer.isEncoding(encoding)) { + throw new TypeError('"encoding" must be a valid string encoding') + } + + var length = byteLength(string, encoding) | 0; + that = createBuffer(that, length); + + var actual = that.write(string, encoding); + + if (actual !== length) { + // Writing a hex string, for example, that contains invalid characters will + // cause everything after the first invalid character to be ignored. (e.g. + // 'abxxcd' will be treated as 'ab') + that = that.slice(0, actual); + } + + return that + } + + function fromArrayLike (that, array) { + var length = array.length < 0 ? 0 : checked(array.length) | 0; + that = createBuffer(that, length); + for (var i = 0; i < length; i += 1) { + that[i] = array[i] & 255; + } + return that + } + + function fromArrayBuffer (that, array, byteOffset, length) { + array.byteLength; // this throws if `array` is not a valid ArrayBuffer + + if (byteOffset < 0 || array.byteLength < byteOffset) { + throw new RangeError('\'offset\' is out of bounds') + } + + if (array.byteLength < byteOffset + (length || 0)) { + throw new RangeError('\'length\' is out of bounds') + } + + if (byteOffset === undefined && length === undefined) { + array = new Uint8Array(array); + } else if (length === undefined) { + array = new Uint8Array(array, byteOffset); + } else { + array = new Uint8Array(array, byteOffset, length); + } + + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = array; + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + that = fromArrayLike(that, array); + } + return that + } + + function fromObject (that, obj) { + if (internalIsBuffer(obj)) { + var len = checked(obj.length) | 0; + that = createBuffer(that, len); + + if (that.length === 0) { + return that + } + + obj.copy(that, 0, 0, len); + return that + } + + if (obj) { + if ((typeof ArrayBuffer !== 'undefined' && + obj.buffer instanceof ArrayBuffer) || 'length' in obj) { + if (typeof obj.length !== 'number' || isnan(obj.length)) { + return createBuffer(that, 0) + } + return fromArrayLike(that, obj) + } + + if (obj.type === 'Buffer' && isArray(obj.data)) { + return fromArrayLike(that, obj.data) + } + } + + throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') + } + + function checked (length) { + // Note: cannot use `length < kMaxLength()` here because that fails when + // length is NaN (which is otherwise coerced to zero.) + if (length >= kMaxLength()) { + throw new RangeError('Attempt to allocate Buffer larger than maximum ' + + 'size: 0x' + kMaxLength().toString(16) + ' bytes') + } + return length | 0 + } + Buffer.isBuffer = isBuffer; + function internalIsBuffer (b) { + return !!(b != null && b._isBuffer) + } + + Buffer.compare = function compare (a, b) { + if (!internalIsBuffer(a) || !internalIsBuffer(b)) { + throw new TypeError('Arguments must be Buffers') + } + + if (a === b) return 0 + + var x = a.length; + var y = b.length; + + for (var i = 0, len = Math.min(x, y); i < len; ++i) { + if (a[i] !== b[i]) { + x = a[i]; + y = b[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + Buffer.isEncoding = function isEncoding (encoding) { + switch (String(encoding).toLowerCase()) { + case 'hex': + case 'utf8': + case 'utf-8': + case 'ascii': + case 'latin1': + case 'binary': + case 'base64': + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return true + default: + return false + } + }; + + Buffer.concat = function concat (list, length) { + if (!isArray(list)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + + if (list.length === 0) { + return Buffer.alloc(0) + } + + var i; + if (length === undefined) { + length = 0; + for (i = 0; i < list.length; ++i) { + length += list[i].length; + } + } + + var buffer = Buffer.allocUnsafe(length); + var pos = 0; + for (i = 0; i < list.length; ++i) { + var buf = list[i]; + if (!internalIsBuffer(buf)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + buf.copy(buffer, pos); + pos += buf.length; + } + return buffer + }; + + function byteLength (string, encoding) { + if (internalIsBuffer(string)) { + return string.length + } + if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && + (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { + return string.byteLength + } + if (typeof string !== 'string') { + string = '' + string; + } + + var len = string.length; + if (len === 0) return 0 + + // Use a for loop to avoid recursion + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'ascii': + case 'latin1': + case 'binary': + return len + case 'utf8': + case 'utf-8': + case undefined: + return utf8ToBytes(string).length + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return len * 2 + case 'hex': + return len >>> 1 + case 'base64': + return base64ToBytes(string).length + default: + if (loweredCase) return utf8ToBytes(string).length // assume utf8 + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + } + Buffer.byteLength = byteLength; + + function slowToString (encoding, start, end) { + var loweredCase = false; + + // No need to verify that "this.length <= MAX_UINT32" since it's a read-only + // property of a typed array. + + // This behaves neither like String nor Uint8Array in that we set start/end + // to their upper/lower bounds if the value passed is out of range. + // undefined is handled specially as per ECMA-262 6th Edition, + // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. + if (start === undefined || start < 0) { + start = 0; + } + // Return early if start > this.length. Done here to prevent potential uint32 + // coercion fail below. + if (start > this.length) { + return '' + } + + if (end === undefined || end > this.length) { + end = this.length; + } + + if (end <= 0) { + return '' + } + + // Force coersion to uint32. This will also coerce falsey/NaN values to 0. + end >>>= 0; + start >>>= 0; + + if (end <= start) { + return '' + } + + if (!encoding) encoding = 'utf8'; + + while (true) { + switch (encoding) { + case 'hex': + return hexSlice(this, start, end) + + case 'utf8': + case 'utf-8': + return utf8Slice(this, start, end) + + case 'ascii': + return asciiSlice(this, start, end) + + case 'latin1': + case 'binary': + return latin1Slice(this, start, end) + + case 'base64': + return base64Slice(this, start, end) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return utf16leSlice(this, start, end) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = (encoding + '').toLowerCase(); + loweredCase = true; + } + } + } + + // The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect + // Buffer instances. + Buffer.prototype._isBuffer = true; + + function swap (b, n, m) { + var i = b[n]; + b[n] = b[m]; + b[m] = i; + } + + Buffer.prototype.swap16 = function swap16 () { + var len = this.length; + if (len % 2 !== 0) { + throw new RangeError('Buffer size must be a multiple of 16-bits') + } + for (var i = 0; i < len; i += 2) { + swap(this, i, i + 1); + } + return this + }; + + Buffer.prototype.swap32 = function swap32 () { + var len = this.length; + if (len % 4 !== 0) { + throw new RangeError('Buffer size must be a multiple of 32-bits') + } + for (var i = 0; i < len; i += 4) { + swap(this, i, i + 3); + swap(this, i + 1, i + 2); + } + return this + }; + + Buffer.prototype.swap64 = function swap64 () { + var len = this.length; + if (len % 8 !== 0) { + throw new RangeError('Buffer size must be a multiple of 64-bits') + } + for (var i = 0; i < len; i += 8) { + swap(this, i, i + 7); + swap(this, i + 1, i + 6); + swap(this, i + 2, i + 5); + swap(this, i + 3, i + 4); + } + return this + }; + + Buffer.prototype.toString = function toString () { + var length = this.length | 0; + if (length === 0) return '' + if (arguments.length === 0) return utf8Slice(this, 0, length) + return slowToString.apply(this, arguments) + }; + + Buffer.prototype.equals = function equals (b) { + if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') + if (this === b) return true + return Buffer.compare(this, b) === 0 + }; + + Buffer.prototype.inspect = function inspect () { + var str = ''; + var max = INSPECT_MAX_BYTES; + if (this.length > 0) { + str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); + if (this.length > max) str += ' ... '; + } + return '' + }; + + Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { + if (!internalIsBuffer(target)) { + throw new TypeError('Argument must be a Buffer') + } + + if (start === undefined) { + start = 0; + } + if (end === undefined) { + end = target ? target.length : 0; + } + if (thisStart === undefined) { + thisStart = 0; + } + if (thisEnd === undefined) { + thisEnd = this.length; + } + + if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { + throw new RangeError('out of range index') + } + + if (thisStart >= thisEnd && start >= end) { + return 0 + } + if (thisStart >= thisEnd) { + return -1 + } + if (start >= end) { + return 1 + } + + start >>>= 0; + end >>>= 0; + thisStart >>>= 0; + thisEnd >>>= 0; + + if (this === target) return 0 + + var x = thisEnd - thisStart; + var y = end - start; + var len = Math.min(x, y); + + var thisCopy = this.slice(thisStart, thisEnd); + var targetCopy = target.slice(start, end); + + for (var i = 0; i < len; ++i) { + if (thisCopy[i] !== targetCopy[i]) { + x = thisCopy[i]; + y = targetCopy[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + // Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, + // OR the last index of `val` in `buffer` at offset <= `byteOffset`. + // + // Arguments: + // - buffer - a Buffer to search + // - val - a string, Buffer, or number + // - byteOffset - an index into `buffer`; will be clamped to an int32 + // - encoding - an optional encoding, relevant is val is a string + // - dir - true for indexOf, false for lastIndexOf + function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { + // Empty buffer means no match + if (buffer.length === 0) return -1 + + // Normalize byteOffset + if (typeof byteOffset === 'string') { + encoding = byteOffset; + byteOffset = 0; + } else if (byteOffset > 0x7fffffff) { + byteOffset = 0x7fffffff; + } else if (byteOffset < -2147483648) { + byteOffset = -2147483648; + } + byteOffset = +byteOffset; // Coerce to Number. + if (isNaN(byteOffset)) { + // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer + byteOffset = dir ? 0 : (buffer.length - 1); + } + + // Normalize byteOffset: negative offsets start from the end of the buffer + if (byteOffset < 0) byteOffset = buffer.length + byteOffset; + if (byteOffset >= buffer.length) { + if (dir) return -1 + else byteOffset = buffer.length - 1; + } else if (byteOffset < 0) { + if (dir) byteOffset = 0; + else return -1 + } + + // Normalize val + if (typeof val === 'string') { + val = Buffer.from(val, encoding); + } + + // Finally, search either indexOf (if dir is true) or lastIndexOf + if (internalIsBuffer(val)) { + // Special case: looking for empty string/buffer always fails + if (val.length === 0) { + return -1 + } + return arrayIndexOf(buffer, val, byteOffset, encoding, dir) + } else if (typeof val === 'number') { + val = val & 0xFF; // Search for a byte value [0-255] + if (Buffer.TYPED_ARRAY_SUPPORT && + typeof Uint8Array.prototype.indexOf === 'function') { + if (dir) { + return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) + } else { + return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) + } + } + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + } + + throw new TypeError('val must be string, number or Buffer') + } + + function arrayIndexOf (arr, val, byteOffset, encoding, dir) { + var indexSize = 1; + var arrLength = arr.length; + var valLength = val.length; + + if (encoding !== undefined) { + encoding = String(encoding).toLowerCase(); + if (encoding === 'ucs2' || encoding === 'ucs-2' || + encoding === 'utf16le' || encoding === 'utf-16le') { + if (arr.length < 2 || val.length < 2) { + return -1 + } + indexSize = 2; + arrLength /= 2; + valLength /= 2; + byteOffset /= 2; + } + } + + function read (buf, i) { + if (indexSize === 1) { + return buf[i] + } else { + return buf.readUInt16BE(i * indexSize) + } + } + + var i; + if (dir) { + var foundIndex = -1; + for (i = byteOffset; i < arrLength; i++) { + if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { + if (foundIndex === -1) foundIndex = i; + if (i - foundIndex + 1 === valLength) return foundIndex * indexSize + } else { + if (foundIndex !== -1) i -= i - foundIndex; + foundIndex = -1; + } + } + } else { + if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; + for (i = byteOffset; i >= 0; i--) { + var found = true; + for (var j = 0; j < valLength; j++) { + if (read(arr, i + j) !== read(val, j)) { + found = false; + break + } + } + if (found) return i + } + } + + return -1 + } + + Buffer.prototype.includes = function includes (val, byteOffset, encoding) { + return this.indexOf(val, byteOffset, encoding) !== -1 + }; + + Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, true) + }; + + Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, false) + }; + + function hexWrite (buf, string, offset, length) { + offset = Number(offset) || 0; + var remaining = buf.length - offset; + if (!length) { + length = remaining; + } else { + length = Number(length); + if (length > remaining) { + length = remaining; + } + } + + // must be an even number of digits + var strLen = string.length; + if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') + + if (length > strLen / 2) { + length = strLen / 2; + } + for (var i = 0; i < length; ++i) { + var parsed = parseInt(string.substr(i * 2, 2), 16); + if (isNaN(parsed)) return i + buf[offset + i] = parsed; + } + return i + } + + function utf8Write (buf, string, offset, length) { + return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) + } + + function asciiWrite (buf, string, offset, length) { + return blitBuffer(asciiToBytes(string), buf, offset, length) + } + + function latin1Write (buf, string, offset, length) { + return asciiWrite(buf, string, offset, length) + } + + function base64Write (buf, string, offset, length) { + return blitBuffer(base64ToBytes(string), buf, offset, length) + } + + function ucs2Write (buf, string, offset, length) { + return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) + } + + Buffer.prototype.write = function write (string, offset, length, encoding) { + // Buffer#write(string) + if (offset === undefined) { + encoding = 'utf8'; + length = this.length; + offset = 0; + // Buffer#write(string, encoding) + } else if (length === undefined && typeof offset === 'string') { + encoding = offset; + length = this.length; + offset = 0; + // Buffer#write(string, offset[, length][, encoding]) + } else if (isFinite(offset)) { + offset = offset | 0; + if (isFinite(length)) { + length = length | 0; + if (encoding === undefined) encoding = 'utf8'; + } else { + encoding = length; + length = undefined; + } + // legacy write(string, encoding, offset, length) - remove in v0.13 + } else { + throw new Error( + 'Buffer.write(string, encoding, offset[, length]) is no longer supported' + ) + } + + var remaining = this.length - offset; + if (length === undefined || length > remaining) length = remaining; + + if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { + throw new RangeError('Attempt to write outside buffer bounds') + } + + if (!encoding) encoding = 'utf8'; + + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'hex': + return hexWrite(this, string, offset, length) + + case 'utf8': + case 'utf-8': + return utf8Write(this, string, offset, length) + + case 'ascii': + return asciiWrite(this, string, offset, length) + + case 'latin1': + case 'binary': + return latin1Write(this, string, offset, length) + + case 'base64': + // Warning: maxLength not taken into account in base64Write + return base64Write(this, string, offset, length) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return ucs2Write(this, string, offset, length) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + }; + + Buffer.prototype.toJSON = function toJSON () { + return { + type: 'Buffer', + data: Array.prototype.slice.call(this._arr || this, 0) + } + }; + + function base64Slice (buf, start, end) { + if (start === 0 && end === buf.length) { + return fromByteArray(buf) + } else { + return fromByteArray(buf.slice(start, end)) + } + } + + function utf8Slice (buf, start, end) { + end = Math.min(buf.length, end); + var res = []; + + var i = start; + while (i < end) { + var firstByte = buf[i]; + var codePoint = null; + var bytesPerSequence = (firstByte > 0xEF) ? 4 + : (firstByte > 0xDF) ? 3 + : (firstByte > 0xBF) ? 2 + : 1; + + if (i + bytesPerSequence <= end) { + var secondByte, thirdByte, fourthByte, tempCodePoint; + + switch (bytesPerSequence) { + case 1: + if (firstByte < 0x80) { + codePoint = firstByte; + } + break + case 2: + secondByte = buf[i + 1]; + if ((secondByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); + if (tempCodePoint > 0x7F) { + codePoint = tempCodePoint; + } + } + break + case 3: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); + if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { + codePoint = tempCodePoint; + } + } + break + case 4: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + fourthByte = buf[i + 3]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); + if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { + codePoint = tempCodePoint; + } + } + } + } + + if (codePoint === null) { + // we did not generate a valid codePoint so insert a + // replacement char (U+FFFD) and advance only 1 byte + codePoint = 0xFFFD; + bytesPerSequence = 1; + } else if (codePoint > 0xFFFF) { + // encode to utf16 (surrogate pair dance) + codePoint -= 0x10000; + res.push(codePoint >>> 10 & 0x3FF | 0xD800); + codePoint = 0xDC00 | codePoint & 0x3FF; + } + + res.push(codePoint); + i += bytesPerSequence; + } + + return decodeCodePointsArray(res) + } + + // Based on http://stackoverflow.com/a/22747272/680742, the browser with + // the lowest limit is Chrome, with 0x10000 args. + // We go 1 magnitude less, for safety + var MAX_ARGUMENTS_LENGTH = 0x1000; + + function decodeCodePointsArray (codePoints) { + var len = codePoints.length; + if (len <= MAX_ARGUMENTS_LENGTH) { + return String.fromCharCode.apply(String, codePoints) // avoid extra slice() + } + + // Decode in chunks to avoid "call stack size exceeded". + var res = ''; + var i = 0; + while (i < len) { + res += String.fromCharCode.apply( + String, + codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) + ); + } + return res + } + + function asciiSlice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i] & 0x7F); + } + return ret + } + + function latin1Slice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i]); + } + return ret + } + + function hexSlice (buf, start, end) { + var len = buf.length; + + if (!start || start < 0) start = 0; + if (!end || end < 0 || end > len) end = len; + + var out = ''; + for (var i = start; i < end; ++i) { + out += toHex(buf[i]); + } + return out + } + + function utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end); + var res = ''; + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + } + return res + } + + Buffer.prototype.slice = function slice (start, end) { + var len = this.length; + start = ~~start; + end = end === undefined ? len : ~~end; + + if (start < 0) { + start += len; + if (start < 0) start = 0; + } else if (start > len) { + start = len; + } + + if (end < 0) { + end += len; + if (end < 0) end = 0; + } else if (end > len) { + end = len; + } + + if (end < start) end = start; + + var newBuf; + if (Buffer.TYPED_ARRAY_SUPPORT) { + newBuf = this.subarray(start, end); + newBuf.__proto__ = Buffer.prototype; + } else { + var sliceLen = end - start; + newBuf = new Buffer(sliceLen, undefined); + for (var i = 0; i < sliceLen; ++i) { + newBuf[i] = this[i + start]; + } + } + + return newBuf + }; + + /* + * Need to make sure that buffer isn't trying to write out of bounds. + */ + function checkOffset (offset, ext, length) { + if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') + if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') + } + + Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + + return val + }; + + Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + checkOffset(offset, byteLength, this.length); + } + + var val = this[offset + --byteLength]; + var mul = 1; + while (byteLength > 0 && (mul *= 0x100)) { + val += this[offset + --byteLength] * mul; + } + + return val + }; + + Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + return this[offset] + }; + + Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return this[offset] | (this[offset + 1] << 8) + }; + + Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return (this[offset] << 8) | this[offset + 1] + }; + + Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return ((this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16)) + + (this[offset + 3] * 0x1000000) + }; + + Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] * 0x1000000) + + ((this[offset + 1] << 16) | + (this[offset + 2] << 8) | + this[offset + 3]) + }; + + Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var i = byteLength; + var mul = 1; + var val = this[offset + --i]; + while (i > 0 && (mul *= 0x100)) { + val += this[offset + --i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + if (!(this[offset] & 0x80)) return (this[offset]) + return ((0xff - this[offset] + 1) * -1) + }; + + Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset] | (this[offset + 1] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset + 1] | (this[offset] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16) | + (this[offset + 3] << 24) + }; + + Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] << 24) | + (this[offset + 1] << 16) | + (this[offset + 2] << 8) | + (this[offset + 3]) + }; + + Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, true, 23, 4) + }; + + Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, false, 23, 4) + }; + + Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, true, 52, 8) + }; + + Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, false, 52, 8) + }; + + function checkInt (buf, value, offset, ext, max, min) { + if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') + if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') + if (offset + ext > buf.length) throw new RangeError('Index out of range') + } + + Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var mul = 1; + var i = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var i = byteLength - 1; + var mul = 1; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + this[offset] = (value & 0xff); + return offset + 1 + }; + + function objectWriteUInt16 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { + buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> + (littleEndian ? i : 1 - i) * 8; + } + } + + Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + function objectWriteUInt32 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffffffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { + buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; + } + } + + Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset + 3] = (value >>> 24); + this[offset + 2] = (value >>> 16); + this[offset + 1] = (value >>> 8); + this[offset] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = 0; + var mul = 1; + var sub = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = byteLength - 1; + var mul = 1; + var sub = 0; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -128); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + if (value < 0) value = 0xff + value + 1; + this[offset] = (value & 0xff); + return offset + 1 + }; + + Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -32768); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -32768); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -2147483648); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + this[offset + 2] = (value >>> 16); + this[offset + 3] = (value >>> 24); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -2147483648); + if (value < 0) value = 0xffffffff + value + 1; + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + function checkIEEE754 (buf, value, offset, ext, max, min) { + if (offset + ext > buf.length) throw new RangeError('Index out of range') + if (offset < 0) throw new RangeError('Index out of range') + } + + function writeFloat (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 4); + } + write(buf, value, offset, littleEndian, 23, 4); + return offset + 4 + } + + Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { + return writeFloat(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { + return writeFloat(this, value, offset, false, noAssert) + }; + + function writeDouble (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 8); + } + write(buf, value, offset, littleEndian, 52, 8); + return offset + 8 + } + + Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { + return writeDouble(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { + return writeDouble(this, value, offset, false, noAssert) + }; + + // copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) + Buffer.prototype.copy = function copy (target, targetStart, start, end) { + if (!start) start = 0; + if (!end && end !== 0) end = this.length; + if (targetStart >= target.length) targetStart = target.length; + if (!targetStart) targetStart = 0; + if (end > 0 && end < start) end = start; + + // Copy 0 bytes; we're done + if (end === start) return 0 + if (target.length === 0 || this.length === 0) return 0 + + // Fatal error conditions + if (targetStart < 0) { + throw new RangeError('targetStart out of bounds') + } + if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') + if (end < 0) throw new RangeError('sourceEnd out of bounds') + + // Are we oob? + if (end > this.length) end = this.length; + if (target.length - targetStart < end - start) { + end = target.length - targetStart + start; + } + + var len = end - start; + var i; + + if (this === target && start < targetStart && targetStart < end) { + // descending copy from end + for (i = len - 1; i >= 0; --i) { + target[i + targetStart] = this[i + start]; + } + } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { + // ascending copy from start + for (i = 0; i < len; ++i) { + target[i + targetStart] = this[i + start]; + } + } else { + Uint8Array.prototype.set.call( + target, + this.subarray(start, start + len), + targetStart + ); + } + + return len + }; + + // Usage: + // buffer.fill(number[, offset[, end]]) + // buffer.fill(buffer[, offset[, end]]) + // buffer.fill(string[, offset[, end]][, encoding]) + Buffer.prototype.fill = function fill (val, start, end, encoding) { + // Handle string cases: + if (typeof val === 'string') { + if (typeof start === 'string') { + encoding = start; + start = 0; + end = this.length; + } else if (typeof end === 'string') { + encoding = end; + end = this.length; + } + if (val.length === 1) { + var code = val.charCodeAt(0); + if (code < 256) { + val = code; + } + } + if (encoding !== undefined && typeof encoding !== 'string') { + throw new TypeError('encoding must be a string') + } + if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { + throw new TypeError('Unknown encoding: ' + encoding) + } + } else if (typeof val === 'number') { + val = val & 255; + } + + // Invalid ranges are not set to a default, so can range check early. + if (start < 0 || this.length < start || this.length < end) { + throw new RangeError('Out of range index') + } + + if (end <= start) { + return this + } + + start = start >>> 0; + end = end === undefined ? this.length : end >>> 0; + + if (!val) val = 0; + + var i; + if (typeof val === 'number') { + for (i = start; i < end; ++i) { + this[i] = val; + } + } else { + var bytes = internalIsBuffer(val) + ? val + : utf8ToBytes(new Buffer(val, encoding).toString()); + var len = bytes.length; + for (i = 0; i < end - start; ++i) { + this[i + start] = bytes[i % len]; + } + } + + return this + }; + + // HELPER FUNCTIONS + // ================ + + var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; + + function base64clean (str) { + // Node strips out invalid characters like \n and \t from the string, base64-js does not + str = stringtrim(str).replace(INVALID_BASE64_RE, ''); + // Node converts strings with length < 2 to '' + if (str.length < 2) return '' + // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not + while (str.length % 4 !== 0) { + str = str + '='; + } + return str + } + + function stringtrim (str) { + if (str.trim) return str.trim() + return str.replace(/^\s+|\s+$/g, '') + } + + function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) + } + + function utf8ToBytes (string, units) { + units = units || Infinity; + var codePoint; + var length = string.length; + var leadSurrogate = null; + var bytes = []; + + for (var i = 0; i < length; ++i) { + codePoint = string.charCodeAt(i); + + // is surrogate component + if (codePoint > 0xD7FF && codePoint < 0xE000) { + // last char was a lead + if (!leadSurrogate) { + // no lead yet + if (codePoint > 0xDBFF) { + // unexpected trail + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } else if (i + 1 === length) { + // unpaired lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } + + // valid lead + leadSurrogate = codePoint; + + continue + } + + // 2 leads in a row + if (codePoint < 0xDC00) { + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + leadSurrogate = codePoint; + continue + } + + // valid surrogate pair + codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; + } else if (leadSurrogate) { + // valid bmp char, but last char was a lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + } + + leadSurrogate = null; + + // encode utf8 + if (codePoint < 0x80) { + if ((units -= 1) < 0) break + bytes.push(codePoint); + } else if (codePoint < 0x800) { + if ((units -= 2) < 0) break + bytes.push( + codePoint >> 0x6 | 0xC0, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x10000) { + if ((units -= 3) < 0) break + bytes.push( + codePoint >> 0xC | 0xE0, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x110000) { + if ((units -= 4) < 0) break + bytes.push( + codePoint >> 0x12 | 0xF0, + codePoint >> 0xC & 0x3F | 0x80, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else { + throw new Error('Invalid code point') + } + } + + return bytes + } + + function asciiToBytes (str) { + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + // Node's code seems to be doing this and not & 0x7F.. + byteArray.push(str.charCodeAt(i) & 0xFF); + } + return byteArray + } + + function utf16leToBytes (str, units) { + var c, hi, lo; + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + if ((units -= 2) < 0) break + + c = str.charCodeAt(i); + hi = c >> 8; + lo = c % 256; + byteArray.push(lo); + byteArray.push(hi); + } + + return byteArray + } + + + function base64ToBytes (str) { + return toByteArray(base64clean(str)) + } + + function blitBuffer (src, dst, offset, length) { + for (var i = 0; i < length; ++i) { + if ((i + offset >= dst.length) || (i >= src.length)) break + dst[i + offset] = src[i]; + } + return i + } + + function isnan (val) { + return val !== val // eslint-disable-line no-self-compare + } + + + // the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence + // The _isBuffer check is for Safari 5-7 support, because it's missing + // Object.prototype.constructor. Remove this eventually + function isBuffer(obj) { + return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) + } + + function isFastBuffer (obj) { + return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) + } + + // For Node v0.10 support. Remove this eventually. + function isSlowBuffer (obj) { + return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) + } + + class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if (Array.isArray(message)) message = message.join(" ").trim(); + super(message); + if (Error.captureStackTrace !== undefined) { + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for (const context of contexts) { + for (const key in context) { + const value = context[key]; + this[key] = isBuffer(value) + ? value.toString(options.encoding) + : value == null + ? value + : JSON.parse(JSON.stringify(value)); + } + } + } + } + + const is_object = function (obj) { + return typeof obj === "object" && obj !== null && !Array.isArray(obj); + }; + + const normalize_columns_array = function (columns) { + const normalizedColumns = []; + for (let i = 0, l = columns.length; i < l; i++) { + const column = columns[i]; + if (column === undefined || column === null || column === false) { + normalizedColumns[i] = { disabled: true }; + } else if (typeof column === "string" || typeof column === "number") { + normalizedColumns[i] = { name: `${column}` }; + } else if (is_object(column)) { + if (typeof column.name !== "string") { + throw new CsvError("CSV_OPTION_COLUMNS_MISSING_NAME", [ + "Option columns missing name:", + `property "name" is required at position ${i}`, + "when column is an object literal", + ]); + } + normalizedColumns[i] = column; + } else { + throw new CsvError("CSV_INVALID_COLUMN_DEFINITION", [ + "Invalid column definition:", + "expect a string or a literal object,", + `got ${JSON.stringify(column)} at position ${i}`, + ]); + } + } + return normalizedColumns; + }; + + class ResizeableBuffer { + constructor(size = 100) { + this.size = size; + this.length = 0; + this.buf = Buffer.allocUnsafe(size); + } + prepend(val) { + if (isBuffer(val)) { + const length = this.length + val.length; + if (length >= this.size) { + this.resize(); + if (length >= this.size) { + throw Error("INVALID_BUFFER_STATE"); + } + } + const buf = this.buf; + this.buf = Buffer.allocUnsafe(this.size); + val.copy(this.buf, 0); + buf.copy(this.buf, val.length); + this.length += val.length; + } else { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + const buf = this.clone(); + this.buf[0] = val; + buf.copy(this.buf, 1, 0, length); + } + } + append(val) { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + this.buf[length] = val; + } + clone() { + return Buffer.from(this.buf.slice(0, this.length)); + } + resize() { + const length = this.length; + this.size = this.size * 2; + const buf = Buffer.allocUnsafe(this.size); + this.buf.copy(buf, 0, 0, length); + this.buf = buf; + } + toString(encoding) { + if (encoding) { + return this.buf.toString(encoding, 0, this.length); + } else { + return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)); + } + } + toJSON() { + return this.toString("utf8"); + } + reset() { + this.length = 0; + } + } + + const init_state = function (options) { + // ECMAScript WhiteSpace + LineTerminator codepoints, encoded under + // `options.encoding`. Aligns trimming with `String.prototype.trim()`. + // https://tc39.es/ecma262/#sec-white-space + // https://tc39.es/ecma262/#sec-line-terminators + // + // Codepoints unrepresentable in the target encoding are dropped: Node's + // Buffer substitutes them with `?` (0x3F), and including those would cause + // literal `?` bytes in the input to be trimmed under `latin1`/`ascii`. + const timchars = [ + // Basic Latin + 0x0020, // [Space](https://www.fileformat.info/info/unicode/char/0020/index.htm) + 0x0009, // [CHARACTER TABULATION (HT)](https://www.fileformat.info/info/unicode/char/0009/index.htm) + 0x000a, // [LINE FEED (LF)](https://www.fileformat.info/info/unicode/char/000a/index.htm) + 0x000d, // [CARRIAGE RETURN (CR)](https://www.fileformat.info/info/unicode/char/000d/index.htm) + 0x000c, // [FORM FEED (FF)](https://www.fileformat.info/info/unicode/char/000c/index.htm) + 0x000b, // [LINE TABULATION (VT)](https://www.fileformat.info/info/unicode/char/000b/index.htm) + // Latin-1 Supplement + 0x00a0, // [NO-BREAK SPACE (NBSP)](https://www.fileformat.info/info/unicode/char/00a0/index.htm) + // Ogham + 0x1680, // [OGHAM SPACE MARK](https://www.fileformat.info/info/unicode/char/1680/index.htm) + // General Punctuation + 0x2000, // [EN QUAD](https://www.fileformat.info/info/unicode/char/2000/index.htm) + 0x2001, // [EM QUAD](https://www.fileformat.info/info/unicode/char/2001/index.htm) + 0x2002, // [EN SPACE](https://www.fileformat.info/info/unicode/char/2002/index.htm) + 0x2003, // [EM SPACE](https://www.fileformat.info/info/unicode/char/2003/index.htm) + 0x2004, // [THREE-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2004/index.htm) + 0x2005, // [FOUR-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2005/index.htm) + 0x2006, // [SIX-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2006/index.htm) + 0x2007, // [FIGURE SPACE](https://www.fileformat.info/info/unicode/char/2007/index.htm) + 0x2008, // [PUNCTUATION SPACE](https://www.fileformat.info/info/unicode/char/2008/index.htm) + 0x2009, // [THIN SPACE](https://www.fileformat.info/info/unicode/char/2009/index.htm) + 0x200a, // [HAIR SPACE](https://www.fileformat.info/info/unicode/char/200a/index.htm) + 0x2028, // [LINE SEPARATOR](https://www.fileformat.info/info/unicode/char/2028/index.htm) + 0x2029, // [PARAGRAPH SEPARATOR](https://www.fileformat.info/info/unicode/char/2029/index.htm) + 0x202f, // [NARROW NO-BREAK SPACE (NNBSP)](https://www.fileformat.info/info/unicode/char/202f/index.htm) + 0x205f, // [MEDIUM MATHEMATICAL SPACE (MMSP)](https://www.fileformat.info/info/unicode/char/205f/index.htm) + 0x3000, // [IDEOGRAPHIC SPACE](https://www.fileformat.info/info/unicode/char/3000/index.htm) + 0xfeff, // [ZERO WIDTH NO-BREAK SPACE (BOM)](https://www.fileformat.info/info/unicode/char/feff/index.htm) + ].reduce((acc, codepoint) => { + const encoded = Buffer.from( + String.fromCharCode(codepoint), + options.encoding, + ); + if (codepoint !== 0x3f && encoded.length === 1 && encoded[0] === 0x3f) { + return acc; + } + acc.push(encoded); + return acc; + }, []); + // First-byte lookup table for `__isCharTrimable`. Non-whitespace bytes + // (the common case) bail out in O(1) without scanning every timchar. + const timcharFirstBytes = new Uint8Array(256); + for (const t of timchars) timcharFirstBytes[t[0]] = 1; + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + delimiterBufPrevious: undefined, + delimiterDiscovered: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: + isBuffer(options.escape) && + isBuffer(options.quote) && + Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) + ? options.columns.length + : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...(options.delimiter + ? options.delimiter.map((delimiter) => delimiter.length) + : []), + // Auto discovery of delimiter is limited to 1 character + options.delimiter_auto ? 1 : 0, + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + // Skip if the remaining buffer can be a multi-byte trim character + ...timchars.map((t) => t.length), + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: + options.record_delimiter.length === 0 + ? 0 + : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [ + Buffer.from(" ", options.encoding)[0], + Buffer.from("\t", options.encoding)[0], + ], + wasQuoting: false, + wasRowDelimiter: false, + timchars: timchars, + timcharFirstBytes: timcharFirstBytes, + }; + }; + + const underscore = function (str) { + return str.replace(/([A-Z])/g, function (_, match) { + return "_" + match.toLowerCase(); + }); + }; + + const normalize_options = function (opts) { + const options = {}; + // Merge with user options + for (const opt in opts) { + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if (options.encoding === undefined || options.encoding === true) { + options.encoding = "utf8"; + } else if (options.encoding === null || options.encoding === false) { + options.encoding = null; + } else if ( + typeof options.encoding !== "string" && + options.encoding !== null + ) { + throw new CsvError( + "CSV_INVALID_OPTION_ENCODING", + [ + "Invalid option encoding:", + "encoding must be a string or null to return a buffer,", + `got ${JSON.stringify(options.encoding)}`, + ], + options, + ); + } + // Normalize option `bom` + if ( + options.bom === undefined || + options.bom === null || + options.bom === false + ) { + options.bom = false; + } else if (options.bom !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_BOM", + [ + "Invalid option bom:", + "bom must be true,", + `got ${JSON.stringify(options.bom)}`, + ], + options, + ); + } + // Normalize option `cast` + options.cast_function = null; + if ( + options.cast === undefined || + options.cast === null || + options.cast === false || + options.cast === "" + ) { + options.cast = undefined; + } else if (typeof options.cast === "function") { + options.cast_function = options.cast; + options.cast = true; + } else if (options.cast !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_CAST", + [ + "Invalid option cast:", + "cast must be true or a function,", + `got ${JSON.stringify(options.cast)}`, + ], + options, + ); + } + // Normalize option `cast_date` + if ( + options.cast_date === undefined || + options.cast_date === null || + options.cast_date === false || + options.cast_date === "" + ) { + options.cast_date = false; + } else if (options.cast_date === true) { + options.cast_date = function (value) { + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + } else if (typeof options.cast_date !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_CAST_DATE", + [ + "Invalid option cast_date:", + "cast_date must be true or a function,", + `got ${JSON.stringify(options.cast_date)}`, + ], + options, + ); + } + // Normalize option `columns` + options.cast_first_line_to_header = undefined; + if (options.columns === true) { + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + } else if (typeof options.columns === "function") { + options.cast_first_line_to_header = options.columns; + options.columns = true; + } else if (Array.isArray(options.columns)) { + options.columns = normalize_columns_array(options.columns); + } else if ( + options.columns === undefined || + options.columns === null || + options.columns === false + ) { + options.columns = false; + } else { + throw new CsvError( + "CSV_INVALID_OPTION_COLUMNS", + [ + "Invalid option columns:", + "expect an array, a function or true,", + `got ${JSON.stringify(options.columns)}`, + ], + options, + ); + } + // Normalize option `group_columns_by_name` + if ( + options.group_columns_by_name === undefined || + options.group_columns_by_name === null || + options.group_columns_by_name === false + ) { + options.group_columns_by_name = false; + } else if (options.group_columns_by_name !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "expect a boolean,", + `got ${JSON.stringify(options.group_columns_by_name)}`, + ], + options, + ); + } else if (options.columns === false) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "the `columns` mode must be activated.", + ], + options, + ); + } + // Normalize option `comment` + if ( + options.comment === undefined || + options.comment === null || + options.comment === false || + options.comment === "" + ) { + options.comment = null; + } else { + if (typeof options.comment === "string") { + options.comment = Buffer.from(options.comment, options.encoding); + } + if (!isBuffer(options.comment)) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment:", + "comment must be a buffer or a string,", + `got ${JSON.stringify(options.comment)}`, + ], + options, + ); + } + } + // Normalize option `comment_no_infix` + if ( + options.comment_no_infix === undefined || + options.comment_no_infix === null || + options.comment_no_infix === false + ) { + options.comment_no_infix = false; + } else if (options.comment_no_infix !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment_no_infix:", + "value must be a boolean,", + `got ${JSON.stringify(options.comment_no_infix)}`, + ], + options, + ); + } + // Normalize option `delimiter_auto` + if ( + options.delimiter_auto === undefined || + options.delimiter_auto === null || + options.delimiter_auto === false + ) { + options.delimiter_auto = false; + } else if (options.delimiter_auto === true) { + options.delimiter_auto = {}; + } else if (!is_object(options.delimiter_auto)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "delimiter_auto must be a boolean or a configuration object,", + `got ${JSON.stringify(options.delimiter_auto)}`, + ], + options, + ); + } + if (options.delimiter_auto) { + if (options.delimiter_auto.preferred === undefined) + options.delimiter_auto.preferred = { + [",".charCodeAt(0)]: 1.8, + ["\t".charCodeAt(0)]: 1.8, + [";".charCodeAt(0)]: 1.6, + [" ".charCodeAt(0)]: 1.6, + [":".charCodeAt(0)]: 1.5, + [".".charCodeAt(0)]: 1.4, + ["/".charCodeAt(0)]: 1.4, + }; + else if (!is_object(options.delimiter_auto.preferred)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "preferred must be an object,", + `got ${JSON.stringify(options.delimiter_auto.preferred)}`, + ], + options, + ); + } + if (options.delimiter_auto.score === undefined) + options.delimiter_auto.score = (info, options) => { + return ( + (info.total - info.std) * (options.preferred[info.char_code] || 1) + ); + }; + else if (typeof options.delimiter_auto.score !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "score must be a function,", + `got ${JSON.stringify(options.delimiter_auto.score)}`, + ], + options, + ); + } + if (options.delimiter_auto.size === undefined) + options.delimiter_auto.size = 2048; + else if (typeof options.delimiter_auto.size !== "number") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "size must be a number,", + `got ${JSON.stringify(options.delimiter_auto.size)}`, + ], + options, + ); + } + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if (options.delimiter_auto !== false) { + options.delimiter = []; + } + if (!Array.isArray(options.delimiter)) { + if ( + options.delimiter === undefined || + options.delimiter === null || + options.delimiter === false + ) { + options.delimiter = Buffer.from(",", options.encoding); + } + options.delimiter = [options.delimiter]; + } + options.delimiter = options.delimiter.map(function (delimiter) { + if (typeof delimiter === "string") { + delimiter = Buffer.from(delimiter, options.encoding); + } + if (!isBuffer(delimiter) || delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER", + [ + "Invalid option delimiter:", + "delimiter must be a non empty string or buffer or array of string|buffer,", + `got ${delimiter_json}`, + ], + options, + ); + } + return delimiter; + }); + // Normalize option `escape` + if (options.escape === undefined || options.escape === true) { + options.escape = Buffer.from('"', options.encoding); + } else if (typeof options.escape === "string") { + options.escape = Buffer.from(options.escape, options.encoding); + } else if (options.escape === null || options.escape === false) { + options.escape = null; + } + if (options.escape !== null) { + if (!isBuffer(options.escape)) { + throw new Error( + `Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`, + ); + } + } + // Normalize option `from` + if (options.from === undefined || options.from === null) { + options.from = 1; + } else { + if (typeof options.from === "string" && /\d+/.test(options.from)) { + options.from = parseInt(options.from); + } + if (Number.isInteger(options.from)) { + if (options.from < 0) { + throw new Error( + `Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`, + ); + } + } + // Normalize option `from_line` + if (options.from_line === undefined || options.from_line === null) { + options.from_line = 1; + } else { + if ( + typeof options.from_line === "string" && + /\d+/.test(options.from_line) + ) { + options.from_line = parseInt(options.from_line); + } + if (Number.isInteger(options.from_line)) { + if (options.from_line <= 0) { + throw new Error( + `Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`, + ); + } + } + // Normalize options `ignore_last_delimiters` + if ( + options.ignore_last_delimiters === undefined || + options.ignore_last_delimiters === null + ) { + options.ignore_last_delimiters = false; + } else if (typeof options.ignore_last_delimiters === "number") { + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if (options.ignore_last_delimiters === 0) { + options.ignore_last_delimiters = false; + } + } else if (typeof options.ignore_last_delimiters !== "boolean") { + throw new CsvError( + "CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS", + [ + "Invalid option `ignore_last_delimiters`:", + "the value must be a boolean value or an integer,", + `got ${JSON.stringify(options.ignore_last_delimiters)}`, + ], + options, + ); + } + if (options.ignore_last_delimiters === true && options.columns === false) { + throw new CsvError( + "CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS", + [ + "The option `ignore_last_delimiters`", + "requires the activation of the `columns` option", + ], + options, + ); + } + // Normalize option `info` + if ( + options.info === undefined || + options.info === null || + options.info === false + ) { + options.info = false; + } else if (options.info !== true) { + throw new Error( + `Invalid Option: info must be true, got ${JSON.stringify(options.info)}`, + ); + } + // Normalize option `max_record_size` + if ( + options.max_record_size === undefined || + options.max_record_size === null || + options.max_record_size === false + ) { + options.max_record_size = 0; + } else if ( + Number.isInteger(options.max_record_size) && + options.max_record_size >= 0 + ) ; else if ( + typeof options.max_record_size === "string" && + /\d+/.test(options.max_record_size) + ) { + options.max_record_size = parseInt(options.max_record_size); + } else { + throw new Error( + `Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`, + ); + } + // Normalize option `objname` + if ( + options.objname === undefined || + options.objname === null || + options.objname === false + ) { + options.objname = undefined; + } else if (isBuffer(options.objname)) { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if (options.encoding === null) ; else { + options.objname = options.objname.toString(options.encoding); + } + } else if (typeof options.objname === "string") { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + } else if (typeof options.objname === "number") ; else { + throw new Error( + `Invalid Option: objname must be a string or a buffer, got ${options.objname}`, + ); + } + if (options.objname !== undefined) { + if (typeof options.objname === "number") { + if (options.columns !== false) { + throw Error( + "Invalid Option: objname index cannot be combined with columns or be defined as a field", + ); + } + } else { + // A string or a buffer + if (options.columns === false) { + throw Error( + "Invalid Option: objname field must be combined with columns or be defined as an index", + ); + } + } + } + // Normalize option `on_record` + if (options.on_record === undefined || options.on_record === null) { + options.on_record = undefined; + } else if (typeof options.on_record !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_ON_RECORD", + [ + "Invalid option `on_record`:", + "expect a function,", + `got ${JSON.stringify(options.on_record)}`, + ], + options, + ); + } + // Normalize option `on_skip` + // options.on_skip ??= (err, chunk) => { + // this.emit('skip', err, chunk); + // }; + if ( + options.on_skip !== undefined && + options.on_skip !== null && + typeof options.on_skip !== "function" + ) { + throw new Error( + `Invalid Option: on_skip must be a function, got ${JSON.stringify(options.on_skip)}`, + ); + } + // Normalize option `quote` + if ( + options.quote === null || + options.quote === false || + options.quote === "" + ) { + options.quote = null; + } else { + if (options.quote === undefined || options.quote === true) { + options.quote = Buffer.from('"', options.encoding); + } else if (typeof options.quote === "string") { + options.quote = Buffer.from(options.quote, options.encoding); + } + if (!isBuffer(options.quote)) { + throw new Error( + `Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`, + ); + } + } + // Normalize option `raw` + if ( + options.raw === undefined || + options.raw === null || + options.raw === false + ) { + options.raw = false; + } else if (options.raw !== true) { + throw new Error( + `Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`, + ); + } + // Normalize option `record_delimiter` + if (options.record_delimiter === undefined) { + options.record_delimiter = []; + } else if ( + typeof options.record_delimiter === "string" || + isBuffer(options.record_delimiter) + ) { + if (options.record_delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = [options.record_delimiter]; + } else if (!Array.isArray(options.record_delimiter)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = options.record_delimiter.map(function (rd, i) { + if (typeof rd !== "string" && !isBuffer(rd)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } else if (rd.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } + if (typeof rd === "string") { + rd = Buffer.from(rd, options.encoding); + } + return rd; + }); + // Normalize option `relax_column_count` + if (typeof options.relax_column_count === "boolean") ; else if ( + options.relax_column_count === undefined || + options.relax_column_count === null + ) { + options.relax_column_count = false; + } else { + throw new Error( + `Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`, + ); + } + if (typeof options.relax_column_count_less === "boolean") ; else if ( + options.relax_column_count_less === undefined || + options.relax_column_count_less === null + ) { + options.relax_column_count_less = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`, + ); + } + if (typeof options.relax_column_count_more === "boolean") ; else if ( + options.relax_column_count_more === undefined || + options.relax_column_count_more === null + ) { + options.relax_column_count_more = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`, + ); + } + // Normalize option `relax_quotes` + if (typeof options.relax_quotes === "boolean") ; else if ( + options.relax_quotes === undefined || + options.relax_quotes === null + ) { + options.relax_quotes = false; + } else { + throw new Error( + `Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`, + ); + } + // Normalize option `skip_empty_lines` + if (typeof options.skip_empty_lines === "boolean") ; else if ( + options.skip_empty_lines === undefined || + options.skip_empty_lines === null + ) { + options.skip_empty_lines = false; + } else { + throw new Error( + `Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`, + ); + } + // Normalize option `skip_records_with_empty_values` + if (typeof options.skip_records_with_empty_values === "boolean") ; else if ( + options.skip_records_with_empty_values === undefined || + options.skip_records_with_empty_values === null + ) { + options.skip_records_with_empty_values = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`, + ); + } + // Normalize option `skip_records_with_error` + if (typeof options.skip_records_with_error === "boolean") ; else if ( + options.skip_records_with_error === undefined || + options.skip_records_with_error === null + ) { + options.skip_records_with_error = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`, + ); + } + // Normalize option `rtrim` + if ( + options.rtrim === undefined || + options.rtrim === null || + options.rtrim === false + ) { + options.rtrim = false; + } else if (options.rtrim !== true) { + throw new Error( + `Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`, + ); + } + // Normalize option `ltrim` + if ( + options.ltrim === undefined || + options.ltrim === null || + options.ltrim === false + ) { + options.ltrim = false; + } else if (options.ltrim !== true) { + throw new Error( + `Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`, + ); + } + // Normalize option `trim` + if ( + options.trim === undefined || + options.trim === null || + options.trim === false + ) { + options.trim = false; + } else if (options.trim !== true) { + throw new Error( + `Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`, + ); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if (options.trim === true && opts.ltrim !== false) { + options.ltrim = true; + } else if (options.ltrim !== true) { + options.ltrim = false; + } + if (options.trim === true && opts.rtrim !== false) { + options.rtrim = true; + } else if (options.rtrim !== true) { + options.rtrim = false; + } + // Normalize option `to` + if (options.to === undefined || options.to === null) { + options.to = -1; + } else if (options.to !== -1) { + if (typeof options.to === "string" && /\d+/.test(options.to)) { + options.to = parseInt(options.to); + } + if (Number.isInteger(options.to)) { + if (options.to <= 0) { + throw new Error( + `Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`, + ); + } + } + // Normalize option `to_line` + if (options.to_line === undefined || options.to_line === null) { + options.to_line = -1; + } else if (options.to_line !== -1) { + if (typeof options.to_line === "string" && /\d+/.test(options.to_line)) { + options.to_line = parseInt(options.to_line); + } + if (Number.isInteger(options.to_line)) { + if (options.to_line <= 0) { + throw new Error( + `Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`, + ); + } + } + return options; + }; + + // Discussed in [issue #400](https://github.com/adaltas/node-csv/issues/400) + // See https://github.com/python/cpython/blob/ea1b1c579f600cc85d145c60862b2e6b98701b24/Lib/csv.py#L349 + const delimiter_discover = function (records, options) { + // Normalize the configuration + if (!options) { + ({ delimiter_auto: options } = normalize_options({ delimiter_auto: true })); + } + // Convert String to Buffer + if (typeof records === "string") { + records = Buffer.from(records); + } + // Convert Buffer to an array of records + if (isBuffer(records)) { + records = ((data) => { + const records = []; + const parser = transform({ delimiter: [] }); + const push = (record) => records.push(record); + const close = () => {}; + const error = parser.parse(data, true, push, close); + if (error !== undefined) throw error; + return records; + })(records); + } + // Info array initialization, 127 entries, one per char code + const info = Array(127) + .fill() + .map(() => ({ lines: [] })); + // Traverse each records, count occurences per char code + records.map(([record], line) => { + for (let i = 0, l = record.length; i < l; i++) { + // Count the character frequency + const code = record.charCodeAt(i); + if (info[code].lines[line] === undefined) info[code].lines[line] = 0; + info[code].lines[line]++; + } + }); + // Traverse each char code, compute the score + info.map((info, i) => { + info.char_code = i; + info.std = std(info.lines); + info.total = info.lines.reduce((acc, val) => acc + val, 0); + info.preferred = !!options.preferred[i]; + info.score = options.score(info, options); + }); + // Extract the dominant character + const result = info.reduce( + (acc, info) => (acc.score > info.score ? acc : info), + {}, + ); + return String.fromCharCode(result.char_code); + }; + + const std = function (array) { + const n = array.length; + if (n === 0) return 0; + const mean = array.reduce((a, b) => a + b) / n; + return Math.sqrt( + array.map((x) => Math.pow(x - mean, 2)).reduce((a, b) => a + b) / n, + ); + }; + + const isRecordEmpty = function (record) { + return record.every( + (field) => + field == null || (field.toString && field.toString().trim() === ""), + ); + }; + + const cr = 13; // `\r`, carriage return, 0x0D in hexadécimal, 13 in decimal + const nl = 10; // `\n`, newline, 0x0A in hexadecimal, 10 in decimal + + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + utf8: Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + utf16le: Buffer.from([255, 254]), + }; + + const transform = function (original_options = {}) { + const info = { + bytes: 0, + bytes_records: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0, + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function (i, bufLen, end) { + if (end) return false; + const { encoding, escape, quote } = this.options; + const { quoting, needMoreDataSize, recordDelimiterMaxLength } = + this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + // If "record_delimiter" is yet to be discovered: + // 1. It is equals to `[]` and "recordDelimiterMaxLength" equals `0` + // 2. We set the length to windows line ending in the current encoding + // Note, that encoding is known from user or bom discovery at that point + // recordDelimiterMaxLength, + recordDelimiterMaxLength === 0 + ? Buffer.from("\r\n", encoding).length + : recordDelimiterMaxLength, + // Skip if remaining buffer can be an escaped quote + quoting ? (escape === null ? 0 : escape.length) + quote.length : 0, + // Skip if remaining buffer can be record delimiter following the closing quote + quoting ? quote.length + recordDelimiterMaxLength : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function (nextBuf, end, push, close) { + const { + bom, + comment_no_infix, + delimiter_auto, + encoding, + from_line, + ltrim, + max_record_size, + raw, + relax_quotes, + rtrim, + skip_empty_lines, + to, + to_line, + } = this.options; + let { comment, escape, quote, record_delimiter } = this.options; + const { + bomSkipped, + delimiterDiscovered, + delimiterBufPrevious, + rawBuffer, + escapeIsQuote, + } = this.state; + // Automatic delimiter discovery + if (!delimiterDiscovered && delimiter_auto) { + let delimiterBuf; + if (delimiterBufPrevious === undefined) { + delimiterBuf = nextBuf; + } else if ( + delimiterBufPrevious !== undefined && + nextBuf === undefined + ) { + delimiterBuf = delimiterBufPrevious; + } else { + delimiterBuf = Buffer.concat([delimiterBufPrevious, nextBuf]); + } + // Ensure that nextBuf is not concatenated a second time during buffer reconciliation + nextBuf = undefined; + // this.delimiterBufPrevious = delimiterBuf; + if (end || delimiterBuf.length > delimiter_auto.size) { + this.options.delimiter = [ + Buffer.from( + delimiter_discover(delimiterBuf, this.options.delimiter_auto), + ), + ]; + this.state.previousBuf = delimiterBuf; + this.state.delimiterBufPrevious = undefined; + this.state.delimiterDiscovered = true; + } else { + this.state.delimiterBufPrevious = delimiterBuf; + return; + } + } + // Previous buffers reconciliation + const { previousBuf } = this.state; + let buf; + if (previousBuf === undefined) { + if (nextBuf === undefined) { + // Handle empty string + close(); + return; + } else { + buf = nextBuf; + } + } else if (previousBuf !== undefined && nextBuf === undefined) { + buf = previousBuf; + } else { + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if (bomSkipped === false) { + if (bom === false) { + this.state.bomSkipped = true; + } else if (buf.length < 3) { + // No enough data + if (end === false) { + // Wait for more data + this.state.previousBuf = buf; + return; + } + } else { + for (const encoding in boms) { + if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) { + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + const options = normalize_options({ + ...this.original_options, + encoding: encoding, + }); + // Properties are merged with the existing options instance + for (const key in options) { + this.options[key] = options[key]; + } + // Options will re-evaluate the Buffer with the new encoding + ({ comment, escape, quote } = this.options); + break; + } + } + this.state.bomSkipped = true; + } + } + const bufLen = buf.length; + let pos; + for (pos = 0; pos < bufLen; pos++) { + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if (this.__needMoreData(pos, bufLen, end)) { + break; + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if (to_line !== -1 && this.info.lines > to_line) { + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if (this.state.quoting === false && record_delimiter.length === 0) { + const record_delimiterCount = this.__autoDiscoverRecordDelimiter( + buf, + pos, + ); + if (record_delimiterCount) { + record_delimiter = this.options.record_delimiter; + } + } + const chr = buf[pos]; + if (raw === true) { + rawBuffer.append(chr); + } + if ( + (chr === cr || chr === nl) && + this.state.wasRowDelimiter === false + ) { + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if (this.state.escaping === true) { + this.state.escaping = false; + } else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if ( + escape !== null && + this.state.quoting === true && + this.__isEscape(buf, pos, chr) && + pos + escape.length < bufLen + ) { + if (escapeIsQuote) { + if (this.__isQuote(buf, pos + escape.length)) { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } else { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if (this.state.commenting === false && this.__isQuote(buf, pos)) { + if (this.state.quoting === true) { + const nextChr = buf[pos + quote.length]; + const isNextChrTrimable = + rtrim && this.__isCharTrimable(buf, pos + quote.length); + const isNextChrComment = + comment !== null && + this.__compareBytes(comment, buf, pos + quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter( + buf, + pos + quote.length, + nextChr, + ); + const isNextChrRecordDelimiter = + record_delimiter.length === 0 + ? this.__autoDiscoverRecordDelimiter(buf, pos + quote.length) + : this.__isRecordDelimiter(nextChr, buf, pos + quote.length); + // Escape a quote + // Treat next char as a regular character + if ( + escape !== null && + this.__isEscape(buf, pos, chr) && + this.__isQuote(buf, pos + escape.length) + ) { + pos += escape.length - 1; + } else if ( + !nextChr || + isNextChrDelimiter || + isNextChrRecordDelimiter || + isNextChrComment || + isNextChrTrimable + ) { + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + } else if (relax_quotes === false) { + const err = this.__error( + new CsvError( + "CSV_INVALID_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + "instead of delimiter, record delimiter, trimable character", + "(if activated) or comment", + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; + } + } else { + if (this.state.field.length !== 0) { + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if (relax_quotes === false) { + const info = this.__infoField(); + const bom = Object.keys(boms) + .map((b) => + boms[b].equals(this.state.field.toString()) ? b : false, + ) + .filter(Boolean)[0]; + const err = this.__error( + new CsvError( + "INVALID_OPENING_QUOTE", + [ + "Invalid Opening Quote:", + `a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`, + bom ? `(${bom} bom)` : undefined, + ], + this.options, + info, + { + field: this.state.field, + }, + ), + ); + if (err !== undefined) return err; + } + } else { + this.state.quoting = true; + pos += quote.length - 1; + continue; + } + } + } + if (this.state.quoting === false) { + const recordDelimiterLength = this.__isRecordDelimiter( + chr, + buf, + pos, + ); + if (recordDelimiterLength !== 0) { + // Do not emit comments which take a full line + const skipCommentLine = + this.state.commenting && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0; + if (skipCommentLine) { + this.info.comment_lines++; + // Skip full comment line + } else { + // Activate records emission if above from_line + if ( + this.state.enabled === false && + this.info.lines + + (this.state.wasRowDelimiter === true ? 1 : 0) >= + from_line + ) { + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if ( + skip_empty_lines === true && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0 + ) { + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + this.info.bytes = + this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + if (to !== -1 && this.info.records >= to) { + this.state.stop = true; + close(); + return; + } + } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if (this.state.commenting) { + continue; + } + if ( + comment !== null && + (comment_no_infix === false || + (this.state.record.length === 0 && + this.state.field.length === 0)) + ) { + const commentCount = this.__compareBytes(comment, buf, pos, chr); + if (commentCount !== 0) { + this.state.commenting = true; + continue; + } + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if (delimiterLength !== 0) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + pos += delimiterLength - 1; + continue; + } + } + } + if (this.state.commenting === false) { + if ( + max_record_size !== 0 && + this.state.record_length + this.state.field.length > max_record_size + ) { + return this.__error( + new CsvError( + "CSV_MAX_RECORD_SIZE", + [ + "Max Record Size:", + "record exceed the maximum number of tolerated bytes", + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } + } + const lappend = + ltrim === false || + this.state.quoting === true || + this.state.field.length !== 0 || + !this.__isCharTrimable(buf, pos); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if (lappend === true && rappend === true) { + this.state.field.append(chr); + } else if (rtrim === true && !this.__isCharTrimable(buf, pos)) { + return this.__error( + new CsvError( + "CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + "found non trimable byte after quote", + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } else { + if (lappend === false) { + pos += this.__isCharTrimable(buf, pos) - 1; + } + continue; + } + } + if (end === true) { + // Ensure we are not ending in a quoting state + if (this.state.quoting === true) { + const err = this.__error( + new CsvError( + "CSV_QUOTE_NOT_CLOSED", + [ + "Quote Not Closed:", + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + // Skip last line if it has no characters + if ( + this.state.wasQuoting === true || + this.state.record.length !== 0 || + this.state.field.length !== 0 + ) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + } else if (this.state.wasRowDelimiter === true) { + this.info.empty_lines++; + } else if (this.state.commenting === true) { + this.info.comment_lines++; + } + } + } else { + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function (push) { + const { + columns, + group_columns_by_name, + encoding, + info, + from, + relax_column_count, + relax_column_count_less, + relax_column_count_more, + raw, + skip_records_with_empty_values, + } = this.options; + const { enabled, record } = this.state; + if (enabled === false) { + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if (columns === true) { + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if (columns === false && this.info.records === 0) { + this.state.expectedRecordLength = recordLength; + } + if (recordLength !== this.state.expectedRecordLength) { + const err = + columns === false + ? new CsvError( + "CSV_RECORD_INCONSISTENT_FIELDS_LENGTH", + [ + "Invalid Record Length:", + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ) + : new CsvError( + "CSV_RECORD_INCONSISTENT_COLUMNS", + [ + "Invalid Record Length:", + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ); + if ( + relax_column_count === true || + (relax_column_count_less === true && + recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && + recordLength > this.state.expectedRecordLength) + ) { + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + } else { + const finalErr = this.__error(err); + if (finalErr) return finalErr; + } + } + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + if (this.state.recordHasError === true) { + this.__resetRecord(); + this.state.recordHasError = false; + return; + } + this.info.records++; + if (from === 1 || this.info.records >= from) { + const { objname } = this.options; + // With columns, records are object + if (columns !== false) { + const obj = {}; + // Transform record array to an object + for (let i = 0, l = record.length; i < l; i++) { + if (columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if ( + group_columns_by_name === true && + Object.hasOwn(obj, columns[i].name) + ) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } + } else { + Object.defineProperty(obj, columns[i].name, { + value: record[i], + enumerable: true, + writable: true, + configurable: true, + }); + } + } + // Without objname (default) + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: obj }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj], + push, + ); + if (err) { + return err; + } + } + // Without columns, records are array + } else { + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: record }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? record : [record[objname], record], + push, + ); + if (err) { + return err; + } + } + } + } + this.__resetRecord(); + }, + __firstLineToColumns: function (record) { + const { firstLineToHeaders } = this.state; + try { + const headers = + firstLineToHeaders === undefined + ? record + : firstLineToHeaders.call(null, record); + if (!Array.isArray(headers)) { + return this.__error( + new CsvError( + "CSV_INVALID_COLUMN_MAPPING", + [ + "Invalid Column Mapping:", + "expect an array from column function,", + `got ${JSON.stringify(headers)}`, + ], + this.options, + this.__infoField(), + { + headers: headers, + }, + ), + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; + } catch (err) { + return err; + } + }, + __resetRecord: function () { + if (this.options.raw === true) { + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function () { + const { cast, encoding, rtrim, max_record_size } = this.options; + const { enabled, wasQuoting } = this.state; + // Short circuit for the from_line options + if (enabled === false) { + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if (rtrim === true && wasQuoting === false) { + field = field.trimRight(); + } + if (cast === true) { + const [err, f] = this.__cast(field); + if (err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if (max_record_size !== 0 && typeof field === "string") { + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function () { + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function (record, push) { + const { on_record } = this.options; + if (on_record !== undefined) { + const info = this.__infoRecord(); + try { + record = on_record.call(null, record, info); + } catch (err) { + return err; + } + if (record === undefined || record === null) { + return; + } + } + this.info.bytes_records += this.info.bytes; + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function (field) { + const { columns, relax_column_count } = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if ( + isColumns === true && + relax_column_count && + this.options.columns.length <= this.state.record.length + ) { + return [undefined, undefined]; + } + if (this.state.castField !== null) { + try { + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + } catch (err) { + return [err]; + } + } + if (this.__isFloat(field)) { + return [undefined, Number(field)]; + } else if (this.options.cast_date !== false) { + const info = this.__infoField(); + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + __compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) { + if (sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for (let i = 1; i < sourceLength; i++) { + if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0; + } + return sourceLength; + }, + // Helper to test if a character is trimable + __isCharTrimable: function (buf, pos) { + const { timchars, timcharFirstBytes } = this.state; + // Fast bail-out: non-whitespace bytes (the common case) are rejected + // without scanning the full timchar list. + const first = buf[pos]; + if (first === undefined || timcharFirstBytes[first] === 0) return 0; + loop1: for (let i = 0; i < timchars.length; i++) { + const timchar = timchars[i]; + for (let j = 0; j < timchar.length; j++) { + if (timchar[j] !== buf[pos + j]) continue loop1; + } + return timchar.length; + } + return 0; + }, + __isDelimiter: function (buf, pos, chr) { + const { delimiter, ignore_last_delimiters } = this.options; + if ( + ignore_last_delimiters === true && + this.state.record.length === this.options.columns.length - 1 + ) { + return 0; + } else if ( + ignore_last_delimiters !== false && + typeof ignore_last_delimiters === "number" && + this.state.record.length === ignore_last_delimiters - 1 + ) { + return 0; + } + loop1: for (let i = 0; i < delimiter.length; i++) { + const del = delimiter[i]; + if (del[0] === chr) { + for (let j = 1; j < del.length; j++) { + if (del[j] !== buf[pos + j]) continue loop1; + } + return del.length; + } + } + return 0; + }, + __isEscape: function (buf, pos, chr) { + const { escape } = this.options; + if (escape === null) return false; + const l = escape.length; + if (escape[0] === chr) { + for (let i = 0; i < l; i++) { + if (escape[i] !== buf[pos + i]) { + return false; + } + } + return true; + } + return false; + }, + __isFloat: function (value) { + return value - parseFloat(value) + 1 >= 0; // Borrowed from jquery + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isQuote: function (buf, pos) { + const { quote } = this.options; + if (quote === null) return false; + const l = quote.length; + for (let i = 0; i < l; i++) { + if (quote[i] !== buf[pos + i]) { + return false; + } + } + return true; + }, + __isRecordDelimiter: function (chr, buf, pos) { + const { record_delimiter } = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for (let i = 0; i < recordDelimiterLength; i++) { + const rd = record_delimiter[i]; + const rdLength = rd.length; + if (rd[0] !== chr) { + continue; + } + for (let j = 1; j < rdLength; j++) { + if (rd[j] !== buf[pos + j]) { + continue loop1; + } + } + return rd.length; + } + return 0; + }, + __autoDiscoverRecordDelimiter: function (buf, pos) { + const { encoding } = this.options; + // Note, we don't need to cache this information in state, + // It is only called on the first line until we find out a suitable + // record delimiter. + const rds = [ + // Important, the windows line ending must be before mac os 9 + Buffer.from("\r\n", encoding), + Buffer.from("\n", encoding), + Buffer.from("\r", encoding), + ]; + loop: for (let i = 0; i < rds.length; i++) { + const l = rds[i].length; + for (let j = 0; j < l; j++) { + if (rds[i][j] !== buf[pos + j]) { + continue loop; + } + } + this.options.record_delimiter.push(rds[i]); + this.state.recordDelimiterMaxLength = rds[i].length; + return rds[i].length; + } + return 0; + }, + __error: function (msg) { + const { encoding, raw, skip_records_with_error } = this.options; + const err = typeof msg === "string" ? new Error(msg) : msg; + if (skip_records_with_error) { + this.state.recordHasError = true; + if (this.options.on_skip !== undefined) { + try { + this.options.on_skip( + err, + raw ? this.state.rawBuffer.toString(encoding) : undefined, + ); + } catch (err) { + return err; + } + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + } else { + return err; + } + }, + __infoDataSet: function () { + return { + ...this.info, + columns: this.options.columns, + }; + }, + __infoRecord: function () { + const { columns, raw, encoding } = this.options; + return { + ...this.__infoDataSet(), + bytes_records: this.info.bytes, + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined, + }; + }, + __infoField: function () { + const { columns } = this.options; + const isColumns = Array.isArray(columns); + // Bytes records are only incremented when all records'fields are parsed + const bytes_records = this.info.bytes_records; + return { + ...this.__infoRecord(), + bytes_records: bytes_records, + column: + isColumns === true + ? columns.length > this.state.record.length + ? columns[this.state.record.length].name + : null + : this.state.record.length, + quoting: this.state.wasQuoting, + }; + }, + }; + }; + + // Parse an asynchronous iterable without a stream wrapper. Drain each chunk's + // records before requesting another chunk, including records preceding an error. + async function* parse(input, opts = {}) { + const api = transform(opts); + let records = []; + let stopped = false; + const push = (record) => records.push(record); + const close = () => { + stopped = true; + }; + for await (const chunk of input) { + const buffer = chunk instanceof Buffer ? chunk : Buffer.from(chunk); + const error = api.parse(buffer, false, push, close); + for (const record of records) yield record; + records = []; + if (error !== undefined) throw error; + if (stopped) return; + } + const error = api.parse(undefined, true, push, close); + for (const record of records) yield record; + if (error !== undefined) throw error; + } + + exports.CsvError = CsvError; + exports.normalize_options = normalize_options; + exports.parse = parse; + + return exports; + +})({}); diff --git a/packages/csv-parse/dist/umd/iterator.js b/packages/csv-parse/dist/umd/iterator.js new file mode 100644 index 000000000..5c5b87d69 --- /dev/null +++ b/packages/csv-parse/dist/umd/iterator.js @@ -0,0 +1,3993 @@ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.csv_parse_iterator = {})); +})(this, (function (exports) { 'use strict'; + + var global$1 = (typeof global !== "undefined" ? global : + typeof self !== "undefined" ? self : + typeof window !== "undefined" ? window : {}); + + var lookup = []; + var revLookup = []; + var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; + var inited = false; + function init () { + inited = true; + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; + } + + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; + } + + function toByteArray (b64) { + if (!inited) { + init(); + } + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; + + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') + } + + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; + + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); + + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; + + var L = 0; + + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + return arr + } + + function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] + } + + function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); + } + return output.join('') + } + + function fromByteArray (uint8) { + if (!inited) { + init(); + } + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 + + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); + } + + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; + } + + parts.push(output); + + return parts.join('') + } + + function read (buffer, offset, isLE, mLen, nBytes) { + var e, m; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var nBits = -7; + var i = isLE ? (nBytes - 1) : 0; + var d = isLE ? -1 : 1; + var s = buffer[offset + i]; + + i += d; + + e = s & ((1 << (-nBits)) - 1); + s >>= (-nBits); + nBits += eLen; + for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + m = e & ((1 << (-nBits)) - 1); + e >>= (-nBits); + nBits += mLen; + for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + if (e === 0) { + e = 1 - eBias; + } else if (e === eMax) { + return m ? NaN : ((s ? -1 : 1) * Infinity) + } else { + m = m + Math.pow(2, mLen); + e = e - eBias; + } + return (s ? -1 : 1) * m * Math.pow(2, e - mLen) + } + + function write (buffer, value, offset, isLE, mLen, nBytes) { + var e, m, c; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); + var i = isLE ? 0 : (nBytes - 1); + var d = isLE ? 1 : -1; + var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; + + value = Math.abs(value); + + if (isNaN(value) || value === Infinity) { + m = isNaN(value) ? 1 : 0; + e = eMax; + } else { + e = Math.floor(Math.log(value) / Math.LN2); + if (value * (c = Math.pow(2, -e)) < 1) { + e--; + c *= 2; + } + if (e + eBias >= 1) { + value += rt / c; + } else { + value += rt * Math.pow(2, 1 - eBias); + } + if (value * c >= 2) { + e++; + c /= 2; + } + + if (e + eBias >= eMax) { + m = 0; + e = eMax; + } else if (e + eBias >= 1) { + m = (value * c - 1) * Math.pow(2, mLen); + e = e + eBias; + } else { + m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); + e = 0; + } + } + + for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} + + e = (e << mLen) | m; + eLen += mLen; + for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} + + buffer[offset + i - d] |= s * 128; + } + + var toString = {}.toString; + + var isArray = Array.isArray || function (arr) { + return toString.call(arr) == '[object Array]'; + }; + + var INSPECT_MAX_BYTES = 50; + + /** + * If `Buffer.TYPED_ARRAY_SUPPORT`: + * === true Use Uint8Array implementation (fastest) + * === false Use Object implementation (most compatible, even IE6) + * + * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, + * Opera 11.6+, iOS 4.2+. + * + * Due to various browser bugs, sometimes the Object implementation will be used even + * when the browser supports typed arrays. + * + * Note: + * + * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, + * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. + * + * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. + * + * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of + * incorrect length in some situations. + + * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they + * get the Object implementation, which is slower but behaves correctly. + */ + Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined + ? global$1.TYPED_ARRAY_SUPPORT + : true; + + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + + function kMaxLength () { + return Buffer.TYPED_ARRAY_SUPPORT + ? 0x7fffffff + : 0x3fffffff + } + + function createBuffer (that, length) { + if (kMaxLength() < length) { + throw new RangeError('Invalid typed array length') + } + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = new Uint8Array(length); + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + if (that === null) { + that = new Buffer(length); + } + that.length = length; + } + + return that + } + + /** + * The Buffer constructor returns instances of `Uint8Array` that have their + * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of + * `Uint8Array`, so the returned instances will have all the node `Buffer` methods + * and the `Uint8Array` methods. Square bracket notation works as expected -- it + * returns a single octet. + * + * The `Uint8Array` prototype remains unmodified. + */ + + function Buffer (arg, encodingOrOffset, length) { + if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { + return new Buffer(arg, encodingOrOffset, length) + } + + // Common case. + if (typeof arg === 'number') { + if (typeof encodingOrOffset === 'string') { + throw new Error( + 'If encoding is specified then the first argument must be a string' + ) + } + return allocUnsafe(this, arg) + } + return from(this, arg, encodingOrOffset, length) + } + + Buffer.poolSize = 8192; // not used by this implementation + + // TODO: Legacy, not needed anymore. Remove in next major version. + Buffer._augment = function (arr) { + arr.__proto__ = Buffer.prototype; + return arr + }; + + function from (that, value, encodingOrOffset, length) { + if (typeof value === 'number') { + throw new TypeError('"value" argument must not be a number') + } + + if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { + return fromArrayBuffer(that, value, encodingOrOffset, length) + } + + if (typeof value === 'string') { + return fromString(that, value, encodingOrOffset) + } + + return fromObject(that, value) + } + + /** + * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError + * if value is a number. + * Buffer.from(str[, encoding]) + * Buffer.from(array) + * Buffer.from(buffer) + * Buffer.from(arrayBuffer[, byteOffset[, length]]) + **/ + Buffer.from = function (value, encodingOrOffset, length) { + return from(null, value, encodingOrOffset, length) + }; + + if (Buffer.TYPED_ARRAY_SUPPORT) { + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; + } + + function assertSize (size) { + if (typeof size !== 'number') { + throw new TypeError('"size" argument must be a number') + } else if (size < 0) { + throw new RangeError('"size" argument must not be negative') + } + } + + function alloc (that, size, fill, encoding) { + assertSize(size); + if (size <= 0) { + return createBuffer(that, size) + } + if (fill !== undefined) { + // Only pay attention to encoding if it's a string. This + // prevents accidentally sending in a number that would + // be interpretted as a start offset. + return typeof encoding === 'string' + ? createBuffer(that, size).fill(fill, encoding) + : createBuffer(that, size).fill(fill) + } + return createBuffer(that, size) + } + + /** + * Creates a new filled Buffer instance. + * alloc(size[, fill[, encoding]]) + **/ + Buffer.alloc = function (size, fill, encoding) { + return alloc(null, size, fill, encoding) + }; + + function allocUnsafe (that, size) { + assertSize(size); + that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) { + for (var i = 0; i < size; ++i) { + that[i] = 0; + } + } + return that + } + + /** + * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. + * */ + Buffer.allocUnsafe = function (size) { + return allocUnsafe(null, size) + }; + /** + * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. + */ + Buffer.allocUnsafeSlow = function (size) { + return allocUnsafe(null, size) + }; + + function fromString (that, string, encoding) { + if (typeof encoding !== 'string' || encoding === '') { + encoding = 'utf8'; + } + + if (!Buffer.isEncoding(encoding)) { + throw new TypeError('"encoding" must be a valid string encoding') + } + + var length = byteLength(string, encoding) | 0; + that = createBuffer(that, length); + + var actual = that.write(string, encoding); + + if (actual !== length) { + // Writing a hex string, for example, that contains invalid characters will + // cause everything after the first invalid character to be ignored. (e.g. + // 'abxxcd' will be treated as 'ab') + that = that.slice(0, actual); + } + + return that + } + + function fromArrayLike (that, array) { + var length = array.length < 0 ? 0 : checked(array.length) | 0; + that = createBuffer(that, length); + for (var i = 0; i < length; i += 1) { + that[i] = array[i] & 255; + } + return that + } + + function fromArrayBuffer (that, array, byteOffset, length) { + array.byteLength; // this throws if `array` is not a valid ArrayBuffer + + if (byteOffset < 0 || array.byteLength < byteOffset) { + throw new RangeError('\'offset\' is out of bounds') + } + + if (array.byteLength < byteOffset + (length || 0)) { + throw new RangeError('\'length\' is out of bounds') + } + + if (byteOffset === undefined && length === undefined) { + array = new Uint8Array(array); + } else if (length === undefined) { + array = new Uint8Array(array, byteOffset); + } else { + array = new Uint8Array(array, byteOffset, length); + } + + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = array; + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + that = fromArrayLike(that, array); + } + return that + } + + function fromObject (that, obj) { + if (internalIsBuffer(obj)) { + var len = checked(obj.length) | 0; + that = createBuffer(that, len); + + if (that.length === 0) { + return that + } + + obj.copy(that, 0, 0, len); + return that + } + + if (obj) { + if ((typeof ArrayBuffer !== 'undefined' && + obj.buffer instanceof ArrayBuffer) || 'length' in obj) { + if (typeof obj.length !== 'number' || isnan(obj.length)) { + return createBuffer(that, 0) + } + return fromArrayLike(that, obj) + } + + if (obj.type === 'Buffer' && isArray(obj.data)) { + return fromArrayLike(that, obj.data) + } + } + + throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') + } + + function checked (length) { + // Note: cannot use `length < kMaxLength()` here because that fails when + // length is NaN (which is otherwise coerced to zero.) + if (length >= kMaxLength()) { + throw new RangeError('Attempt to allocate Buffer larger than maximum ' + + 'size: 0x' + kMaxLength().toString(16) + ' bytes') + } + return length | 0 + } + Buffer.isBuffer = isBuffer; + function internalIsBuffer (b) { + return !!(b != null && b._isBuffer) + } + + Buffer.compare = function compare (a, b) { + if (!internalIsBuffer(a) || !internalIsBuffer(b)) { + throw new TypeError('Arguments must be Buffers') + } + + if (a === b) return 0 + + var x = a.length; + var y = b.length; + + for (var i = 0, len = Math.min(x, y); i < len; ++i) { + if (a[i] !== b[i]) { + x = a[i]; + y = b[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + Buffer.isEncoding = function isEncoding (encoding) { + switch (String(encoding).toLowerCase()) { + case 'hex': + case 'utf8': + case 'utf-8': + case 'ascii': + case 'latin1': + case 'binary': + case 'base64': + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return true + default: + return false + } + }; + + Buffer.concat = function concat (list, length) { + if (!isArray(list)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + + if (list.length === 0) { + return Buffer.alloc(0) + } + + var i; + if (length === undefined) { + length = 0; + for (i = 0; i < list.length; ++i) { + length += list[i].length; + } + } + + var buffer = Buffer.allocUnsafe(length); + var pos = 0; + for (i = 0; i < list.length; ++i) { + var buf = list[i]; + if (!internalIsBuffer(buf)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + buf.copy(buffer, pos); + pos += buf.length; + } + return buffer + }; + + function byteLength (string, encoding) { + if (internalIsBuffer(string)) { + return string.length + } + if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && + (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { + return string.byteLength + } + if (typeof string !== 'string') { + string = '' + string; + } + + var len = string.length; + if (len === 0) return 0 + + // Use a for loop to avoid recursion + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'ascii': + case 'latin1': + case 'binary': + return len + case 'utf8': + case 'utf-8': + case undefined: + return utf8ToBytes(string).length + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return len * 2 + case 'hex': + return len >>> 1 + case 'base64': + return base64ToBytes(string).length + default: + if (loweredCase) return utf8ToBytes(string).length // assume utf8 + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + } + Buffer.byteLength = byteLength; + + function slowToString (encoding, start, end) { + var loweredCase = false; + + // No need to verify that "this.length <= MAX_UINT32" since it's a read-only + // property of a typed array. + + // This behaves neither like String nor Uint8Array in that we set start/end + // to their upper/lower bounds if the value passed is out of range. + // undefined is handled specially as per ECMA-262 6th Edition, + // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. + if (start === undefined || start < 0) { + start = 0; + } + // Return early if start > this.length. Done here to prevent potential uint32 + // coercion fail below. + if (start > this.length) { + return '' + } + + if (end === undefined || end > this.length) { + end = this.length; + } + + if (end <= 0) { + return '' + } + + // Force coersion to uint32. This will also coerce falsey/NaN values to 0. + end >>>= 0; + start >>>= 0; + + if (end <= start) { + return '' + } + + if (!encoding) encoding = 'utf8'; + + while (true) { + switch (encoding) { + case 'hex': + return hexSlice(this, start, end) + + case 'utf8': + case 'utf-8': + return utf8Slice(this, start, end) + + case 'ascii': + return asciiSlice(this, start, end) + + case 'latin1': + case 'binary': + return latin1Slice(this, start, end) + + case 'base64': + return base64Slice(this, start, end) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return utf16leSlice(this, start, end) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = (encoding + '').toLowerCase(); + loweredCase = true; + } + } + } + + // The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect + // Buffer instances. + Buffer.prototype._isBuffer = true; + + function swap (b, n, m) { + var i = b[n]; + b[n] = b[m]; + b[m] = i; + } + + Buffer.prototype.swap16 = function swap16 () { + var len = this.length; + if (len % 2 !== 0) { + throw new RangeError('Buffer size must be a multiple of 16-bits') + } + for (var i = 0; i < len; i += 2) { + swap(this, i, i + 1); + } + return this + }; + + Buffer.prototype.swap32 = function swap32 () { + var len = this.length; + if (len % 4 !== 0) { + throw new RangeError('Buffer size must be a multiple of 32-bits') + } + for (var i = 0; i < len; i += 4) { + swap(this, i, i + 3); + swap(this, i + 1, i + 2); + } + return this + }; + + Buffer.prototype.swap64 = function swap64 () { + var len = this.length; + if (len % 8 !== 0) { + throw new RangeError('Buffer size must be a multiple of 64-bits') + } + for (var i = 0; i < len; i += 8) { + swap(this, i, i + 7); + swap(this, i + 1, i + 6); + swap(this, i + 2, i + 5); + swap(this, i + 3, i + 4); + } + return this + }; + + Buffer.prototype.toString = function toString () { + var length = this.length | 0; + if (length === 0) return '' + if (arguments.length === 0) return utf8Slice(this, 0, length) + return slowToString.apply(this, arguments) + }; + + Buffer.prototype.equals = function equals (b) { + if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') + if (this === b) return true + return Buffer.compare(this, b) === 0 + }; + + Buffer.prototype.inspect = function inspect () { + var str = ''; + var max = INSPECT_MAX_BYTES; + if (this.length > 0) { + str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); + if (this.length > max) str += ' ... '; + } + return '' + }; + + Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { + if (!internalIsBuffer(target)) { + throw new TypeError('Argument must be a Buffer') + } + + if (start === undefined) { + start = 0; + } + if (end === undefined) { + end = target ? target.length : 0; + } + if (thisStart === undefined) { + thisStart = 0; + } + if (thisEnd === undefined) { + thisEnd = this.length; + } + + if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { + throw new RangeError('out of range index') + } + + if (thisStart >= thisEnd && start >= end) { + return 0 + } + if (thisStart >= thisEnd) { + return -1 + } + if (start >= end) { + return 1 + } + + start >>>= 0; + end >>>= 0; + thisStart >>>= 0; + thisEnd >>>= 0; + + if (this === target) return 0 + + var x = thisEnd - thisStart; + var y = end - start; + var len = Math.min(x, y); + + var thisCopy = this.slice(thisStart, thisEnd); + var targetCopy = target.slice(start, end); + + for (var i = 0; i < len; ++i) { + if (thisCopy[i] !== targetCopy[i]) { + x = thisCopy[i]; + y = targetCopy[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + // Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, + // OR the last index of `val` in `buffer` at offset <= `byteOffset`. + // + // Arguments: + // - buffer - a Buffer to search + // - val - a string, Buffer, or number + // - byteOffset - an index into `buffer`; will be clamped to an int32 + // - encoding - an optional encoding, relevant is val is a string + // - dir - true for indexOf, false for lastIndexOf + function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { + // Empty buffer means no match + if (buffer.length === 0) return -1 + + // Normalize byteOffset + if (typeof byteOffset === 'string') { + encoding = byteOffset; + byteOffset = 0; + } else if (byteOffset > 0x7fffffff) { + byteOffset = 0x7fffffff; + } else if (byteOffset < -2147483648) { + byteOffset = -2147483648; + } + byteOffset = +byteOffset; // Coerce to Number. + if (isNaN(byteOffset)) { + // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer + byteOffset = dir ? 0 : (buffer.length - 1); + } + + // Normalize byteOffset: negative offsets start from the end of the buffer + if (byteOffset < 0) byteOffset = buffer.length + byteOffset; + if (byteOffset >= buffer.length) { + if (dir) return -1 + else byteOffset = buffer.length - 1; + } else if (byteOffset < 0) { + if (dir) byteOffset = 0; + else return -1 + } + + // Normalize val + if (typeof val === 'string') { + val = Buffer.from(val, encoding); + } + + // Finally, search either indexOf (if dir is true) or lastIndexOf + if (internalIsBuffer(val)) { + // Special case: looking for empty string/buffer always fails + if (val.length === 0) { + return -1 + } + return arrayIndexOf(buffer, val, byteOffset, encoding, dir) + } else if (typeof val === 'number') { + val = val & 0xFF; // Search for a byte value [0-255] + if (Buffer.TYPED_ARRAY_SUPPORT && + typeof Uint8Array.prototype.indexOf === 'function') { + if (dir) { + return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) + } else { + return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) + } + } + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + } + + throw new TypeError('val must be string, number or Buffer') + } + + function arrayIndexOf (arr, val, byteOffset, encoding, dir) { + var indexSize = 1; + var arrLength = arr.length; + var valLength = val.length; + + if (encoding !== undefined) { + encoding = String(encoding).toLowerCase(); + if (encoding === 'ucs2' || encoding === 'ucs-2' || + encoding === 'utf16le' || encoding === 'utf-16le') { + if (arr.length < 2 || val.length < 2) { + return -1 + } + indexSize = 2; + arrLength /= 2; + valLength /= 2; + byteOffset /= 2; + } + } + + function read (buf, i) { + if (indexSize === 1) { + return buf[i] + } else { + return buf.readUInt16BE(i * indexSize) + } + } + + var i; + if (dir) { + var foundIndex = -1; + for (i = byteOffset; i < arrLength; i++) { + if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { + if (foundIndex === -1) foundIndex = i; + if (i - foundIndex + 1 === valLength) return foundIndex * indexSize + } else { + if (foundIndex !== -1) i -= i - foundIndex; + foundIndex = -1; + } + } + } else { + if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; + for (i = byteOffset; i >= 0; i--) { + var found = true; + for (var j = 0; j < valLength; j++) { + if (read(arr, i + j) !== read(val, j)) { + found = false; + break + } + } + if (found) return i + } + } + + return -1 + } + + Buffer.prototype.includes = function includes (val, byteOffset, encoding) { + return this.indexOf(val, byteOffset, encoding) !== -1 + }; + + Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, true) + }; + + Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, false) + }; + + function hexWrite (buf, string, offset, length) { + offset = Number(offset) || 0; + var remaining = buf.length - offset; + if (!length) { + length = remaining; + } else { + length = Number(length); + if (length > remaining) { + length = remaining; + } + } + + // must be an even number of digits + var strLen = string.length; + if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') + + if (length > strLen / 2) { + length = strLen / 2; + } + for (var i = 0; i < length; ++i) { + var parsed = parseInt(string.substr(i * 2, 2), 16); + if (isNaN(parsed)) return i + buf[offset + i] = parsed; + } + return i + } + + function utf8Write (buf, string, offset, length) { + return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) + } + + function asciiWrite (buf, string, offset, length) { + return blitBuffer(asciiToBytes(string), buf, offset, length) + } + + function latin1Write (buf, string, offset, length) { + return asciiWrite(buf, string, offset, length) + } + + function base64Write (buf, string, offset, length) { + return blitBuffer(base64ToBytes(string), buf, offset, length) + } + + function ucs2Write (buf, string, offset, length) { + return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) + } + + Buffer.prototype.write = function write (string, offset, length, encoding) { + // Buffer#write(string) + if (offset === undefined) { + encoding = 'utf8'; + length = this.length; + offset = 0; + // Buffer#write(string, encoding) + } else if (length === undefined && typeof offset === 'string') { + encoding = offset; + length = this.length; + offset = 0; + // Buffer#write(string, offset[, length][, encoding]) + } else if (isFinite(offset)) { + offset = offset | 0; + if (isFinite(length)) { + length = length | 0; + if (encoding === undefined) encoding = 'utf8'; + } else { + encoding = length; + length = undefined; + } + // legacy write(string, encoding, offset, length) - remove in v0.13 + } else { + throw new Error( + 'Buffer.write(string, encoding, offset[, length]) is no longer supported' + ) + } + + var remaining = this.length - offset; + if (length === undefined || length > remaining) length = remaining; + + if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { + throw new RangeError('Attempt to write outside buffer bounds') + } + + if (!encoding) encoding = 'utf8'; + + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'hex': + return hexWrite(this, string, offset, length) + + case 'utf8': + case 'utf-8': + return utf8Write(this, string, offset, length) + + case 'ascii': + return asciiWrite(this, string, offset, length) + + case 'latin1': + case 'binary': + return latin1Write(this, string, offset, length) + + case 'base64': + // Warning: maxLength not taken into account in base64Write + return base64Write(this, string, offset, length) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return ucs2Write(this, string, offset, length) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + }; + + Buffer.prototype.toJSON = function toJSON () { + return { + type: 'Buffer', + data: Array.prototype.slice.call(this._arr || this, 0) + } + }; + + function base64Slice (buf, start, end) { + if (start === 0 && end === buf.length) { + return fromByteArray(buf) + } else { + return fromByteArray(buf.slice(start, end)) + } + } + + function utf8Slice (buf, start, end) { + end = Math.min(buf.length, end); + var res = []; + + var i = start; + while (i < end) { + var firstByte = buf[i]; + var codePoint = null; + var bytesPerSequence = (firstByte > 0xEF) ? 4 + : (firstByte > 0xDF) ? 3 + : (firstByte > 0xBF) ? 2 + : 1; + + if (i + bytesPerSequence <= end) { + var secondByte, thirdByte, fourthByte, tempCodePoint; + + switch (bytesPerSequence) { + case 1: + if (firstByte < 0x80) { + codePoint = firstByte; + } + break + case 2: + secondByte = buf[i + 1]; + if ((secondByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); + if (tempCodePoint > 0x7F) { + codePoint = tempCodePoint; + } + } + break + case 3: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); + if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { + codePoint = tempCodePoint; + } + } + break + case 4: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + fourthByte = buf[i + 3]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); + if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { + codePoint = tempCodePoint; + } + } + } + } + + if (codePoint === null) { + // we did not generate a valid codePoint so insert a + // replacement char (U+FFFD) and advance only 1 byte + codePoint = 0xFFFD; + bytesPerSequence = 1; + } else if (codePoint > 0xFFFF) { + // encode to utf16 (surrogate pair dance) + codePoint -= 0x10000; + res.push(codePoint >>> 10 & 0x3FF | 0xD800); + codePoint = 0xDC00 | codePoint & 0x3FF; + } + + res.push(codePoint); + i += bytesPerSequence; + } + + return decodeCodePointsArray(res) + } + + // Based on http://stackoverflow.com/a/22747272/680742, the browser with + // the lowest limit is Chrome, with 0x10000 args. + // We go 1 magnitude less, for safety + var MAX_ARGUMENTS_LENGTH = 0x1000; + + function decodeCodePointsArray (codePoints) { + var len = codePoints.length; + if (len <= MAX_ARGUMENTS_LENGTH) { + return String.fromCharCode.apply(String, codePoints) // avoid extra slice() + } + + // Decode in chunks to avoid "call stack size exceeded". + var res = ''; + var i = 0; + while (i < len) { + res += String.fromCharCode.apply( + String, + codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) + ); + } + return res + } + + function asciiSlice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i] & 0x7F); + } + return ret + } + + function latin1Slice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i]); + } + return ret + } + + function hexSlice (buf, start, end) { + var len = buf.length; + + if (!start || start < 0) start = 0; + if (!end || end < 0 || end > len) end = len; + + var out = ''; + for (var i = start; i < end; ++i) { + out += toHex(buf[i]); + } + return out + } + + function utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end); + var res = ''; + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + } + return res + } + + Buffer.prototype.slice = function slice (start, end) { + var len = this.length; + start = ~~start; + end = end === undefined ? len : ~~end; + + if (start < 0) { + start += len; + if (start < 0) start = 0; + } else if (start > len) { + start = len; + } + + if (end < 0) { + end += len; + if (end < 0) end = 0; + } else if (end > len) { + end = len; + } + + if (end < start) end = start; + + var newBuf; + if (Buffer.TYPED_ARRAY_SUPPORT) { + newBuf = this.subarray(start, end); + newBuf.__proto__ = Buffer.prototype; + } else { + var sliceLen = end - start; + newBuf = new Buffer(sliceLen, undefined); + for (var i = 0; i < sliceLen; ++i) { + newBuf[i] = this[i + start]; + } + } + + return newBuf + }; + + /* + * Need to make sure that buffer isn't trying to write out of bounds. + */ + function checkOffset (offset, ext, length) { + if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') + if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') + } + + Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + + return val + }; + + Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + checkOffset(offset, byteLength, this.length); + } + + var val = this[offset + --byteLength]; + var mul = 1; + while (byteLength > 0 && (mul *= 0x100)) { + val += this[offset + --byteLength] * mul; + } + + return val + }; + + Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + return this[offset] + }; + + Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return this[offset] | (this[offset + 1] << 8) + }; + + Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return (this[offset] << 8) | this[offset + 1] + }; + + Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return ((this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16)) + + (this[offset + 3] * 0x1000000) + }; + + Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] * 0x1000000) + + ((this[offset + 1] << 16) | + (this[offset + 2] << 8) | + this[offset + 3]) + }; + + Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var i = byteLength; + var mul = 1; + var val = this[offset + --i]; + while (i > 0 && (mul *= 0x100)) { + val += this[offset + --i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + if (!(this[offset] & 0x80)) return (this[offset]) + return ((0xff - this[offset] + 1) * -1) + }; + + Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset] | (this[offset + 1] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset + 1] | (this[offset] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16) | + (this[offset + 3] << 24) + }; + + Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] << 24) | + (this[offset + 1] << 16) | + (this[offset + 2] << 8) | + (this[offset + 3]) + }; + + Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, true, 23, 4) + }; + + Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, false, 23, 4) + }; + + Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, true, 52, 8) + }; + + Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, false, 52, 8) + }; + + function checkInt (buf, value, offset, ext, max, min) { + if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') + if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') + if (offset + ext > buf.length) throw new RangeError('Index out of range') + } + + Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var mul = 1; + var i = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var i = byteLength - 1; + var mul = 1; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + this[offset] = (value & 0xff); + return offset + 1 + }; + + function objectWriteUInt16 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { + buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> + (littleEndian ? i : 1 - i) * 8; + } + } + + Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + function objectWriteUInt32 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffffffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { + buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; + } + } + + Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset + 3] = (value >>> 24); + this[offset + 2] = (value >>> 16); + this[offset + 1] = (value >>> 8); + this[offset] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = 0; + var mul = 1; + var sub = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = byteLength - 1; + var mul = 1; + var sub = 0; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -128); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + if (value < 0) value = 0xff + value + 1; + this[offset] = (value & 0xff); + return offset + 1 + }; + + Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -32768); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -32768); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -2147483648); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + this[offset + 2] = (value >>> 16); + this[offset + 3] = (value >>> 24); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -2147483648); + if (value < 0) value = 0xffffffff + value + 1; + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + function checkIEEE754 (buf, value, offset, ext, max, min) { + if (offset + ext > buf.length) throw new RangeError('Index out of range') + if (offset < 0) throw new RangeError('Index out of range') + } + + function writeFloat (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 4); + } + write(buf, value, offset, littleEndian, 23, 4); + return offset + 4 + } + + Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { + return writeFloat(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { + return writeFloat(this, value, offset, false, noAssert) + }; + + function writeDouble (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 8); + } + write(buf, value, offset, littleEndian, 52, 8); + return offset + 8 + } + + Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { + return writeDouble(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { + return writeDouble(this, value, offset, false, noAssert) + }; + + // copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) + Buffer.prototype.copy = function copy (target, targetStart, start, end) { + if (!start) start = 0; + if (!end && end !== 0) end = this.length; + if (targetStart >= target.length) targetStart = target.length; + if (!targetStart) targetStart = 0; + if (end > 0 && end < start) end = start; + + // Copy 0 bytes; we're done + if (end === start) return 0 + if (target.length === 0 || this.length === 0) return 0 + + // Fatal error conditions + if (targetStart < 0) { + throw new RangeError('targetStart out of bounds') + } + if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') + if (end < 0) throw new RangeError('sourceEnd out of bounds') + + // Are we oob? + if (end > this.length) end = this.length; + if (target.length - targetStart < end - start) { + end = target.length - targetStart + start; + } + + var len = end - start; + var i; + + if (this === target && start < targetStart && targetStart < end) { + // descending copy from end + for (i = len - 1; i >= 0; --i) { + target[i + targetStart] = this[i + start]; + } + } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { + // ascending copy from start + for (i = 0; i < len; ++i) { + target[i + targetStart] = this[i + start]; + } + } else { + Uint8Array.prototype.set.call( + target, + this.subarray(start, start + len), + targetStart + ); + } + + return len + }; + + // Usage: + // buffer.fill(number[, offset[, end]]) + // buffer.fill(buffer[, offset[, end]]) + // buffer.fill(string[, offset[, end]][, encoding]) + Buffer.prototype.fill = function fill (val, start, end, encoding) { + // Handle string cases: + if (typeof val === 'string') { + if (typeof start === 'string') { + encoding = start; + start = 0; + end = this.length; + } else if (typeof end === 'string') { + encoding = end; + end = this.length; + } + if (val.length === 1) { + var code = val.charCodeAt(0); + if (code < 256) { + val = code; + } + } + if (encoding !== undefined && typeof encoding !== 'string') { + throw new TypeError('encoding must be a string') + } + if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { + throw new TypeError('Unknown encoding: ' + encoding) + } + } else if (typeof val === 'number') { + val = val & 255; + } + + // Invalid ranges are not set to a default, so can range check early. + if (start < 0 || this.length < start || this.length < end) { + throw new RangeError('Out of range index') + } + + if (end <= start) { + return this + } + + start = start >>> 0; + end = end === undefined ? this.length : end >>> 0; + + if (!val) val = 0; + + var i; + if (typeof val === 'number') { + for (i = start; i < end; ++i) { + this[i] = val; + } + } else { + var bytes = internalIsBuffer(val) + ? val + : utf8ToBytes(new Buffer(val, encoding).toString()); + var len = bytes.length; + for (i = 0; i < end - start; ++i) { + this[i + start] = bytes[i % len]; + } + } + + return this + }; + + // HELPER FUNCTIONS + // ================ + + var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; + + function base64clean (str) { + // Node strips out invalid characters like \n and \t from the string, base64-js does not + str = stringtrim(str).replace(INVALID_BASE64_RE, ''); + // Node converts strings with length < 2 to '' + if (str.length < 2) return '' + // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not + while (str.length % 4 !== 0) { + str = str + '='; + } + return str + } + + function stringtrim (str) { + if (str.trim) return str.trim() + return str.replace(/^\s+|\s+$/g, '') + } + + function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) + } + + function utf8ToBytes (string, units) { + units = units || Infinity; + var codePoint; + var length = string.length; + var leadSurrogate = null; + var bytes = []; + + for (var i = 0; i < length; ++i) { + codePoint = string.charCodeAt(i); + + // is surrogate component + if (codePoint > 0xD7FF && codePoint < 0xE000) { + // last char was a lead + if (!leadSurrogate) { + // no lead yet + if (codePoint > 0xDBFF) { + // unexpected trail + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } else if (i + 1 === length) { + // unpaired lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } + + // valid lead + leadSurrogate = codePoint; + + continue + } + + // 2 leads in a row + if (codePoint < 0xDC00) { + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + leadSurrogate = codePoint; + continue + } + + // valid surrogate pair + codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; + } else if (leadSurrogate) { + // valid bmp char, but last char was a lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + } + + leadSurrogate = null; + + // encode utf8 + if (codePoint < 0x80) { + if ((units -= 1) < 0) break + bytes.push(codePoint); + } else if (codePoint < 0x800) { + if ((units -= 2) < 0) break + bytes.push( + codePoint >> 0x6 | 0xC0, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x10000) { + if ((units -= 3) < 0) break + bytes.push( + codePoint >> 0xC | 0xE0, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x110000) { + if ((units -= 4) < 0) break + bytes.push( + codePoint >> 0x12 | 0xF0, + codePoint >> 0xC & 0x3F | 0x80, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else { + throw new Error('Invalid code point') + } + } + + return bytes + } + + function asciiToBytes (str) { + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + // Node's code seems to be doing this and not & 0x7F.. + byteArray.push(str.charCodeAt(i) & 0xFF); + } + return byteArray + } + + function utf16leToBytes (str, units) { + var c, hi, lo; + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + if ((units -= 2) < 0) break + + c = str.charCodeAt(i); + hi = c >> 8; + lo = c % 256; + byteArray.push(lo); + byteArray.push(hi); + } + + return byteArray + } + + + function base64ToBytes (str) { + return toByteArray(base64clean(str)) + } + + function blitBuffer (src, dst, offset, length) { + for (var i = 0; i < length; ++i) { + if ((i + offset >= dst.length) || (i >= src.length)) break + dst[i + offset] = src[i]; + } + return i + } + + function isnan (val) { + return val !== val // eslint-disable-line no-self-compare + } + + + // the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence + // The _isBuffer check is for Safari 5-7 support, because it's missing + // Object.prototype.constructor. Remove this eventually + function isBuffer(obj) { + return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) + } + + function isFastBuffer (obj) { + return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) + } + + // For Node v0.10 support. Remove this eventually. + function isSlowBuffer (obj) { + return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) + } + + class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if (Array.isArray(message)) message = message.join(" ").trim(); + super(message); + if (Error.captureStackTrace !== undefined) { + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for (const context of contexts) { + for (const key in context) { + const value = context[key]; + this[key] = isBuffer(value) + ? value.toString(options.encoding) + : value == null + ? value + : JSON.parse(JSON.stringify(value)); + } + } + } + } + + const is_object = function (obj) { + return typeof obj === "object" && obj !== null && !Array.isArray(obj); + }; + + const normalize_columns_array = function (columns) { + const normalizedColumns = []; + for (let i = 0, l = columns.length; i < l; i++) { + const column = columns[i]; + if (column === undefined || column === null || column === false) { + normalizedColumns[i] = { disabled: true }; + } else if (typeof column === "string" || typeof column === "number") { + normalizedColumns[i] = { name: `${column}` }; + } else if (is_object(column)) { + if (typeof column.name !== "string") { + throw new CsvError("CSV_OPTION_COLUMNS_MISSING_NAME", [ + "Option columns missing name:", + `property "name" is required at position ${i}`, + "when column is an object literal", + ]); + } + normalizedColumns[i] = column; + } else { + throw new CsvError("CSV_INVALID_COLUMN_DEFINITION", [ + "Invalid column definition:", + "expect a string or a literal object,", + `got ${JSON.stringify(column)} at position ${i}`, + ]); + } + } + return normalizedColumns; + }; + + class ResizeableBuffer { + constructor(size = 100) { + this.size = size; + this.length = 0; + this.buf = Buffer.allocUnsafe(size); + } + prepend(val) { + if (isBuffer(val)) { + const length = this.length + val.length; + if (length >= this.size) { + this.resize(); + if (length >= this.size) { + throw Error("INVALID_BUFFER_STATE"); + } + } + const buf = this.buf; + this.buf = Buffer.allocUnsafe(this.size); + val.copy(this.buf, 0); + buf.copy(this.buf, val.length); + this.length += val.length; + } else { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + const buf = this.clone(); + this.buf[0] = val; + buf.copy(this.buf, 1, 0, length); + } + } + append(val) { + const length = this.length++; + if (length === this.size) { + this.resize(); + } + this.buf[length] = val; + } + clone() { + return Buffer.from(this.buf.slice(0, this.length)); + } + resize() { + const length = this.length; + this.size = this.size * 2; + const buf = Buffer.allocUnsafe(this.size); + this.buf.copy(buf, 0, 0, length); + this.buf = buf; + } + toString(encoding) { + if (encoding) { + return this.buf.toString(encoding, 0, this.length); + } else { + return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)); + } + } + toJSON() { + return this.toString("utf8"); + } + reset() { + this.length = 0; + } + } + + const init_state = function (options) { + // ECMAScript WhiteSpace + LineTerminator codepoints, encoded under + // `options.encoding`. Aligns trimming with `String.prototype.trim()`. + // https://tc39.es/ecma262/#sec-white-space + // https://tc39.es/ecma262/#sec-line-terminators + // + // Codepoints unrepresentable in the target encoding are dropped: Node's + // Buffer substitutes them with `?` (0x3F), and including those would cause + // literal `?` bytes in the input to be trimmed under `latin1`/`ascii`. + const timchars = [ + // Basic Latin + 0x0020, // [Space](https://www.fileformat.info/info/unicode/char/0020/index.htm) + 0x0009, // [CHARACTER TABULATION (HT)](https://www.fileformat.info/info/unicode/char/0009/index.htm) + 0x000a, // [LINE FEED (LF)](https://www.fileformat.info/info/unicode/char/000a/index.htm) + 0x000d, // [CARRIAGE RETURN (CR)](https://www.fileformat.info/info/unicode/char/000d/index.htm) + 0x000c, // [FORM FEED (FF)](https://www.fileformat.info/info/unicode/char/000c/index.htm) + 0x000b, // [LINE TABULATION (VT)](https://www.fileformat.info/info/unicode/char/000b/index.htm) + // Latin-1 Supplement + 0x00a0, // [NO-BREAK SPACE (NBSP)](https://www.fileformat.info/info/unicode/char/00a0/index.htm) + // Ogham + 0x1680, // [OGHAM SPACE MARK](https://www.fileformat.info/info/unicode/char/1680/index.htm) + // General Punctuation + 0x2000, // [EN QUAD](https://www.fileformat.info/info/unicode/char/2000/index.htm) + 0x2001, // [EM QUAD](https://www.fileformat.info/info/unicode/char/2001/index.htm) + 0x2002, // [EN SPACE](https://www.fileformat.info/info/unicode/char/2002/index.htm) + 0x2003, // [EM SPACE](https://www.fileformat.info/info/unicode/char/2003/index.htm) + 0x2004, // [THREE-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2004/index.htm) + 0x2005, // [FOUR-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2005/index.htm) + 0x2006, // [SIX-PER-EM SPACE](https://www.fileformat.info/info/unicode/char/2006/index.htm) + 0x2007, // [FIGURE SPACE](https://www.fileformat.info/info/unicode/char/2007/index.htm) + 0x2008, // [PUNCTUATION SPACE](https://www.fileformat.info/info/unicode/char/2008/index.htm) + 0x2009, // [THIN SPACE](https://www.fileformat.info/info/unicode/char/2009/index.htm) + 0x200a, // [HAIR SPACE](https://www.fileformat.info/info/unicode/char/200a/index.htm) + 0x2028, // [LINE SEPARATOR](https://www.fileformat.info/info/unicode/char/2028/index.htm) + 0x2029, // [PARAGRAPH SEPARATOR](https://www.fileformat.info/info/unicode/char/2029/index.htm) + 0x202f, // [NARROW NO-BREAK SPACE (NNBSP)](https://www.fileformat.info/info/unicode/char/202f/index.htm) + 0x205f, // [MEDIUM MATHEMATICAL SPACE (MMSP)](https://www.fileformat.info/info/unicode/char/205f/index.htm) + 0x3000, // [IDEOGRAPHIC SPACE](https://www.fileformat.info/info/unicode/char/3000/index.htm) + 0xfeff, // [ZERO WIDTH NO-BREAK SPACE (BOM)](https://www.fileformat.info/info/unicode/char/feff/index.htm) + ].reduce((acc, codepoint) => { + const encoded = Buffer.from( + String.fromCharCode(codepoint), + options.encoding, + ); + if (codepoint !== 0x3f && encoded.length === 1 && encoded[0] === 0x3f) { + return acc; + } + acc.push(encoded); + return acc; + }, []); + // First-byte lookup table for `__isCharTrimable`. Non-whitespace bytes + // (the common case) bail out in O(1) without scanning every timchar. + const timcharFirstBytes = new Uint8Array(256); + for (const t of timchars) timcharFirstBytes[t[0]] = 1; + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + delimiterBufPrevious: undefined, + delimiterDiscovered: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: + isBuffer(options.escape) && + isBuffer(options.quote) && + Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) + ? options.columns.length + : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...(options.delimiter + ? options.delimiter.map((delimiter) => delimiter.length) + : []), + // Auto discovery of delimiter is limited to 1 character + options.delimiter_auto ? 1 : 0, + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + // Skip if the remaining buffer can be a multi-byte trim character + ...timchars.map((t) => t.length), + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: + options.record_delimiter.length === 0 + ? 0 + : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [ + Buffer.from(" ", options.encoding)[0], + Buffer.from("\t", options.encoding)[0], + ], + wasQuoting: false, + wasRowDelimiter: false, + timchars: timchars, + timcharFirstBytes: timcharFirstBytes, + }; + }; + + const underscore = function (str) { + return str.replace(/([A-Z])/g, function (_, match) { + return "_" + match.toLowerCase(); + }); + }; + + const normalize_options = function (opts) { + const options = {}; + // Merge with user options + for (const opt in opts) { + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if (options.encoding === undefined || options.encoding === true) { + options.encoding = "utf8"; + } else if (options.encoding === null || options.encoding === false) { + options.encoding = null; + } else if ( + typeof options.encoding !== "string" && + options.encoding !== null + ) { + throw new CsvError( + "CSV_INVALID_OPTION_ENCODING", + [ + "Invalid option encoding:", + "encoding must be a string or null to return a buffer,", + `got ${JSON.stringify(options.encoding)}`, + ], + options, + ); + } + // Normalize option `bom` + if ( + options.bom === undefined || + options.bom === null || + options.bom === false + ) { + options.bom = false; + } else if (options.bom !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_BOM", + [ + "Invalid option bom:", + "bom must be true,", + `got ${JSON.stringify(options.bom)}`, + ], + options, + ); + } + // Normalize option `cast` + options.cast_function = null; + if ( + options.cast === undefined || + options.cast === null || + options.cast === false || + options.cast === "" + ) { + options.cast = undefined; + } else if (typeof options.cast === "function") { + options.cast_function = options.cast; + options.cast = true; + } else if (options.cast !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_CAST", + [ + "Invalid option cast:", + "cast must be true or a function,", + `got ${JSON.stringify(options.cast)}`, + ], + options, + ); + } + // Normalize option `cast_date` + if ( + options.cast_date === undefined || + options.cast_date === null || + options.cast_date === false || + options.cast_date === "" + ) { + options.cast_date = false; + } else if (options.cast_date === true) { + options.cast_date = function (value) { + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + } else if (typeof options.cast_date !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_CAST_DATE", + [ + "Invalid option cast_date:", + "cast_date must be true or a function,", + `got ${JSON.stringify(options.cast_date)}`, + ], + options, + ); + } + // Normalize option `columns` + options.cast_first_line_to_header = undefined; + if (options.columns === true) { + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + } else if (typeof options.columns === "function") { + options.cast_first_line_to_header = options.columns; + options.columns = true; + } else if (Array.isArray(options.columns)) { + options.columns = normalize_columns_array(options.columns); + } else if ( + options.columns === undefined || + options.columns === null || + options.columns === false + ) { + options.columns = false; + } else { + throw new CsvError( + "CSV_INVALID_OPTION_COLUMNS", + [ + "Invalid option columns:", + "expect an array, a function or true,", + `got ${JSON.stringify(options.columns)}`, + ], + options, + ); + } + // Normalize option `group_columns_by_name` + if ( + options.group_columns_by_name === undefined || + options.group_columns_by_name === null || + options.group_columns_by_name === false + ) { + options.group_columns_by_name = false; + } else if (options.group_columns_by_name !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "expect a boolean,", + `got ${JSON.stringify(options.group_columns_by_name)}`, + ], + options, + ); + } else if (options.columns === false) { + throw new CsvError( + "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", + [ + "Invalid option group_columns_by_name:", + "the `columns` mode must be activated.", + ], + options, + ); + } + // Normalize option `comment` + if ( + options.comment === undefined || + options.comment === null || + options.comment === false || + options.comment === "" + ) { + options.comment = null; + } else { + if (typeof options.comment === "string") { + options.comment = Buffer.from(options.comment, options.encoding); + } + if (!isBuffer(options.comment)) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment:", + "comment must be a buffer or a string,", + `got ${JSON.stringify(options.comment)}`, + ], + options, + ); + } + } + // Normalize option `comment_no_infix` + if ( + options.comment_no_infix === undefined || + options.comment_no_infix === null || + options.comment_no_infix === false + ) { + options.comment_no_infix = false; + } else if (options.comment_no_infix !== true) { + throw new CsvError( + "CSV_INVALID_OPTION_COMMENT", + [ + "Invalid option comment_no_infix:", + "value must be a boolean,", + `got ${JSON.stringify(options.comment_no_infix)}`, + ], + options, + ); + } + // Normalize option `delimiter_auto` + if ( + options.delimiter_auto === undefined || + options.delimiter_auto === null || + options.delimiter_auto === false + ) { + options.delimiter_auto = false; + } else if (options.delimiter_auto === true) { + options.delimiter_auto = {}; + } else if (!is_object(options.delimiter_auto)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "delimiter_auto must be a boolean or a configuration object,", + `got ${JSON.stringify(options.delimiter_auto)}`, + ], + options, + ); + } + if (options.delimiter_auto) { + if (options.delimiter_auto.preferred === undefined) + options.delimiter_auto.preferred = { + [",".charCodeAt(0)]: 1.8, + ["\t".charCodeAt(0)]: 1.8, + [";".charCodeAt(0)]: 1.6, + [" ".charCodeAt(0)]: 1.6, + [":".charCodeAt(0)]: 1.5, + [".".charCodeAt(0)]: 1.4, + ["/".charCodeAt(0)]: 1.4, + }; + else if (!is_object(options.delimiter_auto.preferred)) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "preferred must be an object,", + `got ${JSON.stringify(options.delimiter_auto.preferred)}`, + ], + options, + ); + } + if (options.delimiter_auto.score === undefined) + options.delimiter_auto.score = (info, options) => { + return ( + (info.total - info.std) * (options.preferred[info.char_code] || 1) + ); + }; + else if (typeof options.delimiter_auto.score !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "score must be a function,", + `got ${JSON.stringify(options.delimiter_auto.score)}`, + ], + options, + ); + } + if (options.delimiter_auto.size === undefined) + options.delimiter_auto.size = 2048; + else if (typeof options.delimiter_auto.size !== "number") { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER_AUTO", + [ + "Invalid option delimiter_auto:", + "size must be a number,", + `got ${JSON.stringify(options.delimiter_auto.size)}`, + ], + options, + ); + } + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if (options.delimiter_auto !== false) { + options.delimiter = []; + } + if (!Array.isArray(options.delimiter)) { + if ( + options.delimiter === undefined || + options.delimiter === null || + options.delimiter === false + ) { + options.delimiter = Buffer.from(",", options.encoding); + } + options.delimiter = [options.delimiter]; + } + options.delimiter = options.delimiter.map(function (delimiter) { + if (typeof delimiter === "string") { + delimiter = Buffer.from(delimiter, options.encoding); + } + if (!isBuffer(delimiter) || delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_DELIMITER", + [ + "Invalid option delimiter:", + "delimiter must be a non empty string or buffer or array of string|buffer,", + `got ${delimiter_json}`, + ], + options, + ); + } + return delimiter; + }); + // Normalize option `escape` + if (options.escape === undefined || options.escape === true) { + options.escape = Buffer.from('"', options.encoding); + } else if (typeof options.escape === "string") { + options.escape = Buffer.from(options.escape, options.encoding); + } else if (options.escape === null || options.escape === false) { + options.escape = null; + } + if (options.escape !== null) { + if (!isBuffer(options.escape)) { + throw new Error( + `Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`, + ); + } + } + // Normalize option `from` + if (options.from === undefined || options.from === null) { + options.from = 1; + } else { + if (typeof options.from === "string" && /\d+/.test(options.from)) { + options.from = parseInt(options.from); + } + if (Number.isInteger(options.from)) { + if (options.from < 0) { + throw new Error( + `Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`, + ); + } + } + // Normalize option `from_line` + if (options.from_line === undefined || options.from_line === null) { + options.from_line = 1; + } else { + if ( + typeof options.from_line === "string" && + /\d+/.test(options.from_line) + ) { + options.from_line = parseInt(options.from_line); + } + if (Number.isInteger(options.from_line)) { + if (options.from_line <= 0) { + throw new Error( + `Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`, + ); + } + } + // Normalize options `ignore_last_delimiters` + if ( + options.ignore_last_delimiters === undefined || + options.ignore_last_delimiters === null + ) { + options.ignore_last_delimiters = false; + } else if (typeof options.ignore_last_delimiters === "number") { + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if (options.ignore_last_delimiters === 0) { + options.ignore_last_delimiters = false; + } + } else if (typeof options.ignore_last_delimiters !== "boolean") { + throw new CsvError( + "CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS", + [ + "Invalid option `ignore_last_delimiters`:", + "the value must be a boolean value or an integer,", + `got ${JSON.stringify(options.ignore_last_delimiters)}`, + ], + options, + ); + } + if (options.ignore_last_delimiters === true && options.columns === false) { + throw new CsvError( + "CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS", + [ + "The option `ignore_last_delimiters`", + "requires the activation of the `columns` option", + ], + options, + ); + } + // Normalize option `info` + if ( + options.info === undefined || + options.info === null || + options.info === false + ) { + options.info = false; + } else if (options.info !== true) { + throw new Error( + `Invalid Option: info must be true, got ${JSON.stringify(options.info)}`, + ); + } + // Normalize option `max_record_size` + if ( + options.max_record_size === undefined || + options.max_record_size === null || + options.max_record_size === false + ) { + options.max_record_size = 0; + } else if ( + Number.isInteger(options.max_record_size) && + options.max_record_size >= 0 + ) ; else if ( + typeof options.max_record_size === "string" && + /\d+/.test(options.max_record_size) + ) { + options.max_record_size = parseInt(options.max_record_size); + } else { + throw new Error( + `Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`, + ); + } + // Normalize option `objname` + if ( + options.objname === undefined || + options.objname === null || + options.objname === false + ) { + options.objname = undefined; + } else if (isBuffer(options.objname)) { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if (options.encoding === null) ; else { + options.objname = options.objname.toString(options.encoding); + } + } else if (typeof options.objname === "string") { + if (options.objname.length === 0) { + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + } else if (typeof options.objname === "number") ; else { + throw new Error( + `Invalid Option: objname must be a string or a buffer, got ${options.objname}`, + ); + } + if (options.objname !== undefined) { + if (typeof options.objname === "number") { + if (options.columns !== false) { + throw Error( + "Invalid Option: objname index cannot be combined with columns or be defined as a field", + ); + } + } else { + // A string or a buffer + if (options.columns === false) { + throw Error( + "Invalid Option: objname field must be combined with columns or be defined as an index", + ); + } + } + } + // Normalize option `on_record` + if (options.on_record === undefined || options.on_record === null) { + options.on_record = undefined; + } else if (typeof options.on_record !== "function") { + throw new CsvError( + "CSV_INVALID_OPTION_ON_RECORD", + [ + "Invalid option `on_record`:", + "expect a function,", + `got ${JSON.stringify(options.on_record)}`, + ], + options, + ); + } + // Normalize option `on_skip` + // options.on_skip ??= (err, chunk) => { + // this.emit('skip', err, chunk); + // }; + if ( + options.on_skip !== undefined && + options.on_skip !== null && + typeof options.on_skip !== "function" + ) { + throw new Error( + `Invalid Option: on_skip must be a function, got ${JSON.stringify(options.on_skip)}`, + ); + } + // Normalize option `quote` + if ( + options.quote === null || + options.quote === false || + options.quote === "" + ) { + options.quote = null; + } else { + if (options.quote === undefined || options.quote === true) { + options.quote = Buffer.from('"', options.encoding); + } else if (typeof options.quote === "string") { + options.quote = Buffer.from(options.quote, options.encoding); + } + if (!isBuffer(options.quote)) { + throw new Error( + `Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`, + ); + } + } + // Normalize option `raw` + if ( + options.raw === undefined || + options.raw === null || + options.raw === false + ) { + options.raw = false; + } else if (options.raw !== true) { + throw new Error( + `Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`, + ); + } + // Normalize option `record_delimiter` + if (options.record_delimiter === undefined) { + options.record_delimiter = []; + } else if ( + typeof options.record_delimiter === "string" || + isBuffer(options.record_delimiter) + ) { + if (options.record_delimiter.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = [options.record_delimiter]; + } else if (!Array.isArray(options.record_delimiter)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer,", + `got ${JSON.stringify(options.record_delimiter)}`, + ], + options, + ); + } + options.record_delimiter = options.record_delimiter.map(function (rd, i) { + if (typeof rd !== "string" && !isBuffer(rd)) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a string, a buffer or array of string|buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } else if (rd.length === 0) { + throw new CsvError( + "CSV_INVALID_OPTION_RECORD_DELIMITER", + [ + "Invalid option `record_delimiter`:", + "value must be a non empty string or buffer", + `at index ${i},`, + `got ${JSON.stringify(rd)}`, + ], + options, + ); + } + if (typeof rd === "string") { + rd = Buffer.from(rd, options.encoding); + } + return rd; + }); + // Normalize option `relax_column_count` + if (typeof options.relax_column_count === "boolean") ; else if ( + options.relax_column_count === undefined || + options.relax_column_count === null + ) { + options.relax_column_count = false; + } else { + throw new Error( + `Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`, + ); + } + if (typeof options.relax_column_count_less === "boolean") ; else if ( + options.relax_column_count_less === undefined || + options.relax_column_count_less === null + ) { + options.relax_column_count_less = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`, + ); + } + if (typeof options.relax_column_count_more === "boolean") ; else if ( + options.relax_column_count_more === undefined || + options.relax_column_count_more === null + ) { + options.relax_column_count_more = false; + } else { + throw new Error( + `Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`, + ); + } + // Normalize option `relax_quotes` + if (typeof options.relax_quotes === "boolean") ; else if ( + options.relax_quotes === undefined || + options.relax_quotes === null + ) { + options.relax_quotes = false; + } else { + throw new Error( + `Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`, + ); + } + // Normalize option `skip_empty_lines` + if (typeof options.skip_empty_lines === "boolean") ; else if ( + options.skip_empty_lines === undefined || + options.skip_empty_lines === null + ) { + options.skip_empty_lines = false; + } else { + throw new Error( + `Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`, + ); + } + // Normalize option `skip_records_with_empty_values` + if (typeof options.skip_records_with_empty_values === "boolean") ; else if ( + options.skip_records_with_empty_values === undefined || + options.skip_records_with_empty_values === null + ) { + options.skip_records_with_empty_values = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`, + ); + } + // Normalize option `skip_records_with_error` + if (typeof options.skip_records_with_error === "boolean") ; else if ( + options.skip_records_with_error === undefined || + options.skip_records_with_error === null + ) { + options.skip_records_with_error = false; + } else { + throw new Error( + `Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`, + ); + } + // Normalize option `rtrim` + if ( + options.rtrim === undefined || + options.rtrim === null || + options.rtrim === false + ) { + options.rtrim = false; + } else if (options.rtrim !== true) { + throw new Error( + `Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`, + ); + } + // Normalize option `ltrim` + if ( + options.ltrim === undefined || + options.ltrim === null || + options.ltrim === false + ) { + options.ltrim = false; + } else if (options.ltrim !== true) { + throw new Error( + `Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`, + ); + } + // Normalize option `trim` + if ( + options.trim === undefined || + options.trim === null || + options.trim === false + ) { + options.trim = false; + } else if (options.trim !== true) { + throw new Error( + `Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`, + ); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if (options.trim === true && opts.ltrim !== false) { + options.ltrim = true; + } else if (options.ltrim !== true) { + options.ltrim = false; + } + if (options.trim === true && opts.rtrim !== false) { + options.rtrim = true; + } else if (options.rtrim !== true) { + options.rtrim = false; + } + // Normalize option `to` + if (options.to === undefined || options.to === null) { + options.to = -1; + } else if (options.to !== -1) { + if (typeof options.to === "string" && /\d+/.test(options.to)) { + options.to = parseInt(options.to); + } + if (Number.isInteger(options.to)) { + if (options.to <= 0) { + throw new Error( + `Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`, + ); + } + } + // Normalize option `to_line` + if (options.to_line === undefined || options.to_line === null) { + options.to_line = -1; + } else if (options.to_line !== -1) { + if (typeof options.to_line === "string" && /\d+/.test(options.to_line)) { + options.to_line = parseInt(options.to_line); + } + if (Number.isInteger(options.to_line)) { + if (options.to_line <= 0) { + throw new Error( + `Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`, + ); + } + } else { + throw new Error( + `Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`, + ); + } + } + return options; + }; + + // Discussed in [issue #400](https://github.com/adaltas/node-csv/issues/400) + // See https://github.com/python/cpython/blob/ea1b1c579f600cc85d145c60862b2e6b98701b24/Lib/csv.py#L349 + const delimiter_discover = function (records, options) { + // Normalize the configuration + if (!options) { + ({ delimiter_auto: options } = normalize_options({ delimiter_auto: true })); + } + // Convert String to Buffer + if (typeof records === "string") { + records = Buffer.from(records); + } + // Convert Buffer to an array of records + if (isBuffer(records)) { + records = ((data) => { + const records = []; + const parser = transform({ delimiter: [] }); + const push = (record) => records.push(record); + const close = () => {}; + const error = parser.parse(data, true, push, close); + if (error !== undefined) throw error; + return records; + })(records); + } + // Info array initialization, 127 entries, one per char code + const info = Array(127) + .fill() + .map(() => ({ lines: [] })); + // Traverse each records, count occurences per char code + records.map(([record], line) => { + for (let i = 0, l = record.length; i < l; i++) { + // Count the character frequency + const code = record.charCodeAt(i); + if (info[code].lines[line] === undefined) info[code].lines[line] = 0; + info[code].lines[line]++; + } + }); + // Traverse each char code, compute the score + info.map((info, i) => { + info.char_code = i; + info.std = std(info.lines); + info.total = info.lines.reduce((acc, val) => acc + val, 0); + info.preferred = !!options.preferred[i]; + info.score = options.score(info, options); + }); + // Extract the dominant character + const result = info.reduce( + (acc, info) => (acc.score > info.score ? acc : info), + {}, + ); + return String.fromCharCode(result.char_code); + }; + + const std = function (array) { + const n = array.length; + if (n === 0) return 0; + const mean = array.reduce((a, b) => a + b) / n; + return Math.sqrt( + array.map((x) => Math.pow(x - mean, 2)).reduce((a, b) => a + b) / n, + ); + }; + + const isRecordEmpty = function (record) { + return record.every( + (field) => + field == null || (field.toString && field.toString().trim() === ""), + ); + }; + + const cr = 13; // `\r`, carriage return, 0x0D in hexadécimal, 13 in decimal + const nl = 10; // `\n`, newline, 0x0A in hexadecimal, 10 in decimal + + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + utf8: Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + utf16le: Buffer.from([255, 254]), + }; + + const transform = function (original_options = {}) { + const info = { + bytes: 0, + bytes_records: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0, + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function (i, bufLen, end) { + if (end) return false; + const { encoding, escape, quote } = this.options; + const { quoting, needMoreDataSize, recordDelimiterMaxLength } = + this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + // If "record_delimiter" is yet to be discovered: + // 1. It is equals to `[]` and "recordDelimiterMaxLength" equals `0` + // 2. We set the length to windows line ending in the current encoding + // Note, that encoding is known from user or bom discovery at that point + // recordDelimiterMaxLength, + recordDelimiterMaxLength === 0 + ? Buffer.from("\r\n", encoding).length + : recordDelimiterMaxLength, + // Skip if remaining buffer can be an escaped quote + quoting ? (escape === null ? 0 : escape.length) + quote.length : 0, + // Skip if remaining buffer can be record delimiter following the closing quote + quoting ? quote.length + recordDelimiterMaxLength : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function (nextBuf, end, push, close) { + const { + bom, + comment_no_infix, + delimiter_auto, + encoding, + from_line, + ltrim, + max_record_size, + raw, + relax_quotes, + rtrim, + skip_empty_lines, + to, + to_line, + } = this.options; + let { comment, escape, quote, record_delimiter } = this.options; + const { + bomSkipped, + delimiterDiscovered, + delimiterBufPrevious, + rawBuffer, + escapeIsQuote, + } = this.state; + // Automatic delimiter discovery + if (!delimiterDiscovered && delimiter_auto) { + let delimiterBuf; + if (delimiterBufPrevious === undefined) { + delimiterBuf = nextBuf; + } else if ( + delimiterBufPrevious !== undefined && + nextBuf === undefined + ) { + delimiterBuf = delimiterBufPrevious; + } else { + delimiterBuf = Buffer.concat([delimiterBufPrevious, nextBuf]); + } + // Ensure that nextBuf is not concatenated a second time during buffer reconciliation + nextBuf = undefined; + // this.delimiterBufPrevious = delimiterBuf; + if (end || delimiterBuf.length > delimiter_auto.size) { + this.options.delimiter = [ + Buffer.from( + delimiter_discover(delimiterBuf, this.options.delimiter_auto), + ), + ]; + this.state.previousBuf = delimiterBuf; + this.state.delimiterBufPrevious = undefined; + this.state.delimiterDiscovered = true; + } else { + this.state.delimiterBufPrevious = delimiterBuf; + return; + } + } + // Previous buffers reconciliation + const { previousBuf } = this.state; + let buf; + if (previousBuf === undefined) { + if (nextBuf === undefined) { + // Handle empty string + close(); + return; + } else { + buf = nextBuf; + } + } else if (previousBuf !== undefined && nextBuf === undefined) { + buf = previousBuf; + } else { + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if (bomSkipped === false) { + if (bom === false) { + this.state.bomSkipped = true; + } else if (buf.length < 3) { + // No enough data + if (end === false) { + // Wait for more data + this.state.previousBuf = buf; + return; + } + } else { + for (const encoding in boms) { + if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) { + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + const options = normalize_options({ + ...this.original_options, + encoding: encoding, + }); + // Properties are merged with the existing options instance + for (const key in options) { + this.options[key] = options[key]; + } + // Options will re-evaluate the Buffer with the new encoding + ({ comment, escape, quote } = this.options); + break; + } + } + this.state.bomSkipped = true; + } + } + const bufLen = buf.length; + let pos; + for (pos = 0; pos < bufLen; pos++) { + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if (this.__needMoreData(pos, bufLen, end)) { + break; + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if (to_line !== -1 && this.info.lines > to_line) { + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if (this.state.quoting === false && record_delimiter.length === 0) { + const record_delimiterCount = this.__autoDiscoverRecordDelimiter( + buf, + pos, + ); + if (record_delimiterCount) { + record_delimiter = this.options.record_delimiter; + } + } + const chr = buf[pos]; + if (raw === true) { + rawBuffer.append(chr); + } + if ( + (chr === cr || chr === nl) && + this.state.wasRowDelimiter === false + ) { + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if (this.state.escaping === true) { + this.state.escaping = false; + } else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if ( + escape !== null && + this.state.quoting === true && + this.__isEscape(buf, pos, chr) && + pos + escape.length < bufLen + ) { + if (escapeIsQuote) { + if (this.__isQuote(buf, pos + escape.length)) { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } else { + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if (this.state.commenting === false && this.__isQuote(buf, pos)) { + if (this.state.quoting === true) { + const nextChr = buf[pos + quote.length]; + const isNextChrTrimable = + rtrim && this.__isCharTrimable(buf, pos + quote.length); + const isNextChrComment = + comment !== null && + this.__compareBytes(comment, buf, pos + quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter( + buf, + pos + quote.length, + nextChr, + ); + const isNextChrRecordDelimiter = + record_delimiter.length === 0 + ? this.__autoDiscoverRecordDelimiter(buf, pos + quote.length) + : this.__isRecordDelimiter(nextChr, buf, pos + quote.length); + // Escape a quote + // Treat next char as a regular character + if ( + escape !== null && + this.__isEscape(buf, pos, chr) && + this.__isQuote(buf, pos + escape.length) + ) { + pos += escape.length - 1; + } else if ( + !nextChr || + isNextChrDelimiter || + isNextChrRecordDelimiter || + isNextChrComment || + isNextChrTrimable + ) { + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + } else if (relax_quotes === false) { + const err = this.__error( + new CsvError( + "CSV_INVALID_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + "instead of delimiter, record delimiter, trimable character", + "(if activated) or comment", + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; + } + } else { + if (this.state.field.length !== 0) { + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if (relax_quotes === false) { + const info = this.__infoField(); + const bom = Object.keys(boms) + .map((b) => + boms[b].equals(this.state.field.toString()) ? b : false, + ) + .filter(Boolean)[0]; + const err = this.__error( + new CsvError( + "INVALID_OPENING_QUOTE", + [ + "Invalid Opening Quote:", + `a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`, + bom ? `(${bom} bom)` : undefined, + ], + this.options, + info, + { + field: this.state.field, + }, + ), + ); + if (err !== undefined) return err; + } + } else { + this.state.quoting = true; + pos += quote.length - 1; + continue; + } + } + } + if (this.state.quoting === false) { + const recordDelimiterLength = this.__isRecordDelimiter( + chr, + buf, + pos, + ); + if (recordDelimiterLength !== 0) { + // Do not emit comments which take a full line + const skipCommentLine = + this.state.commenting && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0; + if (skipCommentLine) { + this.info.comment_lines++; + // Skip full comment line + } else { + // Activate records emission if above from_line + if ( + this.state.enabled === false && + this.info.lines + + (this.state.wasRowDelimiter === true ? 1 : 0) >= + from_line + ) { + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if ( + skip_empty_lines === true && + this.state.wasQuoting === false && + this.state.record.length === 0 && + this.state.field.length === 0 + ) { + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + this.info.bytes = + this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + if (to !== -1 && this.info.records >= to) { + this.state.stop = true; + close(); + return; + } + } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if (this.state.commenting) { + continue; + } + if ( + comment !== null && + (comment_no_infix === false || + (this.state.record.length === 0 && + this.state.field.length === 0)) + ) { + const commentCount = this.__compareBytes(comment, buf, pos, chr); + if (commentCount !== 0) { + this.state.commenting = true; + continue; + } + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if (delimiterLength !== 0) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + pos += delimiterLength - 1; + continue; + } + } + } + if (this.state.commenting === false) { + if ( + max_record_size !== 0 && + this.state.record_length + this.state.field.length > max_record_size + ) { + return this.__error( + new CsvError( + "CSV_MAX_RECORD_SIZE", + [ + "Max Record Size:", + "record exceed the maximum number of tolerated bytes", + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } + } + const lappend = + ltrim === false || + this.state.quoting === true || + this.state.field.length !== 0 || + !this.__isCharTrimable(buf, pos); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if (lappend === true && rappend === true) { + this.state.field.append(chr); + } else if (rtrim === true && !this.__isCharTrimable(buf, pos)) { + return this.__error( + new CsvError( + "CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE", + [ + "Invalid Closing Quote:", + "found non trimable byte after quote", + `at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + } else { + if (lappend === false) { + pos += this.__isCharTrimable(buf, pos) - 1; + } + continue; + } + } + if (end === true) { + // Ensure we are not ending in a quoting state + if (this.state.quoting === true) { + const err = this.__error( + new CsvError( + "CSV_QUOTE_NOT_CLOSED", + [ + "Quote Not Closed:", + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + ), + ); + if (err !== undefined) return err; + } else { + // Skip last line if it has no characters + if ( + this.state.wasQuoting === true || + this.state.record.length !== 0 || + this.state.field.length !== 0 + ) { + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if (errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if (errRecord !== undefined) return errRecord; + } else if (this.state.wasRowDelimiter === true) { + this.info.empty_lines++; + } else if (this.state.commenting === true) { + this.info.comment_lines++; + } + } + } else { + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if (this.state.wasRowDelimiter === true) { + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function (push) { + const { + columns, + group_columns_by_name, + encoding, + info, + from, + relax_column_count, + relax_column_count_less, + relax_column_count_more, + raw, + skip_records_with_empty_values, + } = this.options; + const { enabled, record } = this.state; + if (enabled === false) { + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if (columns === true) { + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if (columns === false && this.info.records === 0) { + this.state.expectedRecordLength = recordLength; + } + if (recordLength !== this.state.expectedRecordLength) { + const err = + columns === false + ? new CsvError( + "CSV_RECORD_INCONSISTENT_FIELDS_LENGTH", + [ + "Invalid Record Length:", + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ) + : new CsvError( + "CSV_RECORD_INCONSISTENT_COLUMNS", + [ + "Invalid Record Length:", + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], + this.options, + this.__infoField(), + { + record: record, + }, + ); + if ( + relax_column_count === true || + (relax_column_count_less === true && + recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && + recordLength > this.state.expectedRecordLength) + ) { + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + } else { + const finalErr = this.__error(err); + if (finalErr) return finalErr; + } + } + if (skip_records_with_empty_values === true && isRecordEmpty(record)) { + this.__resetRecord(); + return; + } + if (this.state.recordHasError === true) { + this.__resetRecord(); + this.state.recordHasError = false; + return; + } + this.info.records++; + if (from === 1 || this.info.records >= from) { + const { objname } = this.options; + // With columns, records are object + if (columns !== false) { + const obj = {}; + // Transform record array to an object + for (let i = 0, l = record.length; i < l; i++) { + if (columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if ( + group_columns_by_name === true && + Object.hasOwn(obj, columns[i].name) + ) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } + } else { + Object.defineProperty(obj, columns[i].name, { + value: record[i], + enumerable: true, + writable: true, + configurable: true, + }); + } + } + // Without objname (default) + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: obj }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj], + push, + ); + if (err) { + return err; + } + } + // Without columns, records are array + } else { + if (raw === true || info === true) { + const extRecord = Object.assign( + { record: record }, + raw === true + ? { raw: this.state.rawBuffer.toString(encoding) } + : {}, + info === true ? { info: this.__infoRecord() } : {}, + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord], + push, + ); + if (err) { + return err; + } + } else { + const err = this.__push( + objname === undefined ? record : [record[objname], record], + push, + ); + if (err) { + return err; + } + } + } + } + this.__resetRecord(); + }, + __firstLineToColumns: function (record) { + const { firstLineToHeaders } = this.state; + try { + const headers = + firstLineToHeaders === undefined + ? record + : firstLineToHeaders.call(null, record); + if (!Array.isArray(headers)) { + return this.__error( + new CsvError( + "CSV_INVALID_COLUMN_MAPPING", + [ + "Invalid Column Mapping:", + "expect an array from column function,", + `got ${JSON.stringify(headers)}`, + ], + this.options, + this.__infoField(), + { + headers: headers, + }, + ), + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; + } catch (err) { + return err; + } + }, + __resetRecord: function () { + if (this.options.raw === true) { + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function () { + const { cast, encoding, rtrim, max_record_size } = this.options; + const { enabled, wasQuoting } = this.state; + // Short circuit for the from_line options + if (enabled === false) { + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if (rtrim === true && wasQuoting === false) { + field = field.trimRight(); + } + if (cast === true) { + const [err, f] = this.__cast(field); + if (err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if (max_record_size !== 0 && typeof field === "string") { + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function () { + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function (record, push) { + const { on_record } = this.options; + if (on_record !== undefined) { + const info = this.__infoRecord(); + try { + record = on_record.call(null, record, info); + } catch (err) { + return err; + } + if (record === undefined || record === null) { + return; + } + } + this.info.bytes_records += this.info.bytes; + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function (field) { + const { columns, relax_column_count } = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if ( + isColumns === true && + relax_column_count && + this.options.columns.length <= this.state.record.length + ) { + return [undefined, undefined]; + } + if (this.state.castField !== null) { + try { + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + } catch (err) { + return [err]; + } + } + if (this.__isFloat(field)) { + return [undefined, Number(field)]; + } else if (this.options.cast_date !== false) { + const info = this.__infoField(); + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + __compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) { + if (sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for (let i = 1; i < sourceLength; i++) { + if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0; + } + return sourceLength; + }, + // Helper to test if a character is trimable + __isCharTrimable: function (buf, pos) { + const { timchars, timcharFirstBytes } = this.state; + // Fast bail-out: non-whitespace bytes (the common case) are rejected + // without scanning the full timchar list. + const first = buf[pos]; + if (first === undefined || timcharFirstBytes[first] === 0) return 0; + loop1: for (let i = 0; i < timchars.length; i++) { + const timchar = timchars[i]; + for (let j = 0; j < timchar.length; j++) { + if (timchar[j] !== buf[pos + j]) continue loop1; + } + return timchar.length; + } + return 0; + }, + __isDelimiter: function (buf, pos, chr) { + const { delimiter, ignore_last_delimiters } = this.options; + if ( + ignore_last_delimiters === true && + this.state.record.length === this.options.columns.length - 1 + ) { + return 0; + } else if ( + ignore_last_delimiters !== false && + typeof ignore_last_delimiters === "number" && + this.state.record.length === ignore_last_delimiters - 1 + ) { + return 0; + } + loop1: for (let i = 0; i < delimiter.length; i++) { + const del = delimiter[i]; + if (del[0] === chr) { + for (let j = 1; j < del.length; j++) { + if (del[j] !== buf[pos + j]) continue loop1; + } + return del.length; + } + } + return 0; + }, + __isEscape: function (buf, pos, chr) { + const { escape } = this.options; + if (escape === null) return false; + const l = escape.length; + if (escape[0] === chr) { + for (let i = 0; i < l; i++) { + if (escape[i] !== buf[pos + i]) { + return false; + } + } + return true; + } + return false; + }, + __isFloat: function (value) { + return value - parseFloat(value) + 1 >= 0; // Borrowed from jquery + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isQuote: function (buf, pos) { + const { quote } = this.options; + if (quote === null) return false; + const l = quote.length; + for (let i = 0; i < l; i++) { + if (quote[i] !== buf[pos + i]) { + return false; + } + } + return true; + }, + __isRecordDelimiter: function (chr, buf, pos) { + const { record_delimiter } = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for (let i = 0; i < recordDelimiterLength; i++) { + const rd = record_delimiter[i]; + const rdLength = rd.length; + if (rd[0] !== chr) { + continue; + } + for (let j = 1; j < rdLength; j++) { + if (rd[j] !== buf[pos + j]) { + continue loop1; + } + } + return rd.length; + } + return 0; + }, + __autoDiscoverRecordDelimiter: function (buf, pos) { + const { encoding } = this.options; + // Note, we don't need to cache this information in state, + // It is only called on the first line until we find out a suitable + // record delimiter. + const rds = [ + // Important, the windows line ending must be before mac os 9 + Buffer.from("\r\n", encoding), + Buffer.from("\n", encoding), + Buffer.from("\r", encoding), + ]; + loop: for (let i = 0; i < rds.length; i++) { + const l = rds[i].length; + for (let j = 0; j < l; j++) { + if (rds[i][j] !== buf[pos + j]) { + continue loop; + } + } + this.options.record_delimiter.push(rds[i]); + this.state.recordDelimiterMaxLength = rds[i].length; + return rds[i].length; + } + return 0; + }, + __error: function (msg) { + const { encoding, raw, skip_records_with_error } = this.options; + const err = typeof msg === "string" ? new Error(msg) : msg; + if (skip_records_with_error) { + this.state.recordHasError = true; + if (this.options.on_skip !== undefined) { + try { + this.options.on_skip( + err, + raw ? this.state.rawBuffer.toString(encoding) : undefined, + ); + } catch (err) { + return err; + } + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + } else { + return err; + } + }, + __infoDataSet: function () { + return { + ...this.info, + columns: this.options.columns, + }; + }, + __infoRecord: function () { + const { columns, raw, encoding } = this.options; + return { + ...this.__infoDataSet(), + bytes_records: this.info.bytes, + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined, + }; + }, + __infoField: function () { + const { columns } = this.options; + const isColumns = Array.isArray(columns); + // Bytes records are only incremented when all records'fields are parsed + const bytes_records = this.info.bytes_records; + return { + ...this.__infoRecord(), + bytes_records: bytes_records, + column: + isColumns === true + ? columns.length > this.state.record.length + ? columns[this.state.record.length].name + : null + : this.state.record.length, + quoting: this.state.wasQuoting, + }; + }, + }; + }; + + // Parse an asynchronous iterable without a stream wrapper. Drain each chunk's + // records before requesting another chunk, including records preceding an error. + async function* parse(input, opts = {}) { + const api = transform(opts); + let records = []; + let stopped = false; + const push = (record) => records.push(record); + const close = () => { + stopped = true; + }; + for await (const chunk of input) { + const buffer = chunk instanceof Buffer ? chunk : Buffer.from(chunk); + const error = api.parse(buffer, false, push, close); + for (const record of records) yield record; + records = []; + if (error !== undefined) throw error; + if (stopped) return; + } + const error = api.parse(undefined, true, push, close); + for (const record of records) yield record; + if (error !== undefined) throw error; + } + + exports.CsvError = CsvError; + exports.normalize_options = normalize_options; + exports.parse = parse; + +})); diff --git a/packages/csv-parse/lib/iterator.d.ts b/packages/csv-parse/lib/iterator.d.ts new file mode 100644 index 000000000..8c5de6241 --- /dev/null +++ b/packages/csv-parse/lib/iterator.d.ts @@ -0,0 +1,20 @@ +import type { + Options, + OptionsWithColumns, + OptionsNormalized, +} from "./options.js"; + +declare function parse( + input: AsyncIterable, + options: OptionsWithColumns, +): AsyncGenerator; +declare function parse( + input: AsyncIterable, + options?: Options, +): AsyncGenerator; + +declare function normalize_options(options: Options): OptionsNormalized; + +export { parse, normalize_options }; +export * from "./api/CsvError.js"; +export type * from "./options.js"; diff --git a/packages/csv-parse/lib/iterator.js b/packages/csv-parse/lib/iterator.js new file mode 100644 index 000000000..da13bf456 --- /dev/null +++ b/packages/csv-parse/lib/iterator.js @@ -0,0 +1,27 @@ +import { CsvError, transform } from "./api/index.js"; +import { normalize_options } from "./api/normalize_options.js"; + +// Parse an asynchronous iterable without a stream wrapper. Drain each chunk's +// records before requesting another chunk, including records preceding an error. +async function* parse(input, opts = {}) { + const api = transform(opts); + let records = []; + let stopped = false; + const push = (record) => records.push(record); + const close = () => { + stopped = true; + }; + for await (const chunk of input) { + const buffer = chunk instanceof Buffer ? chunk : Buffer.from(chunk); + const error = api.parse(buffer, false, push, close); + for (const record of records) yield record; + records = []; + if (error !== undefined) throw error; + if (stopped) return; + } + const error = api.parse(undefined, true, push, close); + for (const record of records) yield record; + if (error !== undefined) throw error; +} + +export { parse, CsvError, normalize_options }; diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 9aafb77c5..8e5e83537 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -52,6 +52,16 @@ "default": "./dist/cjs/sync.cjs" } }, + "./iterator": { + "import": { + "types": "./lib/iterator.d.ts", + "default": "./lib/iterator.js" + }, + "require": { + "types": "./dist/cjs/iterator.d.cts", + "default": "./dist/cjs/iterator.cjs" + } + }, "./stream": { "import": { "types": "./lib/stream.d.ts", @@ -69,6 +79,10 @@ "./browser/esm/sync": { "types": "./dist/esm/sync.d.ts", "default": "./dist/esm/sync.js" + }, + "./browser/esm/iterator": { + "types": "./dist/esm/iterator.d.ts", + "default": "./dist/esm/iterator.js" } }, "devDependencies": { @@ -111,7 +125,7 @@ "scripts": { "build": "npm run build:rollup && npm run build:ts", "build:rollup": "rollup -c", - "build:ts": "mkdir -p dist/cjs/api dist/esm/api && cp lib/index.d.ts dist/cjs/index.d.cts && cp lib/options.d.ts dist/cjs/options.d.cts && cp lib/sync.d.ts dist/cjs/sync.d.cts && cp lib/stream.d.ts dist/cjs/stream.d.cts && cp lib/api/CsvError.d.ts dist/cjs/api/CsvError.d.cts && cp lib/*.ts dist/esm && cp lib/api/*.ts dist/esm/api", + "build:ts": "mkdir -p dist/cjs/api dist/esm/api && cp lib/index.d.ts dist/cjs/index.d.cts && cp lib/options.d.ts dist/cjs/options.d.cts && cp lib/sync.d.ts dist/cjs/sync.d.cts && cp lib/iterator.d.ts dist/cjs/iterator.d.cts && cp lib/stream.d.ts dist/cjs/stream.d.cts && cp lib/api/CsvError.d.ts dist/cjs/api/CsvError.d.cts && cp lib/*.ts dist/esm && cp lib/api/*.ts dist/esm/api", "postbuild:ts": "find dist/cjs -name '*.d.cts' -exec sh -c \"sed -i \"s/\\.js'/\\.cjs'/g\" {} || sed -i '' \"s/\\.js'/\\.cjs'/g\" {}\" \\;", "lint:check": "eslint", "lint:fix": "eslint --fix", @@ -131,11 +145,17 @@ "sync": [ "dist/esm/sync.d.ts" ], + "iterator": [ + "dist/esm/iterator.d.ts" + ], "browser/esm": [ "dist/esm/index.d.ts" ], "browser/esm/sync": [ "dist/esm/sync.d.ts" + ], + "browser/esm/iterator": [ + "dist/esm/iterator.d.ts" ] } } diff --git a/packages/csv-parse/rollup.config.js b/packages/csv-parse/rollup.config.js index 95bfabe0b..874e41a09 100644 --- a/packages/csv-parse/rollup.config.js +++ b/packages/csv-parse/rollup.config.js @@ -114,4 +114,38 @@ export default [ nodeResolve(), ], }, + { + onwarn: function (warning, rollupWarn) { + if (warning.code === "CIRCULAR_DEPENDENCY") return; + rollupWarn(warning); + }, + input: "lib/iterator.js", + output: [ + { + file: `dist/esm/iterator.js`, + format: "esm", + }, + { + file: `dist/iife/iterator.js`, + format: "iife", + name: "csv_parse_iterator", + }, + { + file: `dist/umd/iterator.js`, + format: "umd", + name: "csv_parse_iterator", + }, + ], + plugins: [globals(), builtins(), nodeResolve()], + }, + { + input: "lib/iterator.js", + output: [ + { + file: `dist/cjs/iterator.cjs`, + format: "cjs", + }, + ], + plugins: [nodeResolve()], + }, ]; diff --git a/packages/csv-parse/samples/api.iterator.js b/packages/csv-parse/samples/api.iterator.js new file mode 100644 index 000000000..93cd65499 --- /dev/null +++ b/packages/csv-parse/samples/api.iterator.js @@ -0,0 +1,16 @@ +import assert from "node:assert/strict"; +import { parse } from "csv-parse/iterator"; + +async function* chunks() { + yield "name,age\nAlice,"; + yield new TextEncoder().encode("20\nBob,30"); +} + +const records = []; +for await (const record of parse(chunks(), { columns: true, cast: true })) { + records.push(record); +} +assert.deepEqual(records, [ + { name: "Alice", age: 20 }, + { name: "Bob", age: 30 }, +]); diff --git a/packages/csv-parse/test/api.iterator.js b/packages/csv-parse/test/api.iterator.js new file mode 100644 index 000000000..6f2cb35df --- /dev/null +++ b/packages/csv-parse/test/api.iterator.js @@ -0,0 +1,260 @@ +import assert from "node:assert/strict"; +import { parse, CsvError, normalize_options } from "../lib/iterator.js"; +import { parse as parseSync } from "../lib/sync.js"; +import { parse as parseBrowser } from "../dist/esm/iterator.js"; + +async function* chunks(values) { + for (const value of values) yield value; +} + +async function collect(iterator) { + const records = []; + for await (const record of iterator) records.push(record); + return records; +} + +describe("API iterator", function () { + it("emits every record in a chunk and the final unterminated record", async function () { + const input = "a,b\nc,d\ne,f"; + assert.deepEqual(await collect(parse(chunks([input]))), parseSync(input)); + }); + + it("handles empty input and empty chunks", async function () { + assert.deepEqual(await collect(parse(chunks([]))), []); + assert.deepEqual( + await collect(parse(chunks(["", Buffer.alloc(0), new Uint8Array()]))), + [], + ); + }); + + it("accepts strings, buffers and Uint8Array views", async function () { + const bytes = new TextEncoder().encode("unused\nc,d\nunused"); + const view = bytes.subarray(7, 11); + assert.deepEqual( + await collect(parse(chunks(["a,b\n", view, Buffer.from("e,f")]))), + [ + ["a", "b"], + ["c", "d"], + ["e", "f"], + ], + ); + }); + + it("preserves BOM, UTF-8 and quoted records at every byte boundary", async function () { + const input = Buffer.from('\ufeffname,note\r\n张三,"café\r\n🍊"\r\nZoë,終'); + const options = { bom: true, columns: true }; + const expected = parseSync(input, options); + for (let cut = 0; cut <= input.length; cut++) { + assert.deepEqual( + await collect( + parse(chunks([input.subarray(0, cut), input.subarray(cut)]), options), + ), + expected, + `byte boundary ${cut}`, + ); + } + }); + + it("uses the core encoding and custom delimiter options", async function () { + const input = Buffer.from( + "\ufeffname::value\r\n张三::1\r\nZoë::2", + "utf16le", + ); + const options = { bom: true, delimiter: "::", columns: true, cast: true }; + const expected = parseSync(input, options); + for (let cut = 0; cut <= input.length; cut++) { + assert.deepEqual( + await collect( + parse(chunks([input.subarray(0, cut), input.subarray(cut)]), options), + ), + expected, + `byte boundary ${cut}`, + ); + } + }); + + it("accepts native Buffer chunks in the browser distribution", async function () { + const input = Buffer.from("\ufeffname,note\n张三,🍊\nZoë,café"); + const bytes = Array.from(input, (byte) => Buffer.from([byte])); + assert.deepEqual( + await collect(parseBrowser(chunks(bytes), { bom: true, columns: true })), + parseSync(input, { bom: true, columns: true }), + ); + }); + + it("supports raw, info and binary field output", async function () { + const input = Buffer.from("a,b\nc,d"); + const options = { raw: true, info: true, encoding: null }; + assert.deepEqual( + await collect(parse(chunks([input]), options)), + parseSync(input, options), + ); + }); + + it("yields objname tuples like the streaming API", async function () { + const input = "id,value\na,1\nb,2"; + const options = { columns: true, objname: "id" }; + assert.deepEqual( + await collect(parse(chunks([input]), options)), + Object.entries(parseSync(input, options)), + ); + }); + + it("applies columns, cast, comments and on_record", async function () { + const input = "# ignored\nname,value\n Alice ,1\n Bob ,2\n"; + const options = { + columns: true, + cast: true, + comment: "#", + trim: true, + on_record: (record) => (record.value === 1 ? null : record), + }; + assert.deepEqual( + await collect(parse(chunks([input]), options)), + parseSync(input, options), + ); + }); + + it("preserves skip callbacks and recoverable errors", async function () { + const skipped = []; + const options = { + skip_records_with_error: true, + on_skip: (error) => skipped.push(error.code), + }; + assert.deepEqual(await collect(parse(chunks(["a,b\nx\nc,d\n"]), options)), [ + ["a", "b"], + ["c", "d"], + ]); + assert.deepEqual(skipped, ["CSV_RECORD_INCONSISTENT_FIELDS_LENGTH"]); + }); + + it("is lazy and drains a chunk before pulling the next", async function () { + let reads = 0; + async function* input() { + reads++; + yield "first\nsecond\nthird\n"; + reads++; + yield "fourth\n"; + } + const records = parse(input()); + assert.equal(reads, 0); + assert.deepEqual((await records.next()).value, ["first"]); + assert.deepEqual((await records.next()).value, ["second"]); + assert.equal(reads, 1); + await records.return(); + }); + + it("closes the source on consumer break with queued records", async function () { + let reads = 0; + let closed = false; + async function* input() { + try { + reads++; + yield "a,b\nc,d\ne,f\n"; + reads++; + yield "g,h\n"; + } finally { + await Promise.resolve(); + closed = true; + } + } + for await (const record of parse(input())) { + assert.deepEqual(record, ["a", "b"]); + break; + } + assert.equal(reads, 1); + assert.equal(closed, true); + }); + + it("awaits cleanup on explicit consumer return", async function () { + let closed = false; + async function* input() { + try { + yield "a,b\nc,d\ne,f\n"; + } finally { + await Promise.resolve(); + closed = true; + } + } + const records = parse(input()); + await records.next(); + assert.deepEqual(await records.return(), { done: true, value: undefined }); + assert.equal(closed, true); + }); + + it("propagates a source error", async function () { + const error = new Error("source failed"); + async function* input() { + yield "a,b\nc,d\n"; + throw error; + } + await assert.rejects(collect(parse(input())), (caught) => caught === error); + }); + + it("yields pre-error records and closes iterators without throw", async function () { + let reads = 0; + let returns = 0; + const input = { + [Symbol.asyncIterator]() { + return this; + }, + async next() { + reads++; + return { done: false, value: "a,b\nc,d\nx\nnext,row\n" }; + }, + async return() { + await Promise.resolve(); + returns++; + return { done: true }; + }, + }; + const records = []; + await assert.rejects( + async () => { + for await (const record of parse(input)) records.push(record); + }, + { code: "CSV_RECORD_INCONSISTENT_FIELDS_LENGTH" }, + ); + assert.deepEqual(records, [ + ["a", "b"], + ["c", "d"], + ]); + assert.equal(reads, 1); + assert.equal(returns, 1); + }); + + it("reports errors from the EOF flush", async function () { + await assert.rejects(collect(parse(chunks(['a,b\n"unclosed']))), { + code: "CSV_QUOTE_NOT_CLOSED", + }); + }); + + it("stops and closes the source at record or line limits", async function () { + for (const options of [{ to: 2 }, { to_line: 2 }]) { + let reads = 0; + let closed = false; + async function* input() { + try { + reads++; + yield "1\n2\n3\n4\n"; + reads++; + throw new Error("must not read past the limit"); + } finally { + await Promise.resolve(); + closed = true; + } + } + assert.deepEqual(await collect(parse(input(), options)), [["1"], ["2"]]); + assert.equal(reads, 1); + assert.equal(closed, true); + } + }); + + it("exports the common error and normalization helpers", async function () { + assert.equal( + normalize_options({ delimiter: ":" }).delimiter[0].toString(), + ":", + ); + await assert.rejects(collect(parse(chunks(['"unclosed']))), CsvError); + }); +}); diff --git a/packages/csv-parse/test/api.types.iterator.ts b/packages/csv-parse/test/api.types.iterator.ts new file mode 100644 index 000000000..394c0b6ac --- /dev/null +++ b/packages/csv-parse/test/api.types.iterator.ts @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import { parse, CsvError, normalize_options } from "../lib/iterator.js"; +import type { Options, InfoRecord } from "../lib/iterator.js"; + +async function* input() { + yield "name,age\nAlice,20"; +} + +describe("API iterator types", function () { + it("returns an async generator of default records", async function () { + const records: AsyncGenerator = parse(input()); + assert.deepEqual((await records.next()).value, ["name", "age"]); + await records.return(); + }); + + it("supports typed columns and on_record", async function () { + interface Person { + name: string; + age: number; + } + interface Name { + name: string; + } + const options: Options = { + columns: true, + cast: true, + on_record: (person: Person, context: InfoRecord) => ({ + name: `${person.name}:${context.records}`, + }), + }; + const records: AsyncGenerator = parse( + input(), + options, + ); + assert.deepEqual((await records.next()).value, { name: "Alice:1" }); + }); + + it("uses non-stream options and exports helpers", function () { + const options: Options = { delimiter: ":" }; + normalize_options(options); + const error: CsvError = new CsvError("CSV_INVALID_ARGUMENT", "test"); + assert.equal(error.code, "CSV_INVALID_ARGUMENT"); + // @ts-expect-error The portable entry does not accept Node stream options. + parse(input(), { highWaterMark: 1 }); + }); +}); diff --git a/packages/csv-parse/test/option.group_columns_by_name.js b/packages/csv-parse/test/option.group_columns_by_name.js index b5a58ba4d..02c27bd10 100644 --- a/packages/csv-parse/test/option.group_columns_by_name.js +++ b/packages/csv-parse/test/option.group_columns_by_name.js @@ -8,7 +8,7 @@ describe("Option `group_columns_by_name`", function () { }).should.throw({ code: "CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME", message: - 'Invalid option group_columns_by_name: expect an boolean, got "invalid"', + 'Invalid option group_columns_by_name: expect a boolean, got "invalid"', }); }); });