diff --git a/.github/workflows/check-grammars.yml b/.github/workflows/check-grammars.yml new file mode 100644 index 0000000..3e7a4a2 --- /dev/null +++ b/.github/workflows/check-grammars.yml @@ -0,0 +1,58 @@ +name: Check grammars + +# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex, +# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it +# to that, rather than waiting for a language to quietly stop highlighting. +on: + pull_request: + paths: + - components/** + - generate.js + - libprisma/** + - tools/** + - package.json + - package-lock.json + workflow_dispatch: + +permissions: {} + +jobs: + patterns: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: 22 + + # the committed table is regenerated on master, so a pull request is checked against what + # its own components and generate.js produce + - run: npm ci --ignore-scripts + - run: npm start + + - name: Escapes every engine agrees on + run: node tools/check-escapes.js libprisma/grammars.dat + + - name: Compile every pattern with java.util.regex + run: | + javac -d tools tools/PatternProbe.java + java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt + + - name: Compile every pattern with Boost + run: | + sudo apt-get update + sudo apt-get install -y libboost-regex-dev + g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex + ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt + + - name: Compare what the two engines matched + run: | + if ! diff -u boost.txt java.txt > engines.diff; then + echo "The two engines disagree about what these patterns match." + echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt." + head -40 engines.diff + exit 1 + fi + echo "Boost and java.util.regex agree on every pattern." diff --git a/tools/PatternProbe.java b/tools/PatternProbe.java new file mode 100644 index 0000000..92296a6 --- /dev/null +++ b/tools/PatternProbe.java @@ -0,0 +1,163 @@ +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one + * matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to + * the one thing it promises: that a pattern means the same to every client that reads it. + * + * The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so + * anything above U+007F diverges on offsets alone and would drown out the escape bugs this is + * looking for. + */ +public class PatternProbe { + + /** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */ + private static final class Entry { + String source; + int flags; + } + + private static final class Reader { + private final byte[] bytes; + private int offset; + + Reader(byte[] bytes) { + this.bytes = bytes; + } + + int uint8() { + return bytes[offset++] & 0xFF; + } + + int uint16() { + return uint8() | (uint8() << 8); + } + + String string() { + int length = uint8(); + if (length >= 254) { + length = uint8() | (uint8() << 8) | (uint8() << 16); + } + String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1); + offset += length; + return value; + } + } + + private static List read(String path) throws IOException { + Reader reader = new Reader(Files.readAllBytes(Paths.get(path))); + List entries = new ArrayList<>(); + + for (int count = reader.uint16(); count > 0; count--) { + String item = reader.string(); + int begin = item.indexOf('/'); + int end = item.lastIndexOf('/'); + if (begin < 0 || end <= begin) { + continue; + } + + String options = item.substring(end + 1); + int aliasBegin = options.indexOf(','); + + Entry entry = new Entry(); + entry.source = item.substring(begin + 1, end); + for (int i = 0; i < aliasBegin; ++i) { + switch (options.charAt(i)) { + case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break; + case 'm': entry.flags |= Pattern.MULTILINE; break; + default: break; // l and y are libprisma's, for Prism lookbehind and greedy + } + } + entries.add(entry); + } + + return entries; + } + + /** + * Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a + * line terminator that ends it" in java.util.regex. \z is the Java spelling of the first. + * A Java client has to do this; it cannot go in the table, where \z would read as a z. + */ + private static String endOfInput(String source) { + if (source.indexOf('$') < 0) { + return source; + } + + StringBuilder result = new StringBuilder(source.length()); + boolean inClass = false; + for (int i = 0; i < source.length(); ++i) { + char c = source.charAt(i); + if (c == '\\' && i + 1 < source.length()) { + result.append(c).append(source.charAt(++i)); + } else if (c == '[') { + inClass = true; + result.append(c); + } else if (c == ']') { + inClass = false; + result.append(c); + } else if (c == '$' && !inClass) { + result.append("\\z"); + } else { + result.append(c); + } + } + return result.toString(); + } + + public static void main(String[] args) throws IOException { + List entries = read(args[0]); + String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1) + .replace(String.valueOf((char) 13), ""); + + StringBuilder out = new StringBuilder(); + int failed = 0; + + for (int i = 0; i < entries.size(); ++i) { + Entry entry = entries.get(i); + String source = entry.source; + if ((entry.flags & Pattern.MULTILINE) == 0) { + source = endOfInput(source); + } + + Pattern pattern; + try { + pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES); + } catch (PatternSyntaxException e) { + if (failed++ < 20) { + System.err.println("pattern " + i + ": " + e.getDescription()); + System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length()))); + } + continue; + } + + out.append(i); + java.util.regex.Matcher matcher = pattern.matcher(corpus); + int at = 0; + while (matcher.find(at) && out.length() < Integer.MAX_VALUE) { + out.append(' ').append(matcher.start()).append(':').append(matcher.end()); + at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1; + if (at > corpus.length()) { + break; + } + } + out.append('\n'); + } + + Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII)); + System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns"); + + if (failed > 0) { + System.err.println(failed + " patterns java.util.regex will not take. " + + "Teach normalizeEscapes in generate.js to rewrite them."); + System.exit(1); + } + } +} diff --git a/tools/check-escapes.js b/tools/check-escapes.js new file mode 100644 index 0000000..a7d4afe --- /dev/null +++ b/tools/check-escapes.js @@ -0,0 +1,87 @@ +// Every escape in grammars.dat has to mean the same thing to every engine that reads it. +// A pattern that fails to compile is caught by the probes beside this script; this one is for +// the escapes that compile everywhere and quietly mean something else, which is the worse +// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline +// included, by java.util.regex, which made ini values run over the end of the line. +const fs = require('fs') + +// what generate.js is expected to emit, once normalizeEscapes has run over it +const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split('')) + +const REASONS = { + v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B', + h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere', + H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere', + V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere', + R: 'any line break in java.util.regex, a literal R elsewhere', + X: 'a grapheme cluster in java.util.regex, a literal X elsewhere', + A: 'start of input in java.util.regex, a literal A elsewhere', + Z: 'end of input in java.util.regex, a literal Z elsewhere', + z: 'end of input in java.util.regex, a literal z elsewhere', + G: 'end of the previous match in java.util.regex, a literal G elsewhere', + Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere', + E: 'ends one in java.util.regex, a literal E elsewhere', + p: 'a unicode property in both, but the property names differ', + N: 'a named character in java.util.regex', + 0: 'starts an octal escape in java.util.regex - emit \\x00', +} + +function readPatterns(path) { + const buffer = fs.readFileSync(path) + let offset = 0 + + const uint8 = () => buffer[offset++] + const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value } + const string = () => { + let length = uint8() + if (length >= 254) { + length = uint8() | (uint8() << 8) | (uint8() << 16) + } + const value = buffer.toString('latin1', offset, offset + length) + offset += length + return value + } + + const patterns = [] + for (let count = uint16(); count > 0; count--) { + patterns.push(string()) + } + return patterns +} + +const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat') +const problems = [] + +for (const item of patterns) { + const end = item.lastIndexOf('/') + const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end) + + for (let i = 0; i < source.length; i++) { + if (source[i] !== '\\') { + continue + } + + const escaped = source[++i] + if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) { + problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set']) + } + } +} + +console.log(`${patterns.length} patterns checked`) + +if (problems.length === 0) { + process.exit(0) +} + +const seen = new Set() +for (const [source, escape, reason] of problems) { + if (seen.has(escape)) { + continue + } + seen.add(escape) + console.error(`\n${escape} is ${reason}`) + console.error(` e.g. ${source.slice(0, 120)}`) +} +console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`) +process.exit(1) diff --git a/tools/corpus.txt b/tools/corpus.txt new file mode 100644 index 0000000..2086075 --- /dev/null +++ b/tools/corpus.txt @@ -0,0 +1,13 @@ +class Foo where T : IBar { public int x = 1; } /* c */ // line +{{ tpl }} +def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py +let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3; +SELECT * FROM t WHERE a IN (1,2) -- sql +#include +key = value ; ini +[section] +- item: + nested: [1, 2] +0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y + tab and trailing spaces +#!/bin/sh diff --git a/tools/probe.cpp b/tools/probe.cpp new file mode 100644 index 0000000..4881df4 --- /dev/null +++ b/tools/probe.cpp @@ -0,0 +1,132 @@ +// Compiles every pattern of a grammars.dat with Boost, the way libprisma does, and writes down +// what each one matches in a corpus. PatternProbe.java does the same with java.util.regex and +// the two files are diffed, which is what holds the table to meaning one thing everywhere. +// +// The corpus is deliberately ASCII: Boost matches bytes where java.util.regex matches chars, so +// anything above U+007F diverges on offsets alone and would drown out the escape bugs. +#include + +#include "../libprisma/UnicodeEscapes.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + + std::string readAll(const char* path) { + std::ifstream in(path, std::ios::binary); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + } + + struct Reader { + const std::string& bytes; + size_t offset = 0; + + uint8_t uint8() { + return offset < bytes.size() ? static_cast(bytes[offset++]) : 0; + } + + uint16_t uint16() { + uint16_t low = uint8(); + return static_cast(low | (uint8() << 8)); + } + + std::string string() { + size_t length = uint8(); + if (length >= 254) { + size_t a = uint8(), b = uint8(), c = uint8(); + length = a | (b << 8) | (c << 16); + } + std::string value = bytes.substr(offset, length); + offset += length; + return value; + } + }; + + struct Entry { + std::string source; + boost::regex_constants::syntax_option_type flags; + }; + + // "/source/flags,alias,inside", the record LanguageTree::parsePatterns reads + std::vector read(const std::string& table) { + Reader reader{ table }; + std::vector entries; + + for (uint16_t count = reader.uint16(); count > 0; --count) { + const std::string item = reader.string(); + const size_t begin = item.find_first_of('/'); + const size_t end = item.find_last_of('/'); + if (begin == std::string::npos || end <= begin) { + continue; + } + + const std::string options = item.substr(end + 1); + const size_t aliasBegin = options.find_first_of(','); + + Entry entry; + entry.source = item.substr(begin + 1, end - (begin + 1)); + entry.flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m; + + for (size_t i = 0; i < aliasBegin && i < options.size(); ++i) { + switch (options[i]) { + case 'i': entry.flags |= boost::regex_constants::icase; break; + case 'm': entry.flags &= ~boost::regex_constants::no_mod_m; break; + default: break; // l and y are libprisma's, for Prism lookbehind and greedy + } + } + + entries.push_back(entry); + } + + return entries; + } +} + +int main(int argc, char** argv) { + if (argc < 4) { + fprintf(stderr, "usage: probe \n"); + return 2; + } + + const std::vector entries = read(readAll(argv[1])); + + std::string corpus = readAll(argv[2]); + corpus.erase(std::remove(corpus.begin(), corpus.end(), '\r'), corpus.end()); + + std::ofstream out(argv[3], std::ios::binary); + int failed = 0; + + for (size_t i = 0; i < entries.size(); ++i) { + boost::regex re; + try { + re.assign(libprisma::widenUnicodeClasses(entries[i].source), entries[i].flags); + } catch (const std::exception& e) { + if (failed++ < 20) { + fprintf(stderr, "pattern %zu: %s\n %.140s\n", i, e.what(), entries[i].source.c_str()); + } + continue; + } + + out << i; + auto it = boost::sregex_iterator(corpus.begin(), corpus.end(), re, + boost::regex_constants::match_not_dot_newline); + for (auto end = boost::sregex_iterator(); it != end; ++it) { + out << ' ' << it->position() << ':' << (it->position() + it->length()); + } + out << '\n'; + } + + printf("compiled %zu of %zu patterns\n", entries.size() - failed, entries.size()); + + if (failed > 0) { + fprintf(stderr, "%d patterns Boost will not take.\n", failed); + return 1; + } + return 0; +}