Check every pattern against both regex engines on a pull request

.github/workflows/check-grammars.yml

@@ -0,0 +1,58 @@
+name: Check grammars
+
+# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex,
+# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it
+# to that, rather than waiting for a language to quietly stop highlighting.
+on:
+  pull_request:
+    paths:
+      - components/**
+      - generate.js
+      - libprisma/**
+      - tools/**
+      - package.json
+      - package-lock.json
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  patterns:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        persist-credentials: false
+    - uses: actions/setup-node@v5
+      with:
+        node-version: 22
+
+    # the committed table is regenerated on master, so a pull request is checked against what
+    # its own components and generate.js produce
+    - run: npm ci --ignore-scripts
+    - run: npm start
+
+    - name: Escapes every engine agrees on
+      run: node tools/check-escapes.js libprisma/grammars.dat
+
+    - name: Compile every pattern with java.util.regex
+      run: |
+        javac -d tools tools/PatternProbe.java
+        java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt
+
+    - name: Compile every pattern with Boost
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libboost-regex-dev
+        g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex
+        ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt
+
+    - name: Compare what the two engines matched
+      run: |
+        if ! diff -u boost.txt java.txt > engines.diff; then
+          echo "The two engines disagree about what these patterns match."
+          echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt."
+          head -40 engines.diff
+          exit 1
+        fi
+        echo "Boost and java.util.regex agree on every pattern."

tools/PatternProbe.java

@@ -0,0 +1,163 @@
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one
+ * matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to
+ * the one thing it promises: that a pattern means the same to every client that reads it.
+ *
+ * The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so
+ * anything above U+007F diverges on offsets alone and would drown out the escape bugs this is
+ * looking for.
+ */
+public class PatternProbe {
+
+    /** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */
+    private static final class Entry {
+        String source;
+        int flags;
+    }
+
+    private static final class Reader {
+        private final byte[] bytes;
+        private int offset;
+
+        Reader(byte[] bytes) {
+            this.bytes = bytes;
+        }
+
+        int uint8() {
+            return bytes[offset++] & 0xFF;
+        }
+
+        int uint16() {
+            return uint8() | (uint8() << 8);
+        }
+
+        String string() {
+            int length = uint8();
+            if (length >= 254) {
+                length = uint8() | (uint8() << 8) | (uint8() << 16);
+            }
+            String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1);
+            offset += length;
+            return value;
+        }
+    }
+
+    private static List<Entry> read(String path) throws IOException {
+        Reader reader = new Reader(Files.readAllBytes(Paths.get(path)));
+        List<Entry> entries = new ArrayList<>();
+
+        for (int count = reader.uint16(); count > 0; count--) {
+            String item = reader.string();
+            int begin = item.indexOf('/');
+            int end = item.lastIndexOf('/');
+            if (begin < 0 || end <= begin) {
+                continue;
+            }
+
+            String options = item.substring(end + 1);
+            int aliasBegin = options.indexOf(',');
+
+            Entry entry = new Entry();
+            entry.source = item.substring(begin + 1, end);
+            for (int i = 0; i < aliasBegin; ++i) {
+                switch (options.charAt(i)) {
+                    case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break;
+                    case 'm': entry.flags |= Pattern.MULTILINE; break;
+                    default: break;  // l and y are libprisma's, for Prism lookbehind and greedy
+                }
+            }
+            entries.add(entry);
+        }
+
+        return entries;
+    }
+
+    /**
+     * Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a
+     * line terminator that ends it" in java.util.regex. \z is the Java spelling of the first.
+     * A Java client has to do this; it cannot go in the table, where \z would read as a z.
+     */
+    private static String endOfInput(String source) {
+        if (source.indexOf('$') < 0) {
+            return source;
+        }
+
+        StringBuilder result = new StringBuilder(source.length());
+        boolean inClass = false;
+        for (int i = 0; i < source.length(); ++i) {
+            char c = source.charAt(i);
+            if (c == '\\' && i + 1 < source.length()) {
+                result.append(c).append(source.charAt(++i));
+            } else if (c == '[') {
+                inClass = true;
+                result.append(c);
+            } else if (c == ']') {
+                inClass = false;
+                result.append(c);
+            } else if (c == '$' && !inClass) {
+                result.append("\\z");
+            } else {
+                result.append(c);
+            }
+        }
+        return result.toString();
+    }
+
+    public static void main(String[] args) throws IOException {
+        List<Entry> entries = read(args[0]);
+        String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1)
+            .replace(String.valueOf((char) 13), "");
+
+        StringBuilder out = new StringBuilder();
+        int failed = 0;
+
+        for (int i = 0; i < entries.size(); ++i) {
+            Entry entry = entries.get(i);
+            String source = entry.source;
+            if ((entry.flags & Pattern.MULTILINE) == 0) {
+                source = endOfInput(source);
+            }
+
+            Pattern pattern;
+            try {
+                pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES);
+            } catch (PatternSyntaxException e) {
+                if (failed++ < 20) {
+                    System.err.println("pattern " + i + ": " + e.getDescription());
+                    System.err.println("    " + entry.source.substring(0, Math.min(140, entry.source.length())));
+                }
+                continue;
+            }
+
+            out.append(i);
+            java.util.regex.Matcher matcher = pattern.matcher(corpus);
+            int at = 0;
+            while (matcher.find(at) && out.length() < Integer.MAX_VALUE) {
+                out.append(' ').append(matcher.start()).append(':').append(matcher.end());
+                at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1;
+                if (at > corpus.length()) {
+                    break;
+                }
+            }
+            out.append('\n');
+        }
+
+        Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII));
+        System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns");
+
+        if (failed > 0) {
+            System.err.println(failed + " patterns java.util.regex will not take. "
+                + "Teach normalizeEscapes in generate.js to rewrite them.");
+            System.exit(1);
+        }
+    }
+}

tools/check-escapes.js

@@ -0,0 +1,87 @@
+// Every escape in grammars.dat has to mean the same thing to every engine that reads it.
+// A pattern that fails to compile is caught by the probes beside this script; this one is for
+// the escapes that compile everywhere and quietly mean something else, which is the worse
+// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline
+// included, by java.util.regex, which made ini values run over the end of the line.
+const fs = require('fs')
+
+// what generate.js is expected to emit, once normalizeEscapes has run over it
+const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split(''))
+
+const REASONS = {
+    v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B',
+    h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere',
+    H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere',
+    V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere',
+    R: 'any line break in java.util.regex, a literal R elsewhere',
+    X: 'a grapheme cluster in java.util.regex, a literal X elsewhere',
+    A: 'start of input in java.util.regex, a literal A elsewhere',
+    Z: 'end of input in java.util.regex, a literal Z elsewhere',
+    z: 'end of input in java.util.regex, a literal z elsewhere',
+    G: 'end of the previous match in java.util.regex, a literal G elsewhere',
+    Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere',
+    E: 'ends one in java.util.regex, a literal E elsewhere',
+    p: 'a unicode property in both, but the property names differ',
+    N: 'a named character in java.util.regex',
+    0: 'starts an octal escape in java.util.regex - emit \\x00',
+}
+
+function readPatterns(path) {
+    const buffer = fs.readFileSync(path)
+    let offset = 0
+
+    const uint8 = () => buffer[offset++]
+    const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value }
+    const string = () => {
+        let length = uint8()
+        if (length >= 254) {
+            length = uint8() | (uint8() << 8) | (uint8() << 16)
+        }
+        const value = buffer.toString('latin1', offset, offset + length)
+        offset += length
+        return value
+    }
+
+    const patterns = []
+    for (let count = uint16(); count > 0; count--) {
+        patterns.push(string())
+    }
+    return patterns
+}
+
+const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat')
+const problems = []
+
+for (const item of patterns) {
+    const end = item.lastIndexOf('/')
+    const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end)
+
+    for (let i = 0; i < source.length; i++) {
+        if (source[i] !== '\\') {
+            continue
+        }
+
+        const escaped = source[++i]
+        if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) {
+            problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set'])
+        }
+    }
+}
+
+console.log(`${patterns.length} patterns checked`)
+
+if (problems.length === 0) {
+    process.exit(0)
+}
+
+const seen = new Set()
+for (const [source, escape, reason] of problems) {
+    if (seen.has(escape)) {
+        continue
+    }
+    seen.add(escape)
+    console.error(`\n${escape} is ${reason}`)
+    console.error(`  e.g. ${source.slice(0, 120)}`)
+}
+console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`)
+process.exit(1)

tools/corpus.txt

@@ -0,0 +1,13 @@
+class Foo<T> where T : IBar { public int x = 1; } /* c */ // line
+<!DOCTYPE html><a href="x" [attr]='y'>{{ tpl }}</a>
+def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py
+let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3;
+SELECT * FROM t WHERE a IN (1,2) -- sql
+#include <stdio.h>
+key = value ; ini
+[section]
+- item:
+    nested: [1, 2]
+0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y
+	  tab   and trailing spaces
+#!/bin/sh