From 1c76418771405a994c0185ae376de09447babbc1 Mon Sep 17 00:00:00 2001 From: David Given Date: Tue, 3 Nov 2026 20:14:37 +0100 Subject: [PATCH 1/3] Update ab. --- build/ab.mk | 16 +++++ build/ab.py | 165 ++++++++++++++++++++++++++++++++++------------------ build/c.py | 2 +- 3 files changed, 125 insertions(+), 58 deletions(-) diff --git a/build/ab.mk b/build/ab.mk index e8a9aa501..e84377cc1 100644 --- a/build/ab.mk +++ b/build/ab.mk @@ -25,7 +25,14 @@ CFLAGS ?= $(HOSTCFLAGS) CXXFLAGS ?= $(CFLAGS) LDFLAGS ?= $(HOSTLDFLAGS) +DMD ?= ldmd2 +DFLAGS ?= -g -O0 + +HOSTDMD ?= $(DMD) +HOSTDFLAGS ?= $(DFLAGS) + NINJA ?= ninja +BWRAP ?= bwrap ifdef VERBOSE hide = @@ -81,6 +88,15 @@ pkg-config-hash = $(shell ($(PKG_CONFIG) --list-all && $(HOST_PKG_CONFIG) --list build-files = $(shell find . -name .obj -prune -o \( -name 'build.py' -a -type f \) -print) $(wildcard build/*.py) $(wildcard config.py) build-file-timestamps = $(shell ls -l $(build-files) | md5sum) +# Detect the presence of bubblewrap. + +ifneq ($(shell command -v $(BWRAP) >/dev/null; echo $$?),0) + ifneq ("$(AB_SANDBOX)","no") + $(shell echo "bubblewrap not detected; disabling sandboxing" >&2) + AB_SANDBOX := no + endif +endif + # Wipe the build file (forcing a regeneration) if the make environment is different. # (Conveniently, this includes the pkg-config hash calculated above.) diff --git a/build/ab.py b/build/ab.py index 81e3c9fff..54d942e83 100644 --- a/build/ab.py +++ b/build/ab.py @@ -18,6 +18,7 @@ import string import sys import types +import shlex VERBOSE_NINJA_FILE = False @@ -115,6 +116,17 @@ def _undo_escaped_dollar(s, op): class BracketedFormatter(string.Formatter): def parse(self, format_string): + return self._parse_cached(format_string) + + @staticmethod + @functools.lru_cache(maxsize=4096) + def _parse_cached(format_string): + # Cache a reusable tuple, rather than an exhaustible generator. Parsing + # is target-independent; expression values must still be evaluated live. + return tuple(BracketedFormatter._parse_parts(format_string)) + + @staticmethod + def _parse_parts(format_string): while format_string: m = re.search(f"(?:[^$]|^)()\\$\\[()", format_string) if not m: @@ -149,6 +161,15 @@ def parse(self, format_string): class GlobalFormatter(string.Formatter): def parse(self, format_string): + return self._parse_cached(format_string) + + @staticmethod + @functools.lru_cache(maxsize=4096) + def _parse_cached(format_string): + return tuple(GlobalFormatter._parse_parts(format_string)) + + @staticmethod + def _parse_parts(format_string): while format_string: m = re.search(f"(?:[^$]|^)()\\$\\(([^)]*)\\)()", format_string) if not m: @@ -186,6 +207,8 @@ def format_field(self, value, format_spec): def substituteGlobalVariables(value): + if "$(" not in value: + return value while True: oldValue = value value = globalFormatter.format(value) @@ -253,6 +276,40 @@ def _isiterable(xs): ) +@functools.lru_cache(maxsize=4096) +def _compile_expression(expression): + # eval(str) skips leading spaces and tabs before compiling its expression. + return compile(expression.lstrip(" \t"), "", "eval") + + +class TargetFormatter(BracketedFormatter): + def __init__(self, target): + self.target = target + + def get_field(self, name, a1, a2): + return ( + eval( + _compile_expression(name), + self.target.callback.__globals__, + self.target.args, + ), + False, + ) + + def format_field(self, value, format_spec): + if not value: + return "" + if type(value) == str: + return value + if _isiterable(value): + value = list(value) + if type(value) != list: + value = [value] + return " ".join( + [self.target.templateexpand(f) for f in filenamesof(value)] + ) + + class Target: def __init__(self, cwd, name): self.name = name @@ -277,28 +334,10 @@ def __hash__(self): def __repr__(self): return f"Target('{self.name}')" - def templateexpand(selfi, s): - class Formatter(BracketedFormatter): - def get_field(self, name, a1, a2): - return ( - eval(name, selfi.callback.__globals__, selfi.args), - False, - ) - - def format_field(self, value, format_spec): - if not value: - return "" - if type(value) == str: - return value - if _isiterable(value): - value = list(value) - if type(value) != list: - value = [value] - return " ".join( - [selfi.templateexpand(f) for f in filenamesof(value)] - ) - - s = Formatter().format(s) + def templateexpand(self, s): + if "$[" not in s: + return substituteGlobalVariables(s) + s = TargetFormatter(self).format(s) return substituteGlobalVariables(s) def materialise(self, replacing=False): @@ -430,7 +469,7 @@ def targetof(value, cwd=None): # At this point we have the fully qualified name of a rule. - (path, target) = value.rsplit("+", 1) + path, target = value.rsplit("+", 1) value = join(path, "+" + target) if value not in targets: # Load the new build file. @@ -526,6 +565,7 @@ def generate(xs): def filenameof(x): + x = targetof(x) xs = filenamesof(x.outs) assert ( len(xs) == 1 @@ -573,46 +613,57 @@ def emit_rule(self, ins, outs, cmds=[], label=None): if outs: os.makedirs(self.dir, exist_ok=True) - rule = [] - if G.AB_SANDBOX == "yes": - sandbox = join(self.dir, "sandbox") - emit(f"rm -rf {sandbox}", into=rule) - emit( - f"{G.PYTHON} build/_sandbox.py --link -s", - sandbox, - *fins, - into=rule, - ) - for c in cmds: - emit(f"(cd {sandbox} &&", c, ")", into=rule) - emit( - f"{G.PYTHON} build/_sandbox.py --export -s", - sandbox, - *fouts, - into=rule, - ) - else: - for c in cmds: - emit(c, into=rule) + # Write the actual commands to a shell script. + rule = [] + for c in cmds: + emit(f"( {c} )", into=rule) ruletext = "".join(rule) - if len(ruletext) > 7000: - rulehash = hashlib.sha1(ruletext.encode()).hexdigest() + rulehash = hashlib.sha1(ruletext.encode()).hexdigest() + + rulef = join(self.dir, f"rule-{rulehash}.sh") + with open(rulef, "wt") as fp: + fp.write("set -e\n") + fp.write(ruletext) - rulef = join(self.dir, f"rule-{rulehash}.sh") - with open(rulef, "wt") as fp: - fp.write("set -e\n") - fp.write(ruletext) + # Generate the invocation of the shell script, either directly or through bubblewrap. - emit("build", *fouts, ":rule", *fins) - emit(" command=sh", rulef) + rule = [] + if G.AB_SANDBOX == "yes": + bwrap = join(self.dir, "bwrap.args") + with open(bwrap, "wb") as fp: + cwd = os.getcwd() + + def robind(f): + ff = join(cwd, f) + ffb = f"{ff}\0".encode("utf-8") + fp.write(b"--ro-bind\0") + fp.write(ffb) + fp.write(ffb) + + fp.write(b"--dev-bind\0/\0/\0") + fp.write(b"--bind\0") + fp.write(f"{cwd}/{self.dir}/sandbox".encode("utf-8") + b"\0") + fp.write(f"{cwd}".encode("utf-8") + b"\0") + for f in fins + [rulef]: + robind(f) + emit(f"rm -rf ", f"{self.dir}/sandbox", into=rule) + emit(f"mkdir -p ", f"{self.dir}/sandbox", into=rule) + + emit(G.BWRAP, f"--args 3 3<", bwrap, "sh", rulef, into=rule) + for f in fouts: + emit(f"cp {self.dir}/sandbox/{f} {f}", into=rule) else: - emit("build", *fouts, ":rule", *fins) - emit( - " command=", - "&&".join([s.strip() for s in rule]).replace("$", "$$"), - ) + emit(f"sh", rulef, into=rule) + + # Now generate the ninja build rule. + + emit("build", *fouts, ":rule", *fins) + emit( + " command=", + "&&".join([s.strip() for s in rule]).replace("$", "$$"), + ) if label: emit(" description=", label) emit("build", name, ":phony", *fouts) diff --git a/build/c.py b/build/c.py index c388494c9..687daec49 100644 --- a/build/c.py +++ b/build/c.py @@ -109,7 +109,7 @@ def cfileimpl(self, name, srcs, deps, suffix, commands, label, toolchain, cflags hdr_files = collectattrs(targets=hdr_deps, name="cheader_files") cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags) - t = simplerule( + simplerule( replaces=self, ins=srcs, deps=other_deps + hdr_files, From f7d1cdeff858f7e5905c107d1561e19513c03066 Mon Sep 17 00:00:00 2001 From: David Given Date: Tue, 3 Nov 2026 21:07:20 +0100 Subject: [PATCH 2/3] Don't crash on error. --- util/LLgen/src/gencode.c | 18 ++++---- util/LLgen/src/main.c | 7 +++- util/LLgen/test_errors.py | 88 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 util/LLgen/test_errors.py diff --git a/util/LLgen/src/gencode.c b/util/LLgen/src/gencode.c index 988cfae1d..2d8d3c0ee 100644 --- a/util/LLgen/src/gencode.c +++ b/util/LLgen/src/gencode.c @@ -50,7 +50,7 @@ static int firsts; /* are there any? */ static int listcount; /* In this file the following routines are defined: */ -static void doclose(FILE *); +static void doclose(FILE **); static int *mk_tokenlist(void); static void genhdr(void); static void opentemp(string); @@ -90,13 +90,14 @@ extern void save_grammar(FILE *f); # define NOPOP -20000 -static void doclose(FILE *f) +static void doclose(FILE **f) { - if (ferror(f) != 0) + if (ferror(*f) != 0) { fatal(0, "Write error on temporary", NULL ); } - fclose(f); + fclose(*f); + *f = NULL; } static int *mk_tokenlist(void) @@ -151,7 +152,7 @@ void gencode(int argc) /* generate code ... */ generate(p); getaction(2); - doclose(fpars); + doclose(&fpars); /* And install */ install(genname(p->f_name), p->f_name); p++; @@ -160,6 +161,7 @@ void gencode(int argc) genrecovery(); fclose(fact); + fact = NULL; } static void opentemp(string str) @@ -196,7 +198,7 @@ static void geninclude(void) fprintf(fpars, "#define %sNONCORR\n", prefix ? prefix : "LL"); } #endif - doclose(fpars); + doclose(&fpars); install(f_include, "."); } @@ -320,7 +322,7 @@ static void genrecovery(void) } fputs("#define LL_NEWMESS\n", f); copyfile(rec_file); - doclose(f); + doclose(&fpars); install(f_rec, "."); } @@ -361,7 +363,7 @@ static void genncrecovery(void) copyfile(nc_incl_file); copyfile(nc_rec_file); - doclose(f); + doclose(&fpars); install(f_nc, "."); } #endif diff --git a/util/LLgen/src/main.c b/util/LLgen/src/main.c index b05efe1d2..2fb90e74f 100644 --- a/util/LLgen/src/main.c +++ b/util/LLgen/src/main.c @@ -202,6 +202,7 @@ int main(int argc, string argv[]) if (nerrors) comfatal(); fclose(fact); + fact = NULL; if (argc-- == 1) { fputs("No code generation for input from standard input\n", @@ -326,11 +327,13 @@ static NORETURN void comfatal(void) if (fact != NULL) { fclose(fact); - remove(f_temp); } + if (f_temp != NULL) + remove(f_temp); if (fpars != NULL) fclose(fpars); - remove(f_pars); + if (f_pars != NULL) + remove(f_pars); exit(EXIT_FAILURE); } diff --git a/util/LLgen/test_errors.py b/util/LLgen/test_errors.py new file mode 100644 index 000000000..ebef83cdd --- /dev/null +++ b/util/LLgen/test_errors.py @@ -0,0 +1,88 @@ +"""LLgen error regressions; set LLGEN to test an alternate executable. + +Build util/LLgen/+LLgen, then run: + python3 -m unittest discover -s util/LLgen -p test_errors.py +""" + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class ErrorCleanupTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.executable = Path(os.environ.get( + "LLGEN", ROOT / ".obj/util/LLgen/+LLgen/LLgen")).resolve() + if not cls.executable.is_file(): + raise unittest.SkipTest("build util/LLgen/+LLgen first, or set LLGEN") + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="llgen-error-test-") + self.addCleanup(self.temporary.cleanup) + self.work = Path(self.temporary.name) + self.tmp = self.work / "tmp" + self.tmp.mkdir() + self.out = self.work / "out" + self.out.mkdir() + (self.work / "input.g").write_text("%start parse, start;\nstart: 'x';\n") + self.env = os.environ.copy() + self.env.update(TMP=str(self.tmp), LLGEN_LIB_DIR=str(ROOT / "util/LLgen/lib")) + + def run_llgen(self, *options, source="input.g", output=None): + result = subprocess.run( + [str(self.executable), "-w", *options, + "-o" + str(output if output is not None else self.out), source], + cwd=self.work, env=self.env, capture_output=True, text=True) + self.assertEqual(list(self.tmp.iterdir()), [], result.stderr) + return result + + def assert_fatal(self, result, diagnostic): + # A signal (e.g. SIGABRT from a double fclose) is a negative return code. + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn(diagnostic, result.stderr) + + def test_missing_output_directory(self): + result = self.run_llgen(output=self.work / "missing") + self.assert_fatal(result, "Cannot open file") + + def test_refusing_to_overwrite_outputs(self): + for filename, options in (("input.c", ()), ("Lpars.h", ()), + ("Lpars.c", ()), ("Lncor.c", ("-n",))): + with self.subTest(filename=filename): + # Keep each failure independent of files generated by earlier cases. + for path in self.out.iterdir(): + path.unlink() + target = self.out / filename + target.write_text("This is not generated by LLgen.\n") + result = self.run_llgen(*options) + self.assert_fatal(result, "not a file generated by LLgen") + self.assertEqual(target.read_text(), "This is not generated by LLgen.\n") + + def test_missing_input_file(self): + self.assert_fatal(self.run_llgen(source="missing.g"), "Cannot open missing.g") + + def test_invalid_grammar(self): + (self.work / "input.g").write_text("start: 'x';\n") + self.assert_fatal(self.run_llgen(), "Missing %start") + + def test_temporary_file_creation_failure(self): + self.env["TMP"] = str(self.work / "missing") + self.assert_fatal(self.run_llgen(), "Cannot create temporary file") + + def test_successful_generation(self): + for options in ((), ("-n",)): + with self.subTest(options=options): + result = self.run_llgen(*options) + self.assertEqual(result.returncode, 0, result.stderr) + for filename in ("input.c", "Lpars.h", "Lpars.c"): + self.assertTrue((self.out / filename).is_file(), filename) + + +if __name__ == "__main__": + unittest.main() From a71bef0febc4bb219d4c2ab432fdecde3268a734 Mon Sep 17 00:00:00 2001 From: David Given Date: Tue, 3 Nov 2026 21:17:25 +0100 Subject: [PATCH 3/3] Strip out the weird conditional write code from llgen, as it's of no use on modern systems and makes dealing with temporary directories complicated. --- doc/LLgen/LLgen.n | 27 ++++------ util/LLgen/build.py | 4 +- util/LLgen/doc/LLgen.1 | 19 +++---- util/LLgen/doc/LLgen.n | 27 ++++------ util/LLgen/src/check.c | 7 +-- util/LLgen/src/extern.h | 1 - util/LLgen/src/fileio.h | 4 +- util/LLgen/src/gencode.c | 57 ++++++++------------- util/LLgen/src/global.c | 1 - util/LLgen/src/main.c | 104 ++++++-------------------------------- util/LLgen/test_errors.py | 77 +++++++++++++++++++++++++--- 11 files changed, 141 insertions(+), 187 deletions(-) diff --git a/doc/LLgen/LLgen.n b/doc/LLgen/LLgen.n index dd163a464..aea9f7e2e 100644 --- a/doc/LLgen/LLgen.n +++ b/doc/LLgen/LLgen.n @@ -40,12 +40,8 @@ to resolve these ambiguities. The specification can be split into several files, for each of which \fILLgen\fR generates an output file containing the corresponding part of the parser. -Furthermore, only output files that differ from their previous -version are updated. -Other output files are not affected in any -way. -This allows the user to recompile only those output files that have -changed. +All output files are written directly to the output directory and +existing files are overwritten without comparison or identification checks. .PP The subroutine produced by \fILLgen\fR calls a user supplied routine that must return the next token. This way, the input to the @@ -122,9 +118,8 @@ yacc .]). It demonstrates the struggle \fILLparse\fR and other LL(1) parsers have with expressions. -Appendix C gives an example of the \fILLgen\fR features -allowing the user to recompile only those output files that -have changed, using the \fImake\fR program. +Appendix C gives an example of using \fILLgen\fR with the +\fImake\fR program. .[ make .] @@ -612,17 +607,15 @@ The LLgen working environment file, and two other files: \fILpars.c\fR and \fILpars.h\fR. \fILpars.h\fR contains "#-define"s for the tokennames. \fILpars.c\fR contains the error recovery routines and tables. -Only those output files that differ from their previous version -are updated. See appendix C for a possible application of this -feature. +All output files are overwritten on each run. Their names are reported +on standard error. .PP The names of the output files are constructed as follows: -in the input file name, the suffix after the last point is -replaced by a "c". If no point is present in the input file -name, ".c" is appended to it. \fILLgen\fR checks that the -filename constructed this way in fact represents a previous -version, or does not exist already. +in the basename of the input file, the suffix after the last point is +replaced by a "c". If no point is present, ".c" is appended. +All output files are placed in the directory specified by +\fB-o\fR\fIdirectory\fR, or in the current directory if this option is absent. .PP The user must provide some environment to obtain a complete program. diff --git a/util/LLgen/build.py b/util/LLgen/build.py index bf49dd94b..f6c47e9b6 100644 --- a/util/LLgen/build.py +++ b/util/LLgen/build.py @@ -44,9 +44,7 @@ def llgen(self, name, srcs: Targets = []): outs=(["=Lpars.c", "=Lpars.h"] + [f"={f}.c" for f in fs]), deps=["util/LLgen/lib/incl", "util/LLgen/lib/rec"], commands=[ - f"$[ins[0]] {' '.join(filenamesof(srcs))}", - # llgen emits the files into the current directory (of the sandbox). - f"mv *.[ch] $[dir]", + f"$[ins[0]] -o$[dir] {' '.join(filenamesof(srcs))}", ], label="LLGEN", ) diff --git a/util/LLgen/doc/LLgen.1 b/util/LLgen/doc/LLgen.1 index a50b900b3..0a73edc2a 100644 --- a/util/LLgen/doc/LLgen.1 +++ b/util/LLgen/doc/LLgen.1 @@ -4,7 +4,7 @@ .SH NAME LLgen, an extended LL(1) parser generator .SH SYNOPSIS -LLgen [ \-vxwans ] [ \-j[\fInum\fP] ] [ \-l\fInum\fP ] [ \-h\fInum\fP ] file ... +LLgen [ \-o\fIdirectory\fP ] [ \-vxwans ] [ \-j[\fInum\fP] ] [ \-l\fInum\fP ] [ \-h\fInum\fP ] file ... .SH DESCRIPTION \fILLgen\fP converts a context-free grammar into a set of @@ -46,13 +46,9 @@ also generates a file that contains the non-correcting recovery mechanism. .PP \fILLgen\fP -will only update those output files that differ from their previous -version. -This allows -\fILLgen\fP -to be used with -\fImake\fP -(1) convieniently. +writes all output files directly to the output directory, overwriting +existing files without comparing their contents or checking their origin. +Each output filename is reported on standard error. .PP To obtain a working program, the user must also supply a lexical analyzer, as well as @@ -76,9 +72,10 @@ If the flag is given more than once, will be more "verbose". If it is given three times, a complete description of the grammar will be supplied. -.IP \fB\-o\fP -Specifies the location of the output files. If not set, they -are placed in the current directory. +.IP \fB\-o\fP\fIdirectory\fP +Specifies the directory for all output files. The directory must already exist. +The directory name must immediately follow the option, without a space. +If not set, output files are placed in the current directory. .IP \fB\-x\fP the sets that are computed are extended with the nonterminal symbols and these extended sets are also included in the diff --git a/util/LLgen/doc/LLgen.n b/util/LLgen/doc/LLgen.n index 2120aa685..3265c8744 100644 --- a/util/LLgen/doc/LLgen.n +++ b/util/LLgen/doc/LLgen.n @@ -43,12 +43,8 @@ to resolve these ambiguities. The specification can be split into several files, for each of which \fILLgen\fR generates an output file containing the corresponding part of the parser. -Furthermore, only output files that differ from their previous -version are updated. -Other output files are not affected in any -way. -This allows the user to recompile only those output files that have -changed. +All output files are written directly to the output directory and +existing files are overwritten without comparison or identification checks. .PP The subroutine produced by \fILLgen\fR calls a user supplied routine that must return the next token. This way, the input to the @@ -125,9 +121,8 @@ yacc .]). It demonstrates the struggle \fILLparse\fR and other LL(1) parsers have with expressions. -Appendix C gives an example of the \fILLgen\fR features -allowing the user to recompile only those output files that -have changed, using the \fImake\fR program. +Appendix C gives an example of using \fILLgen\fR with the +\fImake\fR program. .[ make .] @@ -615,17 +610,15 @@ The LLgen working environment file, and two other files: \fILpars.c\fR and \fILpars.h\fR. \fILpars.h\fR contains "#-define"s for the tokennames. \fILpars.c\fR contains the error recovery routines and tables. -Only those output files that differ from their previous version -are updated. See appendix C for a possible application of this -feature. +All output files are overwritten on each run. Their names are reported +on standard error. .PP The names of the output files are constructed as follows: -in the input file name, the suffix after the last point is -replaced by a "c". If no point is present in the input file -name, ".c" is appended to it. \fILLgen\fR checks that the -filename constructed this way in fact represents a previous -version, or does not exist already. +in the basename of the input file, the suffix after the last point is +replaced by a "c". If no point is present, ".c" is appended. +All output files are placed in the directory specified by +\fB-o\fR\fIdirectory\fR, or in the current directory if this option is absent. .PP The user must provide some environment to obtain a complete program. diff --git a/util/LLgen/src/check.c b/util/LLgen/src/check.c index b358b86b1..377cfbfb1 100644 --- a/util/LLgen/src/check.c +++ b/util/LLgen/src/check.c @@ -66,10 +66,7 @@ void conflchecks(void) p->n_flags |= VERBOSE; } if (verbose) - { - if ((fout = fopen(f_out, "w")) == NULL) - fatal(1, e_noopen, f_out); - } + fout = openoutput(f_out); /* * Check the rules in the order in which they are declared, * and input file by input file, to give proper error messages @@ -117,7 +114,7 @@ void conflchecks(void) } } if (verbose) - fclose(fout); + closeoutput(&fout); } static void prline(char* s) diff --git a/util/LLgen/src/extern.h b/util/LLgen/src/extern.h index 37b0853f0..6b3e0d61a 100644 --- a/util/LLgen/src/extern.h +++ b/util/LLgen/src/extern.h @@ -124,7 +124,6 @@ extern void error(int lineno,string s,string t); extern void warning(int lineno,string s,string t); extern NORETURN void fatal(int lineno,string s,string t); extern void copyfile(string); -extern void install(string, string); /* name.c */ extern void name_init(void); diff --git a/util/LLgen/src/fileio.h b/util/LLgen/src/fileio.h index c4d21e596..67b174b9c 100644 --- a/util/LLgen/src/fileio.h +++ b/util/LLgen/src/fileio.h @@ -32,7 +32,6 @@ extern FILE* fpars; extern FILE* fact; extern FILE* fout; extern string f_dir; -extern string f_pars; extern string f_temp; extern string f_out; extern string f_input; @@ -41,3 +40,6 @@ extern string f_rec; #ifdef NON_CORRECTING extern string f_nc; #endif + +extern FILE* openoutput(string); +extern void closeoutput(FILE**); diff --git a/util/LLgen/src/gencode.c b/util/LLgen/src/gencode.c index 2d8d3c0ee..f339d2aee 100644 --- a/util/LLgen/src/gencode.c +++ b/util/LLgen/src/gencode.c @@ -50,10 +50,9 @@ static int firsts; /* are there any? */ static int listcount; /* In this file the following routines are defined: */ -static void doclose(FILE **); static int *mk_tokenlist(void); static void genhdr(void); -static void opentemp(string); +static void openparser(string, string); static void geninclude(void); static void genrecovery(void); #ifdef NON_CORRECTING @@ -90,16 +89,6 @@ extern void save_grammar(FILE *f); # define NOPOP -20000 -static void doclose(FILE **f) -{ - if (ferror(*f) != 0) - { - fatal(0, "Write error on temporary", NULL ); - } - fclose(*f); - *f = NULL; -} - static int *mk_tokenlist(void) { int i = ntokens; @@ -125,6 +114,7 @@ static void genhdr(void) void gencode(int argc) { p_file p = files; + string output; /* Set up for code generation */ if ((fact = fopen(f_temp, "r")) == NULL ) @@ -145,16 +135,16 @@ void gencode(int argc) /* For every source file .... */ while (argc--) { - /* Open temporary */ + /* Write the parser directly to the output directory. */ f_input = p->f_name; - opentemp(f_input); + output = genname(f_input); + openparser(output, f_input); + free(output); correct_prefix(); /* generate code ... */ generate(p); getaction(2); - doclose(&fpars); - /* And install */ - install(genname(p->f_name), p->f_name); + closeoutput(&fpars); p++; } geninclude(); @@ -164,16 +154,10 @@ void gencode(int argc) fact = NULL; } -static void opentemp(string str) +static void openparser(string filename, string source) { - - if ((fpars = fopen(f_pars, "w")) == NULL ) - { - fatal(0, e_noopen, f_pars); - } - if (!str) - str = "."; - fprintf(fpars, LLgenid, str); + fpars = openoutput(filename); + fprintf(fpars, LLgenid, source ? source : "."); } static void geninclude(void) @@ -181,7 +165,7 @@ static void geninclude(void) p_token p; int maxno = 0; - opentemp((string) 0); + openparser(f_include, NULL); for (p = tokens; p < maxt; p++) { if (p->t_tokno > maxno) @@ -198,8 +182,7 @@ static void geninclude(void) fprintf(fpars, "#define %sNONCORR\n", prefix ? prefix : "LL"); } #endif - doclose(&fpars); - install(f_include, "."); + closeoutput(&fpars); } static void genrecovery(void) @@ -213,7 +196,7 @@ static void genrecovery(void) int i; p_start st; - opentemp((string) 0); + openparser(f_rec, NULL); f = fpars; correct_prefix(); genhdr(); @@ -322,8 +305,7 @@ static void genrecovery(void) } fputs("#define LL_NEWMESS\n", f); copyfile(rec_file); - doclose(&fpars); - install(f_rec, "."); + closeoutput(&fpars); } #ifdef NON_CORRECTING @@ -336,7 +318,7 @@ static void genncrecovery(void) /* Generate the non-correcting error recovery file */ - opentemp((string) 0); + openparser(f_nc, NULL); f = fpars; genhdr(); @@ -363,8 +345,7 @@ static void genncrecovery(void) copyfile(nc_incl_file); copyfile(nc_rec_file); - doclose(&fpars); - install(f_nc, "."); + closeoutput(&fpars); } #endif @@ -1372,7 +1353,7 @@ static void gencases(int *tokenlist, int caseno, int compacted) */ static string genname(string s) { - string namebuf, c, d; + string namebuf, c, d, output; namebuf = alloc(strlen(s) + 3); c = namebuf; @@ -1397,7 +1378,9 @@ static string genname(string s) *d++ = '.'; *d++ = 'c'; *d++ = '\0'; - return ralloc(namebuf, d - namebuf); + output = aprintf("%s/%s", f_dir, namebuf); + free(namebuf); + return output; } static void genpush(int d) diff --git a/util/LLgen/src/global.c b/util/LLgen/src/global.c index 048de669b..4883530f2 100644 --- a/util/LLgen/src/global.c +++ b/util/LLgen/src/global.c @@ -46,7 +46,6 @@ FILE *fpars; FILE *finput; FILE *fact; string f_dir; -string f_pars; string f_temp; #ifdef NON_CORRECTING string f_nc; diff --git a/util/LLgen/src/main.c b/util/LLgen/src/main.c index 2fb90e74f..5f9c4991c 100644 --- a/util/LLgen/src/main.c +++ b/util/LLgen/src/main.c @@ -39,7 +39,6 @@ int main(int argc, string argv[]) f_dir = "."; f_temp = maketempfile(); - f_pars = maketempfile(); /* Initialize */ @@ -211,7 +210,6 @@ int main(int argc, string argv[]) else gencode(argc); remove(f_temp); - remove(f_pars); if (verbose) { fprintf(stderr, "number of nonterminals: %d\n", nnonterms); @@ -332,8 +330,8 @@ static NORETURN void comfatal(void) remove(f_temp); if (fpars != NULL) fclose(fpars); - if (f_pars != NULL) - remove(f_pars); + if (fout != NULL) + fclose(fout); exit(EXIT_FAILURE); } @@ -347,98 +345,28 @@ void copyfile(string file) if ((f = fopen(file, "r")) == NULL) { - fatal(0, "Cannot open library file %s, call an expert", file); + fatal(0, "Cannot open library file '%s', call an expert", file); } while ((c = getc(f)) != EOF) putc(c, fpars); fclose(f); } -void copyto(string target, string source) +FILE* openoutput(string filename) { - FILE *fsource; - FILE *ftarget; - int c; - - ftarget = fopen(target,"wb+"); - if (ftarget == NULL) - { - fatal(0, "Cannot open file %s, call an expert", target); - } - fsource = fopen(source,"rb"); - if (fsource == NULL) - { - fatal(0, "Cannot open file %s, call an expert", source); - } - while ((c = getc(fsource)) != EOF) - putc(c, ftarget); - fclose(fsource); - fclose(ftarget); -} - -static int isabspath(string path) -{ - if (path[0] == '/') /* Unix paths */ - return 1; - if (path[0] && (path[1] == ':')) /* Windows paths */ - return 1; - return 0; + FILE* f = fopen(filename, "w"); + if (f == NULL) + fatal(0, "Cannot open output file '%s', call an expert", filename); + fprintf(stderr, "%s\n", filename); + return f; } -void install(string target, string source) +void closeoutput(FILE** f) { - /* - * Copy the temporary file generated from source to target - * if allowed (which means that the target must be generated - * by LLgen from the source, or that the target is not present - */ - int c1, c2; - FILE *f1, *f2; - int cnt; - string realtarget = !isabspath(target) ? aprintf("%s/%s", f_dir, target) : target; - - /* - * First open temporary, generated for source - */ - if ((f1 = fopen(f_pars, "r")) == NULL) - { - fatal(0, e_noopen, f_pars); - } - /* - * Now open target for reading - */ - if ((f2 = fopen(realtarget, "r")) == NULL) - { - fclose(f1); - copyto(realtarget, f_pars); - return; - } - /* - * Compute length of LLgen identification string. The target must - * start with that! - */ - cnt = strlen(LLgenid) + strlen(source) - 2; - /* - * Now compare the target with the temporary - */ - do - { - c1 = getc(f1); - c2 = getc(f2); - if (cnt >= 0) - cnt--; - } while (c1 == c2 && c1 != EOF); - fclose(f1); - fclose(f2); - /* - * Here, if c1 != c2 the target must be recreated - */ - if (c1 != c2) - { - if (cnt >= 0) - { - fatal(0, "%s : not a file generated by LLgen", target); - } - copyto(realtarget, f_pars); - } + int failed = ferror(*f); + if (fclose(*f) != 0) + failed = 1; + *f = NULL; + if (failed) + fatal(0, "Write error on output", NULL); } diff --git a/util/LLgen/test_errors.py b/util/LLgen/test_errors.py index ebef83cdd..16216f32e 100644 --- a/util/LLgen/test_errors.py +++ b/util/LLgen/test_errors.py @@ -1,4 +1,4 @@ -"""LLgen error regressions; set LLGEN to test an alternate executable. +"""LLgen output and error regressions; set LLGEN to test an alternate executable. Build util/LLgen/+LLgen, then run: python3 -m unittest discover -s util/LLgen -p test_errors.py @@ -35,9 +35,10 @@ def setUp(self): self.env.update(TMP=str(self.tmp), LLGEN_LIB_DIR=str(ROOT / "util/LLgen/lib")) def run_llgen(self, *options, source="input.g", output=None): + sources = [source] if isinstance(source, str) else list(source) result = subprocess.run( [str(self.executable), "-w", *options, - "-o" + str(output if output is not None else self.out), source], + "-o" + str(output if output is not None else self.out), *sources], cwd=self.work, env=self.env, capture_output=True, text=True) self.assertEqual(list(self.tmp.iterdir()), [], result.stderr) return result @@ -49,9 +50,9 @@ def assert_fatal(self, result, diagnostic): def test_missing_output_directory(self): result = self.run_llgen(output=self.work / "missing") - self.assert_fatal(result, "Cannot open file") + self.assert_fatal(result, "Cannot open output file") - def test_refusing_to_overwrite_outputs(self): + def test_overwrites_existing_outputs_without_identification_checks(self): for filename, options in (("input.c", ()), ("Lpars.h", ()), ("Lpars.c", ()), ("Lncor.c", ("-n",))): with self.subTest(filename=filename): @@ -61,8 +62,8 @@ def test_refusing_to_overwrite_outputs(self): target = self.out / filename target.write_text("This is not generated by LLgen.\n") result = self.run_llgen(*options) - self.assert_fatal(result, "not a file generated by LLgen") - self.assertEqual(target.read_text(), "This is not generated by LLgen.\n") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(target.read_text().startswith("/* LLgen generated code")) def test_missing_input_file(self): self.assert_fatal(self.run_llgen(source="missing.g"), "Cannot open missing.g") @@ -83,6 +84,70 @@ def test_successful_generation(self): for filename in ("input.c", "Lpars.h", "Lpars.c"): self.assertTrue((self.out / filename).is_file(), filename) + def test_reports_outputs_on_every_run(self): + result = self.run_llgen() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(set(result.stderr.splitlines()), + {str(self.out / filename) + for filename in ("input.c", "Lpars.h", "Lpars.c")}) + result = self.run_llgen() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(set(result.stderr.splitlines()), + {str(self.out / filename) + for filename in ("input.c", "Lpars.h", "Lpars.c")}) + + def test_reports_verbose_output_file(self): + result = self.run_llgen("-v") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(str(self.out / "LL.output"), result.stderr.splitlines()) + self.assertTrue((self.out / "LL.output").is_file()) + + def test_all_outputs_use_relative_directory_and_custom_prefix(self): + (self.work / "input.g").write_text( + "%prefix Custom;\n%start parse, start;\nstart: 'x';\n") + output = Path("relative") / "nested" + (self.work / output).mkdir(parents=True) + result = self.run_llgen("-n", "-v", output=output) + self.assertEqual(result.returncode, 0, result.stderr) + names = {"input.c", "Custompars.h", "Custompars.c", "Customncor.c", "Custom.output"} + self.assertEqual({p.name for p in (self.work / output).iterdir()}, names) + self.assertEqual(list(self.work.glob("*.c")), []) + self.assertEqual(list(self.work.glob("*.h")), []) + for name in names: + self.assertIn(str(output / name), result.stderr.splitlines()) + + def test_default_directory(self): + result = self.run_llgen(output=".") + self.assertEqual(result.returncode, 0, result.stderr) + for name in ("input.c", "Lpars.h", "Lpars.c"): + self.assertTrue((self.work / name).is_file(), name) + + def test_multiple_input_files_use_output_directory(self): + (self.work / "tokens.g").write_text("%token TOKEN;\n") + (self.work / "input.g").write_text("%start parse, start;\nstart: TOKEN;\n") + result = self.run_llgen(source=["tokens.g", "input.g"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual({p.name for p in self.out.iterdir()}, + {"tokens.c", "input.c", "Lpars.h", "Lpars.c"}) + self.assertEqual(list(self.work.glob("*.c")), []) + + def test_late_output_open_failure_exits_cleanly(self): + (self.out / "Lpars.h").mkdir() + result = self.run_llgen() + self.assert_fatal(result, "Cannot open output file") + self.assertTrue((self.out / "input.c").is_file()) + + @unittest.skipUnless(Path("/dev/full").exists(), "requires /dev/full") + def test_output_write_failure_exits_cleanly(self): + for filename, options in (("input.c", ()), ("Lpars.h", ()), + ("Lpars.c", ()), ("Lncor.c", ("-n",)), + ("LL.output", ("-vvv",))): + with self.subTest(filename=filename): + for path in self.out.iterdir(): + path.unlink() + (self.out / filename).symlink_to("/dev/full") + self.assert_fatal(self.run_llgen(*options), "Write error on output") + if __name__ == "__main__": unittest.main()