Update ab.

build/ab.mk

@@ -25,7 +25,14 @@ CFLAGS ?= $(HOSTCFLAGS)
 CXXFLAGS ?= $(CFLAGS)
 LDFLAGS ?= $(HOSTLDFLAGS)
 
+DMD ?= ldmd2
+DFLAGS ?= -g -O0
+
+HOSTDMD ?= $(DMD)
+HOSTDFLAGS ?= $(DFLAGS)
+
 NINJA ?= ninja
+BWRAP ?= bwrap
 
 ifdef VERBOSE
 	hide =
@@ -81,6 +88,15 @@ pkg-config-hash = $(shell ($(PKG_CONFIG) --list-all && $(HOST_PKG_CONFIG) --list
 build-files = $(shell find . -name .obj -prune -o \( -name 'build.py' -a -type f \) -print) $(wildcard build/*.py) $(wildcard config.py)
 build-file-timestamps = $(shell ls -l $(build-files) | md5sum)
 
+# Detect the presence of bubblewrap.
+
+ifneq ($(shell command -v $(BWRAP) >/dev/null; echo $$?),0)
+  ifneq ("$(AB_SANDBOX)","no")
+    $(shell echo "bubblewrap not detected; disabling sandboxing" >&2)
+    AB_SANDBOX := no
+  endif
+endif
+
 # Wipe the build file (forcing a regeneration) if the make environment is different.
 # (Conveniently, this includes the pkg-config hash calculated above.)
 

build/ab.py

@@ -18,6 +18,7 @@
 import string
 import sys
 import types
+import shlex
 
 VERBOSE_NINJA_FILE = False
 
@@ -115,6 +116,17 @@ def _undo_escaped_dollar(s, op):
 
 class BracketedFormatter(string.Formatter):
     def parse(self, format_string):
+        return self._parse_cached(format_string)
+
+    @staticmethod
+    @functools.lru_cache(maxsize=4096)
+    def _parse_cached(format_string):
+        # Cache a reusable tuple, rather than an exhaustible generator. Parsing
+        # is target-independent; expression values must still be evaluated live.
+        return tuple(BracketedFormatter._parse_parts(format_string))
+
+    @staticmethod
+    def _parse_parts(format_string):
         while format_string:
             m = re.search(f"(?:[^$]|^)()\\$\\[()", format_string)
             if not m:
@@ -149,6 +161,15 @@ def parse(self, format_string):
 
 class GlobalFormatter(string.Formatter):
     def parse(self, format_string):
+        return self._parse_cached(format_string)
+
+    @staticmethod
+    @functools.lru_cache(maxsize=4096)
+    def _parse_cached(format_string):
+        return tuple(GlobalFormatter._parse_parts(format_string))
+
+    @staticmethod
+    def _parse_parts(format_string):
         while format_string:
             m = re.search(f"(?:[^$]|^)()\\$\\(([^)]*)\\)()", format_string)
             if not m:
@@ -186,6 +207,8 @@ def format_field(self, value, format_spec):
 
 
 def substituteGlobalVariables(value):
+    if "$(" not in value:
+        return value
     while True:
         oldValue = value
         value = globalFormatter.format(value)
@@ -253,6 +276,40 @@ def _isiterable(xs):
     )
 
 
+@functools.lru_cache(maxsize=4096)
+def _compile_expression(expression):
+    # eval(str) skips leading spaces and tabs before compiling its expression.
+    return compile(expression.lstrip(" \t"), "<string>", "eval")
+
+
+class TargetFormatter(BracketedFormatter):
+    def __init__(self, target):
+        self.target = target
+
+    def get_field(self, name, a1, a2):
+        return (
+            eval(
+                _compile_expression(name),
+                self.target.callback.__globals__,
+                self.target.args,
+            ),
+            False,
+        )
+
+    def format_field(self, value, format_spec):
+        if not value:
+            return ""
+        if type(value) == str:
+            return value
+        if _isiterable(value):
+            value = list(value)
+        if type(value) != list:
+            value = [value]
+        return " ".join(
+            [self.target.templateexpand(f) for f in filenamesof(value)]
+        )
+
+
 class Target:
     def __init__(self, cwd, name):
         self.name = name
@@ -277,28 +334,10 @@ def __hash__(self):
     def __repr__(self):
         return f"Target('{self.name}')"
 
-    def templateexpand(selfi, s):
-        class Formatter(BracketedFormatter):
-            def get_field(self, name, a1, a2):
-                return (
-                    eval(name, selfi.callback.__globals__, selfi.args),
-                    False,
-                )
-
-            def format_field(self, value, format_spec):
-                if not value:
-                    return ""
-                if type(value) == str:
-                    return value
-                if _isiterable(value):
-                    value = list(value)
-                if type(value) != list:
-                    value = [value]
-                return " ".join(
-                    [selfi.templateexpand(f) for f in filenamesof(value)]
-                )
-
-        s = Formatter().format(s)
+    def templateexpand(self, s):
+        if "$[" not in s:
+            return substituteGlobalVariables(s)
+        s = TargetFormatter(self).format(s)
         return substituteGlobalVariables(s)
 
     def materialise(self, replacing=False):
@@ -430,7 +469,7 @@ def targetof(value, cwd=None):
 
         # At this point we have the fully qualified name of a rule.
 
-        (path, target) = value.rsplit("+", 1)
+        path, target = value.rsplit("+", 1)
         value = join(path, "+" + target)
         if value not in targets:
             # Load the new build file.
@@ -526,6 +565,7 @@ def generate(xs):
 
 
 def filenameof(x):
+    x = targetof(x)
     xs = filenamesof(x.outs)
     assert (
         len(xs) == 1
@@ -573,46 +613,57 @@ def emit_rule(self, ins, outs, cmds=[], label=None):
 
     if outs:
         os.makedirs(self.dir, exist_ok=True)
-        rule = []
 
-        if G.AB_SANDBOX == "yes":
-            sandbox = join(self.dir, "sandbox")
-            emit(f"rm -rf {sandbox}", into=rule)
-            emit(
-                f"{G.PYTHON} build/_sandbox.py --link -s",
-                sandbox,
-                *fins,
-                into=rule,
-            )
-            for c in cmds:
-                emit(f"(cd {sandbox} &&", c, ")", into=rule)
-            emit(
-                f"{G.PYTHON} build/_sandbox.py --export -s",
-                sandbox,
-                *fouts,
-                into=rule,
-            )
-        else:
-            for c in cmds:
-                emit(c, into=rule)
+        # Write the actual commands to a shell script.
 
+        rule = []
+        for c in cmds:
+            emit(f"( {c} )", into=rule)
         ruletext = "".join(rule)
-        if len(ruletext) > 7000:
-            rulehash = hashlib.sha1(ruletext.encode()).hexdigest()
+        rulehash = hashlib.sha1(ruletext.encode()).hexdigest()
+
+        rulef = join(self.dir, f"rule-{rulehash}.sh")
+        with open(rulef, "wt") as fp:
+            fp.write("set -e\n")
+            fp.write(ruletext)
 
-            rulef = join(self.dir, f"rule-{rulehash}.sh")
-            with open(rulef, "wt") as fp:
-                fp.write("set -e\n")
-                fp.write(ruletext)
+        # Generate the invocation of the shell script, either directly or through bubblewrap.
 
-            emit("build", *fouts, ":rule", *fins)
-            emit(" command=sh", rulef)
+        rule = []
+        if G.AB_SANDBOX == "yes":
+            bwrap = join(self.dir, "bwrap.args")
+            with open(bwrap, "wb") as fp:
+                cwd = os.getcwd()
+
+                def robind(f):
+                    ff = join(cwd, f)
+                    ffb = f"{ff}\0".encode("utf-8")
+                    fp.write(b"--ro-bind\0")
+                    fp.write(ffb)
+                    fp.write(ffb)
+
+                fp.write(b"--dev-bind\0/\0/\0")
+                fp.write(b"--bind\0")
+                fp.write(f"{cwd}/{self.dir}/sandbox".encode("utf-8") + b"\0")
+                fp.write(f"{cwd}".encode("utf-8") + b"\0")
+                for f in fins + [rulef]:
+                    robind(f)
+            emit(f"rm -rf ", f"{self.dir}/sandbox", into=rule)
+            emit(f"mkdir -p ", f"{self.dir}/sandbox", into=rule)
+
+            emit(G.BWRAP, f"--args 3 3<", bwrap, "sh", rulef, into=rule)
+            for f in fouts:
+                emit(f"cp {self.dir}/sandbox/{f} {f}", into=rule)
         else:
-            emit("build", *fouts, ":rule", *fins)
-            emit(
-                " command=",
-                "&&".join([s.strip() for s in rule]).replace("$", "$$"),
-            )
+            emit(f"sh", rulef, into=rule)
+
+        # Now generate the ninja build rule.
+
+        emit("build", *fouts, ":rule", *fins)
+        emit(
+            " command=",
+            "&&".join([s.strip() for s in rule]).replace("$", "$$"),
+        )
         if label:
             emit(" description=", label)
         emit("build", name, ":phony", *fouts)

build/c.py

@@ -109,7 +109,7 @@ def cfileimpl(self, name, srcs, deps, suffix, commands, label, toolchain, cflags
     hdr_files = collectattrs(targets=hdr_deps, name="cheader_files")
     cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
 
-    t = simplerule(
+    simplerule(
         replaces=self,
         ins=srcs,
         deps=other_deps + hdr_files,

doc/LLgen/LLgen.n

@@ -40,12 +40,8 @@ to resolve these ambiguities.
 The specification can be split into several files, for each of
 which \fILLgen\fR generates an output file containing the
 corresponding part of the parser.
-Furthermore, only output files that differ from their previous
-version are updated.
-Other output files are not affected in any
-way.
-This allows the user to recompile only those output files that have
-changed.
+All output files are written directly to the output directory and
+existing files are overwritten without comparison or identification checks.
 .PP
 The subroutine produced by \fILLgen\fR calls a user supplied routine
 that must return the next token. This way, the input to the
@@ -122,9 +118,8 @@ yacc
 .]).
 It demonstrates the struggle \fILLparse\fR and other LL(1)
 parsers have with expressions.
-Appendix C gives an example of the \fILLgen\fR features
-allowing the user to recompile only those output files that
-have changed, using the \fImake\fR program.
+Appendix C gives an example of using \fILLgen\fR with the
+\fImake\fR program.
 .[
 make
 .]
@@ -612,17 +607,15 @@ The LLgen working environment
 file, and two other files: \fILpars.c\fR and \fILpars.h\fR.
 \fILpars.h\fR contains "#-define"s for the tokennames.
 \fILpars.c\fR contains the error recovery routines and tables.
-Only those output files that differ from their previous version
-are updated. See appendix C for a possible application of this
-feature.
+All output files are overwritten on each run. Their names are reported
+on standard error.
 .PP
 The names of the output files are constructed as
 follows:
-in the input file name, the suffix after the last point is
-replaced by a "c". If no point is present in the input file
-name, ".c" is appended to it. \fILLgen\fR checks that the
-filename constructed this way in fact represents a previous
-version, or does not exist already.
+in the basename of the input file, the suffix after the last point is
+replaced by a "c". If no point is present, ".c" is appended.
+All output files are placed in the directory specified by
+\fB-o\fR\fIdirectory\fR, or in the current directory if this option is absent.
 .PP
 The user must provide some environment to obtain a complete
 program.

util/LLgen/build.py

@@ -44,9 +44,7 @@ def llgen(self, name, srcs: Targets = []):
         outs=(["=Lpars.c", "=Lpars.h"] + [f"={f}.c" for f in fs]),
         deps=["util/LLgen/lib/incl", "util/LLgen/lib/rec"],
         commands=[
-            f"$[ins[0]] {' '.join(filenamesof(srcs))}",
-            # llgen emits the files into the current directory (of the sandbox).
-            f"mv *.[ch] $[dir]",
+            f"$[ins[0]] -o$[dir] {' '.join(filenamesof(srcs))}",
         ],
         label="LLGEN",
     )

util/LLgen/doc/LLgen.1

@@ -4,7 +4,7 @@
 .SH NAME
 LLgen, an extended LL(1) parser generator
 .SH SYNOPSIS
-LLgen [ \-vxwans ] [ \-j[\fInum\fP] ] [ \-l\fInum\fP ] [ \-h\fInum\fP ] file ...
+LLgen [ \-o\fIdirectory\fP ] [ \-vxwans ] [ \-j[\fInum\fP] ] [ \-l\fInum\fP ] [ \-h\fInum\fP ] file ...
 .SH DESCRIPTION
 \fILLgen\fP
 converts a context-free grammar into a set of
@@ -46,13 +46,9 @@ also generates a file
 that contains the non-correcting recovery mechanism.
 .PP
 \fILLgen\fP
-will only update those output files that differ from their previous
-version.
-This allows
-\fILLgen\fP
-to be used with
-\fImake\fP
-(1) convieniently.
+writes all output files directly to the output directory, overwriting
+existing files without comparing their contents or checking their origin.
+Each output filename is reported on standard error.
 .PP
 To obtain a working program, the user must also supply a
 lexical analyzer, as well as
@@ -76,9 +72,10 @@ If the flag is given more than once,
 will be more "verbose".
 If it is given three times, a complete description of the
 grammar will be supplied.
-.IP \fB\-o\fP
-Specifies the location of the output files. If not set, they
-are placed in the current directory.
+.IP \fB\-o\fP\fIdirectory\fP
+Specifies the directory for all output files. The directory must already exist.
+The directory name must immediately follow the option, without a space.
+If not set, output files are placed in the current directory.
 .IP \fB\-x\fP
 the sets that are computed are extended with the nonterminal
 symbols and these extended sets are also included in the