nitls: Improve resiliency to parsing errors

src/loader.nit

@@ -670,6 +670,9 @@ redef class ModelBuilder
 		return path.realpath
 	end
 
+	# Errors of the lexer or parser that prevented a module from loading
+	var parse_errors = new Array[AError]
+
 	# Try to load a module AST using a path.
 	# Display an error if there is a problem (IO / lexer / parser) and return null
 	#
@@ -702,6 +705,7 @@ redef class ModelBuilder
 			var neof = tree.n_eof
 			assert neof isa AError
 			error(neof, neof.message)
+			parse_errors.add neof
 			return null
 		end
 

src/loader_header_only.nit

@@ -0,0 +1,147 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Load only the header of the modules: module declaration and imports
+#
+# The AST of the loaded modules have only what's needed for the importation
+# hierarchy and no classes, top-level functions, top-level code, or top-level
+# foreign code bodies. This parsing is fast and robust to syntax errors
+# in the body of the modules.
+module loader_header_only
+
+import loader
+
+redef class Lexer
+	# Tokens already read ahead of the parser, to be returned next
+	private var ahead = new Array[Token]
+
+	# Location of the first token of the body
+	private var header_end: nullable Location = null
+
+	# Normal token stream up to the imports, then return EOF
+	#
+	# Look ahead for a definition or block opening as a way to detect that
+	# we're done with imports. The look ahead allows to skip over loose tokens
+	# and keywords before the definition markers like visibility and `redef`.
+	redef fun get_token
+	do
+		var header_end = header_end
+		if header_end == null then
+			if ahead.not_empty then return ahead.shift
+
+			var t = super
+			if t == null then return null
+			if not is_body_start(t) then return t
+
+			header_end = t.location
+			self.header_end = header_end
+		end
+
+		return new EOF.init_tk(header_end)
+	end
+
+	# Is `t` the first token of the body? Looking ahead as needed
+	private fun is_body_start(t: Token): Bool
+	do
+		if not t.is_definition_prefix then return t.starts_body
+
+		var n = scan_significant
+		ahead.add n
+		while n.can_be_loose or n.is_definition_prefix do
+			n = scan_significant
+			ahead.add n
+		end
+		return not n isa TKwimport
+	end
+
+	# Scan the next raw token that is not an ignored blank space
+	private fun scan_significant: Token
+	do
+		var t = scan_token
+		while t == null do t = scan_token
+		return t
+	end
+end
+
+redef class Token
+	private fun starts_body: Bool do return false
+end
+
+# Definitions
+
+redef class TKwclass
+	redef fun starts_body do return true
+end
+
+redef class TKwabstract
+	redef fun starts_body do return true
+end
+
+redef class TKwinterface
+	redef fun starts_body do return true
+end
+
+redef class TKwenum
+	redef fun starts_body do return true
+end
+
+redef class TKwsubset
+	redef fun starts_body do return true
+end
+
+redef class TKwextern
+	redef fun starts_body do return true
+end
+
+redef class TKwmeth
+	redef fun starts_body do return true
+end
+
+redef class TKwinit
+	redef fun starts_body do return true
+end
+
+redef class TKwtype
+	redef fun starts_body do return true
+end
+
+redef class TKwin
+	redef fun starts_body do return true
+end
+
+# Statements opening a block
+
+redef class TKwif
+	redef fun starts_body do return true
+end
+
+redef class TKwwhile
+	redef fun starts_body do return true
+end
+
+redef class TKwfor
+	redef fun starts_body do return true
+end
+
+redef class TKwloop
+	redef fun starts_body do return true
+end
+
+redef class TKwdo
+	redef fun starts_body do return true
+end
+
+redef class TKwwith
+	redef fun starts_body do return true
+end

src/nitls.nit

@@ -18,6 +18,7 @@
 module nitls
 
 import modelbuilder
+import loader_header_only
 import ordered_tree
 import console
 
@@ -206,6 +207,15 @@ if opt_source.value then
 	end
 end
 
+if opt_make.value then
+	# The modules that could not be parsed didn't result in a mmodule that
+	# could be listed above. List them here from the collected errors.
+	for error in mb.parse_errors do
+		var file = error.location.file
+		if file != null then print file.filename.escape_to_mk
+	end
+end
+
 if opt_package.value then
 	var mpackages = new Array[MPackage]
 	for m in mmodules do

src/parser/lexer_work.nit

@@ -140,9 +140,23 @@ class Lexer
 		return result
 	end
 
-	# Primitive method to return a token, or return null if it is discarded
-	# Is used to implement `peek` and `next`
-	protected fun get_token: nullable Token
+	# Interface method to get the next token, or null if it is discarded
+	#
+	# Returns null on whitespaces ignored by the grammar. Otherwise returns
+	# the next token from the abstract source, including an `EOF` token when at
+	# the end of the source file.
+	#
+	# This method is called by higher level services `peek` and `next`.
+	# Redefine this to filter the token stream, keeping `scan_token` for
+	# the actual read of raw tokens.
+	protected fun get_token: nullable Token do return scan_token
+
+	# Implementation of the scan reading the next token from the source
+	#
+	# This returns the raw tokens. To alter the token stream clients should
+	# prefer to redef `get_token` over `scan_token` as it can still
+	# call `scan_token` for looking ahead.
+	protected fun scan_token: nullable Token
 	do
 		var dfa_state = 0
 

src/parser/parser_nodes.nit

@@ -389,6 +389,18 @@ abstract class Token
 	# See `blank_before` to get the whitespace that separate tokens.
 	var is_loose = false
 
+	# Can `self` be discarded by the parser and become `is_loose`?
+	#
+	# True for end of lines (`TEol`) and comments (`TComment`).
+	#
+	# For example, a comment acting as documentation is attached to a definition
+	# and not marked `is_loose`, while an unattached comment is loose. In both
+	# cases, we consider the token kind to be potentially loose before parsing.
+	fun can_be_loose: Bool do return false
+
+	# Is `self` a `redef` or visibility keyword that may precede a definition?
+	fun is_definition_prefix: Bool do return false
+
 	redef fun is_structural do return true
 
 	redef fun dump_info(v) do
@@ -498,6 +510,7 @@ end
 # Token of end of line (basically `\n`)
 class TEol
 	super Token
+	redef fun can_be_loose do return true
 	redef fun to_s
 	do
 		return "end of line"
@@ -508,6 +521,7 @@ end
 # Starts with the `#` and contains the final end-of-line (if any)
 class TComment
 	super Token
+	redef fun can_be_loose do return true
 end
 
 # A token associated with a keyword
@@ -582,6 +596,7 @@ end
 # The keyword `redef`
 class TKwredef
 	super TokenKeyword
+	redef fun is_definition_prefix do return true
 end
 
 # The keyword `is`
@@ -612,21 +627,25 @@ end
 # The keyword `public`
 class TKwpublic
 	super TokenKeyword
+	redef fun is_definition_prefix do return true
 end
 
 # The keyword `protected`
 class TKwprotected
 	super TokenKeyword
+	redef fun is_definition_prefix do return true
 end
 
 # The keyword `private`
 class TKwprivate
 	super TokenKeyword
+	redef fun is_definition_prefix do return true
 end
 
 # The keyword `intrude`
 class TKwintrude
 	super TokenKeyword
+	redef fun is_definition_prefix do return true
 end
 
 # The keyword `if`

tests/nitls.args

@@ -6,3 +6,9 @@ base_simple3.nit project1
 -td project1/module3.nit project1/subdir/subdir2/subdir3/submodule.nit
 test_prog --no-color
 test_prog/game/excluded.nit test_prog/game/excluded_dir/more.nit -t --no-color
+-M nitls_error_body.nit
+-M nitls_header_annotations.nit
+-M nitls_header_visibility.nit
+-M nitls_header_redef.nit
+-M error_mod_unk.nit --no-color
+-M nitls_error_header.nit

tests/nitls_error_body.nit

@@ -0,0 +1,18 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The imports can be found, despite the syntax error below
+import base_simple_import
+
+fun foo(label: Int) do end

tests/nitls_error_header.nit

@@ -0,0 +1,18 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base_simple_import
+
+# Syntax error in the header
+import 5

tests/nitls_header_annotations.nit

@@ -0,0 +1,29 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module nitls_header_annotations is
+	ldflags("-lm",
+		"-lpthread")
+end
+
+import end
+import base_simple
+
+# The annotations of an import can span lines too
+import base_simple_import is
+	conditional(base_simple)
+end
+
+class NitlsHeaderAnnotations
+end

tests/nitls_header_redef.nit

@@ -0,0 +1,23 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module nitls_header_redef
+
+import end
+
+# The grammar accepts `redef` before an import, should it really?
+redef import base_simple3
+
+redef class A
+end