Auto merge of #37607 - dns2utf8:doc_grammar, r=alexcrichton

Fix grammar verification

 * Use make check-lexer to verify the grammar.
 * Extend grammar/README
 * Add make clean-grammar rule
 * Add target check-build-lexer-verifier to make tidy, so it will build the verifier with every build and catch future errors

This is the continuation of #34994

r? @steveklabnik @jonathandturner @alexcrichton

configure

@@ -854,6 +854,12 @@ probe_need CFG_CMAKE cmake
 # probe for it only in this case.
 if [ -n "$CFG_ANTLR4" ]
 then
+   CFG_ANTLR4_JAR="\"$(find /usr/ -name antlr-complete.jar 2>/dev/null | head -n 1)\""
+   if [ "x" -eq "x$CFG_ANTLR4_JAR" ]
+   then
+     CFG_ANTLR4_JAR="\"$(find ~ -name antlr-complete.jar 2>/dev/null | head -n 1)\""
+   fi
+   putvar CFG_ANTLR4_JAR      $CFG_ANTLR4_JAR
    probe CFG_JAVAC            javac
 fi
 

mk/clean.mk

@@ -35,7 +35,7 @@ clean-all: clean clean-llvm
 
 clean-llvm: $(CLEAN_LLVM_RULES)
 
-clean: clean-misc $(CLEAN_STAGE_RULES)
+clean: clean-misc clean-grammar $(CLEAN_STAGE_RULES)
 
 clean-misc:
 	@$(call E, cleaning)
@@ -47,6 +47,9 @@ clean-misc:
 	$(Q)rm -Rf dist/*
 	$(Q)rm -Rf doc
 
+clean-grammar:
+	@$(call E, cleaning grammar verification)
+	$(Q)rm -Rf grammar
 define CLEAN_GENERIC
 
 clean-generic-$(2)-$(1):

mk/grammar.mk

@@ -37,7 +37,7 @@ $(BG):
 
 $(BG)RustLexer.class: $(BG) $(SG)RustLexer.g4
 	$(Q)$(CFG_ANTLR4) -o $(BG) $(SG)RustLexer.g4
-	$(Q)$(CFG_JAVAC) -d $(BG) $(BG)RustLexer.java
+	$(Q)$(CFG_JAVAC) -d $(BG) -classpath $(CFG_ANTLR4_JAR) $(BG)RustLexer.java
 
 check-build-lexer-verifier: $(BG)verify
 

mk/tests.mk

@@ -243,7 +243,8 @@ cleantestlibs:
 
 .PHONY: tidy
 tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
-		$(SNAPSHOT_RUSTC_POST_CLEANUP)
+		$(SNAPSHOT_RUSTC_POST_CLEANUP) \
+		check-build-lexer-verifier
 	$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src
 
 $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \

src/grammar/README.md

@@ -1,14 +1,18 @@
-Reference grammar.
+# Reference grammar.
 
 Uses [antlr4](http://www.antlr.org/) and a custom Rust tool to compare
-ASTs/token streams generated. You can use the `check-lexer` make target to
+ASTs/token streams generated. You can use the `make check-lexer` target to
 run all of the available tests.
 
-To use manually:
+The build of the rust part is included with `make tidy` and can be run with `make check-build-lexer-verifier`.
+
+# Manual build
+
+To use manually, assuming antlr4 ist installed at `/usr/share/java/antlr-complete.jar`:
 
 ```
 antlr4 RustLexer.g4
-javac *.java
+javac -classpath /usr/share/java/antlr-complete.jar *.java
 rustc -O verify.rs
 for file in ../*/**.rs; do
     echo $file;
@@ -18,3 +22,12 @@ done
 
 Note That the `../*/**.rs` glob will match every `*.rs` file in the above
 directory and all of its recursive children. This is a zsh extension.
+
+
+## Cleanup
+
+To cleanup you can use a command like this:
+
+```bash
+rm -f verify *.class *.java *.tokens
+```

src/grammar/check.sh

@@ -20,11 +20,11 @@ skipped=0
 check() {
     grep --silent "// ignore-lexer-test" "$1";
 
-    # if it's *not* found...
+    # if it is *not* found...
     if [ $? -eq 1 ]; then
-        cd $2 # This `cd` is so java will pick up RustLexer.class. I couldn't
+        cd $2 # This `cd` is so java will pick up RustLexer.class. I could not
         # figure out how to wrangle the CLASSPATH, just adding build/grammar
-        # didn't seem to have any effect.
+        # did not seem to have any effect.
         if $3 RustLexer tokens -tokens < $1 | $4 $1 $5; then
             echo "pass: $1"
             passed=`expr $passed + 1`

src/grammar/verify.rs

@@ -11,6 +11,7 @@
 #![feature(plugin, rustc_private)]
 
 extern crate syntax;
+extern crate syntax_pos;
 extern crate rustc;
 
 #[macro_use]
@@ -290,9 +291,10 @@ fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
 
     let options = config::basic_options();
     let session = session::build_session(options, &DepGraph::new(false), None,
-                                         syntax::diagnostics::registry::Registry::new(&[]),
+                                         syntax::errors::registry::Registry::new(&[]),
                                          Rc::new(DummyCrateStore));
-    let filemap = session.parse_sess.codemap().new_filemap(String::from("<n/a>"), code);
+    let filemap = session.parse_sess.codemap()
+                         .new_filemap("<n/a>".to_string(), None, code);
     let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
     let cm = session.codemap();