Introduced hash valued location markers in the spec

Introduced support for adding SHA1 hash valued location markers
at several levels in the language specification, added long explanatory
comment at the end, added a script 'addlatexhash.dart' to normalize
spacing, remove comments, etc., in the spec, such that the hash values
are more robust than they would be with a direct usage of the spec.

The script passes the "dvi2tty test", that is, when the location markers
are empty, the resulting *.dvi files created from dartLangSpec.tex and
from the version processed by the script give rise to the same text via
dvi2tty, i.e., the script does not destroy the spec.

R=gbracha@google.com, ricow@google.com

Review URL: https://codereview.chromium.org//646003002

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@41191 260f80e4-7a28-3924-810f-c04153c831b5

docs/language/dart.sty

@@ -113,3 +113,39 @@
 }{
   %\end{verbatim}
 }
+
+
+% ----------------------------------------------------------------------
+% Support for hash valued Location Markers
+
+% very small font, to enable 33 char hash values in the margin
+\makeatletter
+\ifcase \@ptsize \relax% 10pt
+  \newcommand{\miniscule}{\@setfontsize\miniscule{2}{3}}% \tiny: 5/6
+\or% 11pt
+  \newcommand{\miniscule}{\@setfontsize\miniscule{3}{4}}% \tiny: 6/7
+\or% 12pt
+  \newcommand{\miniscule}{\@setfontsize\miniscule{3}{4}}% \tiny: 6/7
+\fi
+\makeatother
+
+% white: location markers should not create visual noise
+\definecolor{LMdim}{gray}{1.0}
+
+% insert location marker showing hash value of following paragraph
+\newcommand{\LMHash}[1]{%
+  \hspace{0pt}\marginpar{\raisebox{0.5ex}{\miniscule{\color{LMdim}#1}}}}
+
+% support convenient renewcommand
+\let\OriginalLMHash\LMHash
+
+% define a label, and show the associated logical location marker
+\newcommand{\LMLabel}[1]{%
+  \vspace{-\baselineskip}\hspace{0pt}\OriginalLMHash{\raisebox{10ex}{sec:#1}}%
+  \label{#1}}
+
+% dummy version of LMHash, always shows the same arbitrary hash value
+\renewcommand{\LMHash}[1]{\OriginalLMHash{ba01b04d58c8c4e259764498f823cc65}}
+
+% ----------------------------------------------------------------------
+

tests/standalone/io/addlatexhash_test.dart

@@ -0,0 +1,122 @@
+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// testing ../../../tools/addlatexhash.dart
+
+import 'dart:io';
+import 'package:path/path.dart' as path;
+import '../../../tools/addlatexhash.dart';
+
+final scriptDir = path.dirname(path.fromUri(Platform.script));
+final dartRootDir = path.dirname(path.dirname(path.dirname(scriptDir)));
+final dartRootPath = dartRootDir.toString();
+
+// Check that the given ProcessResult indicates success; if so
+// return the standard output, otherwise report the failure
+checkAction(result, errorMessage) {
+  if (result.exitCode != 0) {
+    print(result.stdout);
+    print(result.stderr);
+    throw errorMessage;
+  }
+  return result.stdout;
+}
+
+oneTestCutMatch(line, re, expected) {
+  var result = cutMatch(line, new RegExp(re).firstMatch(line));
+  if (result != expected) {
+    throw "cutMatch '$re' from '$line' yields '$result' != '$expected'";
+  }
+}
+
+void testCutMatch() {
+  oneTestCutMatch("test", "", "test");
+  oneTestCutMatch("test", "e", "tst");
+  oneTestCutMatch("test", "te", "st");
+  oneTestCutMatch("test", "st", "te");
+  oneTestCutMatch("test", "test", "");
+}
+
+oneTestSisp(sispFun, nameSuffix, line, expected) {
+  var result = sispFun(line);
+  if (result != expected) {
+    throw "sispIsDart$nameSuffix '$line' yields $result";
+  }
+}
+
+testSisp() {
+  oneTestSisp(sispIsDartBegin, "Begin", "\\begin{dartCode}\n", true);
+  oneTestSisp(sispIsDartBegin, "Begin", " \\begin{dartCode}\n", true);
+  oneTestSisp(sispIsDartBegin, "Begin", "whatever else ..", false);
+  oneTestSisp(sispIsDartEnd, "End", "\\end{dartCode}", true);
+  oneTestSisp(sispIsDartEnd, "End", " \\end{dartCode}\t  \n", true);
+  oneTestSisp(sispIsDartEnd, "End", "whatever else ..", false);
+}
+
+// Check that the LaTeX source transformation done by addlatexhash.dart
+// does not affect the generated output, as seen via dvi2tty and diff.
+// NB: Not part of normal testing (only local): latex and dvi2tty are
+// not installed in the standard test environment.
+testNoChange() {
+  // set up /tmp directory to hold output
+  final tmpDir = Directory.systemTemp.createTempSync("addlatexhash_test");
+  final tmpDirPath = tmpDir.path;
+
+  // file names/paths for original spec
+  const specName = "dartLangSpec";
+  const specFileName = "$specName.tex";
+  final specDirPath = path.join(dartRootDir, "docs", "language");
+  final specPath = path.join(specDirPath, specFileName);
+  final tmpSpecPath = path.join(tmpDirPath, specFileName);
+  const specDviFileName = "$specName.dvi";
+  final specDviPath = path.join(tmpDirPath, specDviFileName);
+
+  // file names/paths for associated sty
+  const styFileName = "dart.sty";
+  final styPath = path.join(specDirPath, styFileName);
+  final tmpStyPath = path.join(tmpDirPath, styFileName);
+
+  // file names paths for output
+  const hashName = "dartLangSpec-hash";
+  const hashFileName = "$hashName.tex";
+  final hashPath = path.join(tmpDirPath, hashFileName);
+  final hashDviPath = path.join(tmpDirPath, "$hashName.dvi");
+
+  // actions to take
+  runLatex(fileName,workingDirectory) =>
+      Process.runSync("latex", [fileName], workingDirectory: workingDirectory);
+
+  runAddHash() =>
+      Process.runSync("dart",
+                      [path.join(dartRootPath, "tools", "addlatexhash.dart"),
+                       tmpSpecPath,
+                       hashPath]);
+
+  runDvi2tty(dviFile) =>
+      Process.runSync("dvi2tty", [dviFile], workingDirectory: tmpDir.path);
+
+  chkDvi2tty(file, subject) =>
+      checkAction(runDvi2tty(file), "dvitty on $subject failed");
+
+  // perform test
+  new File(styPath).copySync(tmpStyPath);
+  new File(specPath).copySync(tmpSpecPath);
+  for (var i = 0; i < 5; i++) {
+    checkAction(runLatex(specName, tmpDirPath), "LaTeX on spec failed");
+  }
+  checkAction(runAddHash(),"addlatexhash.dart failed");
+  for (var i = 0; i < 5; i++) {
+    checkAction(runLatex(hashFileName, tmpDirPath), "LaTeX on output failed");
+  }
+  if (chkDvi2tty(specDviPath, "spec") != chkDvi2tty(hashDviPath, "output")) {
+    throw "dvi2tty spec != dvitty output";
+  }
+}
+
+main([args]) {
+  testCutMatch();
+  testSisp();
+  // latex and dvi2tty are not installed in the standard test environment
+  if (args.length > 0 && args[0] == "local") testNoChange();
+}

tools/addlatexhash.dart

@@ -0,0 +1,204 @@
+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+//
+// ----------------------------------------------------------------------
+// This is a very specialized tool which was created in order to support
+// adding hash values used as location markers in the LaTeX source of the
+// language specification.  It is intended to take its input file as the
+// first argument and the output file name as the second argument. From
+// docs/language a typical usage would be as follows:
+//
+//   dart ../../tools/addlatexhash.dart dartLangSpec.tex tmp.tex
+//
+// This will yield a normalized variant tmp.tex of the language
+// specification with hash values filled in.  For more details, please
+// check the language specification source itself.
+//
+// NB: This utility assumes UN*X style line endings, \n, in the LaTeX
+// source file receieved as input; it will not work with other styles.
+//
+// TODO: The current version does not fill in hash values, it only
+// standardizes the LaTeX source by removing comments and normalizing
+// white space.
+
+import 'dart:io';
+import 'dart:convert';
+import '../pkg/crypto/lib/crypto.dart';
+
+// Normalization of the text, i.e., removal or normalization
+// of elements that do not affect the output from latex
+
+final commentRE = new RegExp(r"[^\\]%.*"); // NB: . does not match \n
+final whitespaceAllRE = new RegExp(r"^\s+$");
+final whitespaceRE = new RegExp(r"[ \t]{2,}");
+
+// normalization steps
+
+cutMatch(line, match, {startOffset: 0, endOffset: 0, glue: ""}) {
+  if (match == null) return line;
+  var start = match.start + startOffset;
+  var end = match.end + endOffset;
+  var len = line.length;
+  if (start < 0) start = 0;
+  if (end > len) end = len;
+  return line.substring(0, start) + glue + line.substring(end);
+}
+
+cutRegexp(line, re, {startOffset: 0, endOffset: 0, glue: ""}) {
+  return cutMatch(line, re.firstMatch(line),
+                  startOffset: startOffset,
+                  endOffset: endOffset,
+                  glue: glue);
+}
+
+cutFromMatch(line, match, {offset: 0, glue: ""}) {
+  if (match == null) return line;
+  return line.substring(0, match.start + offset) + glue;
+}
+
+cutFromRegexp(line, re, {offset: 0, glue: ""}) {
+  return cutFromMatch(line, re.firstMatch(line), offset: offset, glue: glue);
+}
+
+isWsOnly(line) => whitespaceAllRE.firstMatch(line) != null;
+isCommentOnly(line) => line.startsWith("%");
+
+justEol(line) {
+  return line.endsWith("\n") ? "\n" : "";
+}
+
+stripComment(line) {
+  // NB: it is tempting to remove everything from the '%' and out,
+  // including the final newline, if any, but this does not work.
+  // The problem is that TeX will do exactly this, but then it will
+  // add back a character that depends on its state (S, M, or N),
+  // and it is tricky to maintain a similar state that matches the
+  // state of TeX faithfully.  Hence, we remove the content of
+  // comments but do not remove the comments themselves, we just
+  // leave the '%' at the end of the line and let TeX manage its
+  // states in a way that does not differ from the file from before
+  // stripComment
+  if (isCommentOnly(line)) return "%\n";
+  return cutRegexp(line, commentRE, startOffset: 2);
+}
+
+// Reduce a wsOnly line to its eol, remove leading ws
+// entirely, and reduce multiple ws chars to one
+normalizeWhitespace(line) {
+  var trimLine = line.trimLeft();
+  if (trimLine.isEmpty) return justEol(line);
+  return trimLine.replaceAll(whitespaceRE, " ");
+}
+
+// Reduce sequences of >1 wsOnly lines to 1, and sequences of >1
+// commentOnly lines to 1; moreover, treat commentOnly lines as
+// wsOnly when occurring in wsOnly line blocks
+multilineNormalize(lines) {
+  var afterBlankLines = false; // does 'line' succeed >0 empty lines?
+  var afterCommentLines = false; // .. succeed >0 commentOnly lines?
+  var newLines = new List();
+  for (var line in lines) {
+    if (afterBlankLines && afterCommentLines) {
+      // can never happen
+      throw "Bug, please report to eernst@";
+    } else if (afterBlankLines && !afterCommentLines) {
+      // at least one line before 'line' is wsOnly
+      if (!isWsOnly(line)) {
+        // blank line block ended
+        afterCommentLines = isCommentOnly(line);
+        // special case: it seems to be safe to remove commentOnly lines
+        // after wsOnly lines, so the TeX state must be predictably right;
+        // next line will then be afterCommentLines and be dropped, so
+        // we drop the entire comment block---which is very useful; we can
+        // also consider this comment line to be an empty line, such that
+        // subsequent empty lines can be considered to be in a block of
+        // empty lines; note that almost all variants of this will break..
+        if (afterCommentLines) {
+          // _current_ 'line' a commentOnly here
+          afterBlankLines = true;
+          afterCommentLines = false;
+          // and do not add 'line'
+        } else {
+          // after blanks, but current 'line' is neither blank nor comment
+          afterBlankLines = false;
+          newLines.add(line);
+        }
+      } else {
+        // blank line block continues, do not add 'line'
+      }
+    } else if (!afterBlankLines && afterCommentLines) {
+      // at least one line before 'line' is commentOnly
+      if (!isCommentOnly(line)) {
+        // comment line block ended
+        afterBlankLines = isWsOnly(line);
+        afterCommentLines = false;
+        newLines.add(line);
+      } else {
+        // comment line block continues, do not add 'line'
+      }
+    } else {
+      assert(!afterBlankLines && !afterCommentLines);
+      // no wsOnly or commentOnly lines preceed 'line'
+      afterBlankLines = isWsOnly(line);
+      afterCommentLines = isCommentOnly(line);
+      if (!afterCommentLines) newLines.add(line);
+      // else skipping commentOnly line after nonWs, nonComment text
+    }
+  }
+  return newLines;
+}
+
+// Selecting the elements in the pipeline
+
+normalize(line) => normalizeWhitespace(stripComment(line));
+sispNormalize(line) => stripComment(line);
+
+// Managing fragments with significant spacing
+
+final dartCodeBeginRE = new RegExp(r"^\s*\\begin\{dartCode\}");
+final dartCodeEndRE = new RegExp (r"^\s*\\end\{dartCode\}");
+
+sispIs(line, targetRE) {
+  return targetRE.firstMatch(line) != null;
+}
+
+sispIsDartBegin(line) => sispIs(line, dartCodeBeginRE);
+sispIsDartEnd(line) => sispIs(line, dartCodeEndRE);
+
+// Transform input file into output file
+
+main ([args]) {
+  if (args.length != 2) {
+    print("Usage: addlatexhash.dart <input-file> <output-file>");
+    throw "Received ${args.length} arguments, expected two";
+  }
+
+  var inputFile = new File(args[0]);
+  var outputFile = new File(args[1]);
+  assert(inputFile.existsSync());
+
+  var lines = inputFile.readAsLinesSync();
+  // single-line normalization
+  var inDartCode = false;
+  var newLines = new List();
+
+  for (var line in lines) {
+    if (sispIsDartBegin(line)) {
+      inDartCode = true;
+    } else if (sispIsDartEnd(line)) {
+      inDartCode = false;
+    }
+    if (inDartCode) {
+      newLines.add(sispNormalize(line + "\n"));
+    } else {
+      newLines.add(normalize(line + "\n"));
+    }
+  }
+
+  // multi-line normalization
+  newLines = multilineNormalize(newLines);
+
+  // output result
+  outputFile.writeAsStringSync(newLines.join());
+}