Reland "[llvm] Add initial scaffolding"

This is a reland of b71d2d9996

Original change's description:
> [llvm] Add initial scaffolding
>
> This change adds the gclient and GN changes needed to build
> an executable using LLVM in the Dart tree as well as a basic
> testing framework based on llvm-lit.
>
> Change-Id: I9009a98ff95043cc3754966f31697ba7f1712310
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/106434
> Commit-Queue: Jake Ehrlich <jakehehrlich@google.com>
> Reviewed-by: Alexander Thomas <athom@google.com>
> Reviewed-by: Vyacheslav Egorov <vegorov@google.com>

Change-Id: Ib3cd3299ed463133616c666285f9a58fa387b5bd
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/107829
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
Commit-Queue: Jake Ehrlich <jakehehrlich@google.com>

.gitignore

@@ -23,6 +23,11 @@
 /*.vcxproj.user
 *.stamp
 
+# LLVM prebuilts
+/third_party/llvm/include
+/third_party/llvm/lib
+/third_party/llvm/.versions
+
 # Gyp generated files
 *.xcodeproj
 *.intermediate
@@ -96,3 +101,4 @@ editor/util/testing/mac/Samples.suite/Results
 /outline.dill
 /generated/
 /crash_logs/
+/build/config/gclient_args.gni

BUILD.gn

@@ -2,6 +2,7 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+import("build/config/gclient_args.gni")
 import("build/dart/dart_host_sdk_toolchain.gni")
 
 targetting_fuchsia = target_os == "fuchsia"
@@ -15,6 +16,11 @@ group("default") {
   deps = [
     ":runtime",
   ]
+  if (checkout_llvm) {
+    deps += [
+      ":llvm_codegen"
+    ]
+  }
 }
 
 group("most") {
@@ -116,6 +122,19 @@ group("analysis_server") {
   ]
 }
 
+group("check_llvm") {
+  deps = [
+    "runtime/llvm_codegen/test",
+  ]
+}
+
+group("llvm_codegen") {
+  deps = [
+    "runtime/llvm_codegen/codegen",
+    "runtime/llvm_codegen/bit",
+  ]
+}
+
 # This is the target that is built on the dart2js build bots.
 # It must depend on anything that is required by the dart2js
 # test suites.

DEPS

@@ -146,8 +146,20 @@ vars = {
   "crashpad_rev": "bf327d8ceb6a669607b0dbab5a83a275d03f99ed",
   "minichromium_rev": "8d641e30a8b12088649606b912c2bc4947419ccc",
   "googletest_rev": "f854f1d27488996dc8a6db3c9453f80b02585e12",
+
+  # An LLVM backend needs LLVM binaries and headers. To avoid build time
+  # increases we can use prebuilts. We don't want to download this on every
+  # CQ/CI bot nor do we want the average Dart developer to incur that cost.
+  # So by default we will not download prebuilts.
+  "checkout_llvm": False,
+  "llvm_revision": "fe8bd96ebd6c490ea0b5c1fb342db2d7c393a109"
 }
 
+gclient_gn_args_file = Var("dart_root") + '/build/config/gclient_args.gni'
+gclient_gn_args = [
+  'checkout_llvm'
+]
+
 deps = {
   # Stuff needed for GN build.
   Var("dart_root") + "/buildtools/clang_format/script":
@@ -404,6 +416,17 @@ deps = {
       ],
       "dep_type": "cipd",
   },
+
+  Var("dart_root") + "/third_party/llvm": {
+      "packages": [
+          {
+              "package": "fuchsia/lib/llvm/${{platform}}",
+              "version": "git_revision:" + Var("llvm_revision"),
+          },
+      ],
+      "condition": "checkout_llvm",
+      "dep_type": "cipd",
+  },
 }
 
 deps_os = {

runtime/bin/BUILD.gn

@@ -2,6 +2,8 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+
+import("../../build/config/gclient_args.gni")
 import("../../build/dart/dart_action.gni")
 import("../runtime_args.gni")
 import("../vm/compiler/compiler_sources.gni")
@@ -923,6 +925,9 @@ executable("run_vm_tests") {
     "..:libdart_nosnapshot_with_precompiler",
     "//third_party/zlib",
   ]
+  if (checkout_llvm) {
+    deps += [ "//runtime/llvm_codegen/bit:test" ]
+  }
   include_dirs = [
     "..",
     "$target_gen_dir",

runtime/llvm_codegen/bit/BUILD.gn

@@ -0,0 +1,31 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+executable("bit") {
+  sources = [
+    "main.cc",
+  ]
+  root_dir = rebase_path(root_out_dir)
+  defines = [ "LIT_BINARY_DIR=\"$root_dir\"" ]
+  deps = [
+    "//third_party/llvm:LLVMSupport",
+  ]
+  data_deps = [
+    "//runtime/llvm_codegen/codegen",
+  ]
+}
+
+source_set("test") {
+  sources = [
+    "test.cc",
+  ]
+
+  deps = [
+    "//third_party/llvm:LLVMSupport",
+  ]
+
+  include_dirs = [ "//runtime" ]
+
+  defines = [ "TESTING" ]
+}

runtime/llvm_codegen/bit/bit.h

@@ -0,0 +1,76 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include <stdio.h>
+
+#include <map>
+#include <regex>
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/WithColor.h"
+
+namespace {
+
+using namespace llvm;
+
+struct Config {
+  StringRef filename;
+  StringRef out_dir;
+};
+
+StringMap<std::string> GetSubstitutions(const Config& config) {
+  // Compute all of our strings needed for substitutions.
+  StringRef test_dir = sys::path::parent_path(config.filename);
+  StringRef basename = sys::path::filename(config.filename);
+  SmallString<128> tmp_file;
+  sys::path::append(tmp_file, sys::path::Style::native, config.out_dir,
+                    basename + ".tmp");
+  SmallString<128> codegen;
+  sys::path::append(codegen, sys::path::Style::native, LIT_BINARY_DIR,
+                    "codegen");
+  SmallString<128> bit;
+  sys::path::append(bit, sys::path::Style::native, LIT_BINARY_DIR, "bit");
+
+  // Set up our substitutions.
+  StringMap<std::string> subs;
+  subs["s"] = config.filename.str();
+  subs["p"] = test_dir.str();
+  subs["P"] = test_dir.str();
+  subs["t"] = tmp_file.str().str();
+  subs["codegen"] = codegen.str().str();
+  subs["bit"] = bit.str().str();
+  return subs;
+}
+
+std::string PerformSubstitutions(const StringMap<std::string>& subs,
+                                 StringRef string) {
+  std::string out = string.str();
+  for (const auto& sub : subs) {
+    std::string key = (Twine("%") + sub.getKeyData()).str();
+    size_t pos = 0;
+    while ((pos = out.find(key, pos)) != std::string::npos) {
+      if (pos != 0 && out[pos - 1] == '%') {
+        pos += key.size();
+        continue;
+      }
+      out.replace(pos, key.size(), sub.getValue());
+      pos += sub.second.size();
+    }
+  }
+  return out;
+}
+
+Optional<std::string> GetCommand(StringRef line) {
+  static Regex run_line("^;[ ]*RUN:[ ]*(.*)$");
+  SmallVector<StringRef, 2> cmd;
+  if (!run_line.match(line, &cmd)) return Optional<std::string>{};
+  assert(cmd.size() == 2);
+  return cmd[1].str();
+}
+
+}  // namespace

runtime/llvm_codegen/bit/main.cc

@@ -0,0 +1,122 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include <stdio.h>
+
+#include <cctype>
+#include <map>
+#include <regex>
+
+#include "bit.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Support/WithColor.h"
+
+using namespace llvm;
+
+namespace {
+
+StringRef tool_name;
+
+LLVM_ATTRIBUTE_NORETURN void Fail(Twine message) {
+  WithColor::error(errs(), tool_name) << message << ".\n";
+  errs().flush();
+  exit(1);
+}
+
+LLVM_ATTRIBUTE_NORETURN void Fail(Error e) {
+  assert(E);
+  std::string buf;
+  raw_string_ostream os(buf);
+  logAllUnhandledErrors(std::move(e), os);
+  os.flush();
+  WithColor::error(errs(), tool_name) << buf;
+  exit(1);
+}
+
+LLVM_ATTRIBUTE_NORETURN void ReportError(StringRef file, std::error_code ec) {
+  assert(ec);
+  Fail(createFileError(file, ec));
+}
+
+std::string ReadFile(FILE* file) {
+  std::string output;
+  constexpr size_t buf_size = 256;
+  char buf[buf_size];
+  size_t size;
+  while ((size = fread(buf, buf_size, sizeof(buf[0]), file)))
+    output.append(buf, buf + buf_size);
+  return output;
+}
+
+bool IsPosixFullyPortablePath(StringRef path) {
+  const char* extra = "._-/";
+  for (auto c : path)
+    if (!isalnum(c) && !strchr(extra, c)) return false;
+  return true;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  InitLLVM X(argc, argv);
+
+  // Make sure we have both arguments.
+  tool_name = argv[0];
+  if (argc != 3) Fail("expected exactly 2 arguments");
+
+  // Makes sure that stdin/stdout are setup correctly.
+  if (sys::Process::FixupStandardFileDescriptors())
+    Fail("std in/out fixup failed");
+
+  // Set our config.
+  Config config;
+  config.filename = argv[1];
+  config.out_dir = argv[2];
+
+  // Make sure we have valid filepaths.
+  if (!IsPosixFullyPortablePath(config.filename))
+    Fail("'" + config.filename + "' is not a posix fully portable filename");
+  if (!IsPosixFullyPortablePath(config.out_dir))
+    Fail("'" + config.out_dir + "' is not a posix fully portable filename");
+
+  // Compute substitutions.
+  auto subs = GetSubstitutions(config);
+
+  // The lines we execute are allowed to assume that %p will exist.
+  sys::fs::create_directory(subs["p"]);
+
+  // Open the file for reading.
+  auto buf_or = vfs::getRealFileSystem()->getBufferForFile(config.filename);
+  if (!buf_or) ReportError(config.filename, buf_or.getError());
+  auto buf = std::move(*buf_or);
+
+  // Now iterate over the lines in the file.
+  line_iterator it{*buf};
+  int count = 0;
+  for (StringRef line = *it; !it.is_at_end(); line = *++it) {
+    auto cmd = GetCommand(line);
+    if (!cmd) continue;
+    ++count;
+    auto subbed = PerformSubstitutions(subs, *cmd);
+    FILE* file = popen(subbed.c_str(), "r");
+    std::string output = ReadFile(file);
+    if (pclose(file) != 0) {
+      errs() << output << "\n";
+      Fail("Failure on line " + Twine(it.line_number()) + "\n\t" + subbed + "");
+    }
+  }
+  if (count == 0) {
+    Fail("No commands to run");
+  }
+  outs() << "Commands run: " << count << "\n";
+  return 0;
+}

runtime/llvm_codegen/bit/test.cc

@@ -0,0 +1,71 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include <stdio.h>
+
+#include <utility>
+#include <vector>
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/WithColor.h"
+#include "platform/assert.h"
+#include "vm/unit_test.h"
+
+#define LIT_BINARY_DIR "/path/to/bins"
+#include "bit.h"
+
+UNIT_TEST_CASE(BasicGetSubstitutions) {
+  Config config;
+  config.filename = "/foo/bar/baz.ll";
+  config.out_dir = "/test/out/dir";
+
+  StringMap<std::string> expected;
+  expected["s"] = "/foo/bar/baz.ll";
+  expected["p"] = "/foo/bar";
+  expected["P"] = "/test/out/dir";
+  expected["t"] = "/test/out/dir/baz.ll.tmp";
+  expected["codegen"] = "/path/to/bins/codegen";
+  expected["bit"] = "/path/to/bins/bit";
+
+  StringMap<std::string> actual = GetSubstitutions(config);
+
+  EXPECT_EQ(actual.size(), expected.size());
+  for (const auto& p : actual)
+    EXPECT_EQ(p.getValue(), expected[p.getKey()]);
+}
+
+UNIT_TEST_CASE(BasicPerformSubstitutions) {
+  StringMap<std::string> subs;
+  subs["foo"] = "/foo/path";
+  subs["bar"] = "/bar/path";
+  subs["baz"] = "/baz/path";
+  std::vector<std::pair<std::string, std::string>> cases = {
+      {"%foo", "/foo/path"},
+      {"%bar", "/bar/path"},
+      {"%baz", "/baz/path"},
+      {"this has %foo, and %bar, and %baz2",
+       "this has /foo/path, and /bar/path, and /baz/path2"},
+      {"we don't want %this to expand", "we don't want %this to expand"},
+      {"%", "%"}};
+  for (const auto& test : cases) {
+    auto out = PerformSubstitutions(subs, test.first);
+    EXPECT_EQ(out, test.second);
+  }
+}
+
+UNIT_TEST_CASE(BasicGetCommand) {
+  EXPECT(!GetCommand("; this is some test"));
+  EXPECT(!GetCommand("2 + 2"));
+  EXPECT(!GetCommand("echo $VAR > %bit"));
+
+  EXPECT(GetCommand(";RUN: blarg") == Optional<std::string>{"blarg"});
+  EXPECT(GetCommand(";      RUN:        foo") == Optional<std::string>{"blarg"});
+  EXPECT(
+      GetCommand("; RUN: echo %bit %p/Input/$(%t2) > $(baz \"$BAR->%t\")") ==
+      Optional<std::string>("echo %bit %p/Input/$(%t2) > $(baz \"$BAR->%t\")"));
+}

runtime/llvm_codegen/codegen/BUILD.gn

@@ -0,0 +1,14 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+executable("codegen") {
+  sources = [
+    "main.cc",
+  ]
+
+  deps = [
+    "//third_party/llvm:LLVMAsmPrinter",
+    "//third_party/llvm:LLVMIRReader",
+  ]
+}

runtime/llvm_codegen/codegen/main.cc

@@ -0,0 +1,42 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include <memory>
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/WithColor.h"
+
+using namespace llvm;
+
+namespace {
+
+StringRef tool_name;
+
+LLVM_ATTRIBUTE_NORETURN void error(Twine message) {
+  WithColor::error(errs(), "llvm-codegen") << message << ".\n";
+  errs().flush();
+  exit(1);
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  InitLLVM X(argc, argv);
+
+  LLVMContext context;
+  SMDiagnostic err;
+  if (argc != 2) error("exactly one argument is taken");
+  tool_name = argv[0];
+  std::unique_ptr<Module> mod = parseIRFile(argv[1], err, context);
+  if (mod == nullptr) {
+    err.print(tool_name.data(), errs());
+    exit(1);
+  }
+  mod->print(outs(), nullptr);
+}

runtime/llvm_codegen/test/BUILD.gn

@@ -0,0 +1,10 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+group("test") {
+  deps = [
+    "codegen",
+    "bit",
+  ]
+}

runtime/llvm_codegen/test/bit/BUILD.gn

@@ -0,0 +1,9 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("//runtime/llvm_codegen/test/bit_test.gni")
+
+bit_test("bit") {
+  tests = [ "basic.test" ]
+}

runtime/llvm_codegen/test/bit/Inputs/basic_input.test

@@ -0,0 +1 @@
+; RUN: %codegen %p/../../codegen/basic.ll

runtime/llvm_codegen/test/bit/Inputs/percent.test

@@ -0,0 +1 @@
+this is a percent sign: %

runtime/llvm_codegen/test/bit/basic.test

@@ -0,0 +1,8 @@
+; RUN: echo this is a test > %t1
+; RUN: echo this is a test > %t2
+; RUN: cmp %t1 %t2
+; RUN: echo this is a percent sign: % > %t3
+; RUN: cmp %t3 %p/Inputs/percent.test
+; RUN: %bit %p/Inputs/basic_input.test %P > %t.test
+; RUN: echo Commands run: 1 > %t.test2
+; RUN: cmp %t.test %t.test2

runtime/llvm_codegen/test/bit_test.gni

@@ -0,0 +1,40 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# This file defines a template for running bit tests.
+#
+# - bit_test()
+#   Runs bit on the specified file.
+
+# A template for running bit. This lets bit commands be run as ninja commands.
+#
+# Parameters:
+#   tests:
+#     The list of files to input into bit
+template("bit_test") {
+  assert(defined(invoker.tests), "tests must be defined for $target_name")
+
+  action_foreach(target_name) {
+    script = "//build/gn_run_binary.py"
+    sources = invoker.tests
+
+    deps = [
+      "//runtime/llvm_codegen/bit",
+    ]
+    inputs = [
+      "${root_out_dir}/bit",
+    ]
+
+    # This output is always dirty so ninja will always run this step when asked to.
+    outputs = [
+      "$target_gen_dir/{{source_name_part}}}",
+    ]
+    args = [
+      "compiled_action",
+      rebase_path("${root_out_dir}/bit"),
+      "{{source}}",
+      rebase_path(target_gen_dir),
+    ]
+  }
+}

runtime/llvm_codegen/test/codegen/BUILD.gn

@@ -0,0 +1,9 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("//runtime/llvm_codegen/test/bit_test.gni")
+
+bit_test("codegen") {
+  tests = [ "basic.ll" ]
+}

runtime/llvm_codegen/test/codegen/basic.ll

@@ -0,0 +1,6 @@
+; RUN: %codegen %s > %t
+
+define i32 @mult(i32, i32) {
+  %3 = mul i32 %0, %1
+  ret i32 %3
+}