Reland "[vm/wasm] Boilerplate for package:wasm"

This is a reland of 2bafc322fd

Original change's description:
> [vm/wasm] Boilerplate for package:wasm
>
> So far this just builds the wasmer library, copies it into the sdk
> directory, loads the library, and allows you to compile WASM modules.
> You can't actually do anything with the modules yet.
>
> Bug: https://github.com/dart-lang/sdk/issues/37882
> Change-Id: I7d7cfe5721bbe38a6afe76f326518e714d236ed4
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/158367
> Commit-Queue: Liam Appelbe <liama@google.com>
> Reviewed-by: Ryan Macnak <rmacnak@google.com>

Bug: https://github.com/dart-lang/sdk/issues/37882
Change-Id: I8056df1e301acde2772ba2273148faa53d03173e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/159321
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Commit-Queue: Liam Appelbe <liama@google.com>

.dart_tool/package_config.json

@@ -11,7 +11,7 @@
     "constraint, update this by running tools/generate_package_config.dart."
   ],
   "configVersion": 2,
-  "generated": "2020-08-17T14:05:33.104579",
+  "generated": "2020-08-18T11:26:08.472483",
   "generator": "tools/generate_package_config.dart",
   "packages": [
     {
@@ -515,7 +515,7 @@
       "name": "shelf",
       "rootUri": "../third_party/pkg/shelf",
       "packageUri": "lib/",
-      "languageVersion": "2.0"
+      "languageVersion": "2.1"
     },
     {
       "name": "shelf_packages_handler",
@@ -715,6 +715,12 @@
       "packageUri": "lib/",
       "languageVersion": "2.8"
     },
+    {
+      "name": "wasm",
+      "rootUri": "../pkg/wasm",
+      "packageUri": "lib/",
+      "languageVersion": "2.6"
+    },
     {
       "name": "watcher",
       "rootUri": "../third_party/pkg/watcher",

.packages

@@ -116,6 +116,7 @@ vector_math:third_party/pkg/vector_math/lib
 vm:pkg/vm/lib
 vm_service:pkg/vm_service/lib
 vm_snapshot_analysis:pkg/vm_snapshot_analysis/lib
+wasm:pkg/wasm/lib
 watcher:third_party/pkg/watcher/lib
 webdriver:third_party/pkg/webdriver/lib
 web_components:third_party/pkg/web_components/lib

DEPS

@@ -133,7 +133,7 @@ vars = {
   "quiver-dart_tag": "246e754fe45cecb6aa5f3f13b4ed61037ff0d784",
   "resource_rev": "f8e37558a1c4f54550aa463b88a6a831e3e33cd6",
   "root_certificates_rev": "7e5ec82c99677a2e5b95ce296c4d68b0d3378ed8",
-  "rust_revision": "60960a260f7b5c695fd0717311d72ce62dd4eb43",
+  "rust_revision": "cbe7c5ce705896d4e22bf6096590bc1f17993b78",
   "shelf_static_rev": "v0.2.8",
   "shelf_packages_handler_tag": "2.0.0",
   "shelf_proxy_tag": "0.1.0+7",

build/rust/rust.gni

@@ -2,40 +2,103 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+_dart_root = get_path_info("../..", "abspath")
+
 template("rust_library") {
   manifest = rebase_path("Cargo.toml")
   if (defined(invoker.manifest)) {
     manifest = invoker.manifest
   }
 
+  debug = defined(invoker.debug) && invoker.debug
+  shared = defined(invoker.shared) && invoker.shared
+
   cmd = [
-    rebase_path("//buildtools/${current_os}-${current_cpu}/rust/bin/cargo"),
+    rebase_path("//buildtools/${host_os}-${host_cpu}/rust/bin/cargo"),
     "build",
     "--target-dir",
     rebase_path(target_out_dir),
     "--manifest-path",
     manifest,
   ]
-  output = "$target_out_dir/lib${invoker.lib_name}.a"
-  debug = defined(invoker.debug) && invoker.debug
 
-  if (!debug) {
+  # For cross compilation, figure out the target triple. You can get a full list
+  # of the targets that rust supports like this: rustc --print target-list
+  cargo_out_dir = target_out_dir
+  if (is_linux) {
+    rust_os = "unknown-linux-gnu"
+  } else if (is_mac) {
+    rust_os = "apple-darwin"
+  } else if (is_win) {
+    rust_os = "pc-windows-gnu"
+  } else if (is_android) {
+    rust_os = "linux-android"
+  } else if (is_fuchsia) {
+    rust_os = "fuchsia"
+  }
+  if (defined(rust_os)) {
+    if (current_cpu == "x86") {
+      rust_target = "i686-${rust_os}"
+    } else if (current_cpu == "x64") {
+      rust_target = "x86_64-${rust_os}"
+    } else if (current_cpu == "arm") {
+      rust_target = "arm-${rust_os}eabi"
+    } else if (current_cpu == "arm64") {
+      rust_target = "aarch64-${rust_os}"
+    }
+  }
+  if (defined(rust_target)) {
+    cmd += [
+      "--target",
+      rust_target,
+    ]
+    cargo_out_dir += "/${rust_target}"
+  }
+
+  if (debug) {
+    cargo_out_dir += "/debug"
+  } else {
+    cargo_out_dir += "/release"
     cmd += [ "--release" ]
   }
 
-  action(target_name) {
-    script = "//build/rust/run.py"
+  output_file = ""
+  if (shared) {
+    if (is_win) {
+      output_file = "${invoker.lib_name}.dll"
+    } else if (is_mac) {
+      output_file = "lib${invoker.lib_name}.dylib"
+    } else {
+      output_file = "lib${invoker.lib_name}.so"
+    }
+  } else {
+    if (is_win) {
+      output_file = "${invoker.lib_name}.lib"
+    }else {
+      output_file = "lib${invoker.lib_name}.a"
+    }
+  }
+
+  action("${target_name}_cargo") {
+    script = "${_dart_root}/build/rust/run.py"
     args = cmd
-    outputs = [ output ]
+    outputs = [ "${cargo_out_dir}/${output_file}" ]
     public_configs = [ ":${target_name}_config" ]
   }
 
-  config("${target_name}_config") {
-    libs = [ "wasmer" ]
-    if (debug) {
-      lib_dirs = [ "$target_out_dir/debug" ]
-    } else {
-      lib_dirs = [ "$target_out_dir/release" ]
+  config("${target_name}_cargo_config") {
+    if (!shared) {
+      libs = [ invoker.lib_name ]
+      lib_dirs = [ out_dir ]
     }
   }
+
+  # Cargo leaves the library in cargo_out_dir, which varies based on the target.
+  # So we need to copy it to target_out_dir to make it easier for dependees to
+  # locate the library.
+  copy(target_name) {
+    deps = [ ":${target_name}_cargo" ]
+    sources = [ "${cargo_out_dir}/${output_file}" ]
+    outputs = [ "${target_out_dir}/${output_file}" ]
+  }
 }

pkg/wasm/.gitignore

@@ -0,0 +1,3 @@
+.dart_tool/
+.packages
+pubspec.lock

pkg/wasm/AUTHORS

@@ -0,0 +1,6 @@
+# Below is a list of people and organizations that have contributed
+# to the Dart project. Names should be added to the list like so:
+#
+#   Name/Organization <email address>
+
+Google LLC

pkg/wasm/LICENSE

@@ -0,0 +1,26 @@
+Copyright 2020, the Dart project authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/wasm/README.md

@@ -0,0 +1,4 @@
+# wasm
+
+This package provides utilities for loading and running WASM modules. It is
+built on top of the [Wasmer](https://github.com/wasmerio/wasmer) runtime.

pkg/wasm/analysis_options.yaml

@@ -0,0 +1 @@
+include: package:pedantic/analysis_options.1.8.0.yaml

pkg/wasm/lib/module.dart

@@ -0,0 +1,15 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'runtime.dart';
+import 'dart:typed_data';
+import 'dart:ffi';
+
+class WasmModule {
+  Pointer<WasmerModule> _module;
+
+  WasmModule(Uint8List data) {
+    _module = WasmRuntime().compile(data);
+  }
+}

pkg/wasm/lib/runtime.dart

@@ -0,0 +1,101 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:ffi';
+import 'dart:io';
+import 'dart:typed_data';
+import 'package:ffi/ffi.dart';
+import 'package:path/path.dart' as path;
+
+const int WasmerResultOk = 1;
+const int WasmerResultError = 2;
+
+const int WasmerValueTagI32 = 0;
+const int WasmerValueTagI64 = 1;
+const int WasmerValueTagF32 = 2;
+const int WasmerValueTagF64 = 3;
+
+class WasmerModule extends Struct {}
+
+typedef NativeWasmerCompileFn = Uint32 Function(
+    Pointer<Pointer<WasmerModule>>, Pointer<Uint8>, Uint32);
+typedef WasmerCompileFn = int Function(
+    Pointer<Pointer<WasmerModule>>, Pointer<Uint8>, int);
+
+class WasmRuntime {
+  static WasmRuntime _inst;
+
+  DynamicLibrary _lib;
+  WasmerCompileFn _compile;
+
+  factory WasmRuntime() {
+    if (_inst == null) {
+      _inst = WasmRuntime._init();
+    }
+    return _inst;
+  }
+
+  static String _getLibName() {
+    if (Platform.isMacOS) return "libwasmer.dylib";
+    if (Platform.isLinux) return "libwasmer.so";
+    throw Exception("Wasm not currently supported on this platform");
+  }
+
+  static String _getLibDir() {
+    // The common case, and how cli_util.dart computes the Dart SDK directory,
+    // path.dirname called twice on Platform.resolvedExecutable.
+    var commonLibDir = path.join(
+        path.absolute(path.dirname(path.dirname(Platform.resolvedExecutable))),
+        'bin',
+        'third_party',
+        'wasmer');
+    if (Directory(commonLibDir).existsSync()) {
+      return commonLibDir;
+    }
+
+    // This is the less common case where the user is in the checked out Dart
+    // SDK, and is executing dart via:
+    // ./out/ReleaseX64/dart ...
+    var checkedOutLibDir = path.join(
+        path.absolute(path.dirname(Platform.resolvedExecutable)),
+        'dart-sdk',
+        'bin',
+        'third_party',
+        'wasmer');
+    if (Directory(checkedOutLibDir).existsSync()) {
+      return checkedOutLibDir;
+    }
+
+    // If neither returned above, we return the common case:
+    return commonLibDir;
+  }
+
+  WasmRuntime._init() {
+    var libPath = path.join(_getLibDir(), _getLibName());
+    _lib = DynamicLibrary.open(libPath);
+    _compile = _lib
+        .lookup<NativeFunction<NativeWasmerCompileFn>>('wasmer_compile')
+        .asFunction();
+  }
+
+  Pointer<WasmerModule> compile(Uint8List data) {
+    var dataPtr = allocate<Uint8>(count: data.length);
+    for (int i = 0; i < data.length; ++i) {
+      dataPtr[i] = data[i];
+    }
+
+    var modulePtrPtr = allocate<Pointer<WasmerModule>>();
+    int result = _compile(modulePtrPtr, dataPtr, data.length);
+    Pointer<WasmerModule> modulePtr = modulePtrPtr.value;
+
+    free(modulePtrPtr);
+    free(dataPtr);
+
+    if (result != WasmerResultOk) {
+      throw Exception("Wasm module compile failed");
+    }
+
+    return modulePtr;
+  }
+}

pkg/wasm/lib/wasm.dart

@@ -0,0 +1,5 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+export 'module.dart';

pkg/wasm/pubspec.yaml

@@ -0,0 +1,12 @@
+name: wasm
+version: 0.1.0
+description: Load and run wasm bytecode.
+author: Dart Team <misc@dartlang.org>
+homepage: https://github.com/dart-lang/sdk/tree/master/pkg/wasm
+# This package is not intended for consumption on pub.dev. DO NOT publish.
+publish_to: none
+environment:
+  sdk: ">=2.6.0"
+dependencies:
+  ffi: ^0.1.3
+  path: ^1.0.0

runtime/BUILD.gn

@@ -221,10 +221,6 @@ library_for_all_configs("libdart") {
     ":generate_version_cc_file",
     "third_party/double-conversion/src:libdouble_conversion",
   ]
-  if (dart_enable_wasm) {
-    extra_deps += [ "//third_party/wasmer" ]
-    defines = [ "DART_ENABLE_WASM" ]
-  }
   if (is_fuchsia) {
     if (using_fuchsia_gn_sdk) {
       extra_deps += [

runtime/runtime_args.gni

@@ -91,7 +91,7 @@ declare_args() {
   # Whether libdart should export the symbols of the Dart API.
   dart_lib_export_symbols = true
 
-  # Whether dart:wasm should be enabled.
+  # Whether package:wasm should be enabled.
   dart_enable_wasm = false
 }
 

runtime/vm/BUILD.gn

@@ -156,9 +156,6 @@ library_for_all_configs("libdart_lib") {
       ]
     }
   }
-  if (dart_enable_wasm) {
-    defines = [ "DART_ENABLE_WASM" ]
-  }
   include_dirs = [ ".." ]
   allsources = async_runtime_cc_files + collection_runtime_cc_files +
                core_runtime_cc_files + developer_runtime_cc_files +

sdk/BUILD.gn

@@ -582,6 +582,22 @@ copy("copy_analysis_summaries") {
   outputs = [ "$root_out_dir/dart-sdk/lib/_internal/{{source_file_part}}" ]
 }
 
+copy("copy_wasmer") {
+  visibility = [ ":create_common_sdk" ]
+  deps = [
+    ":copy_libraries",
+    "../third_party/wasmer:wasmer_lib",
+  ]
+  outputs = [ "$root_out_dir/dart-sdk/bin/third_party/wasmer/{{source_file_part}}" ]
+  if (is_win) {
+    sources = [ "$target_out_dir/../third_party/wasmer/wasmer.dll" ]
+  } else if (is_mac) {
+    sources = [ "$target_out_dir/../third_party/wasmer/libwasmer.dylib" ]
+  } else {
+    sources = [ "$target_out_dir/../third_party/wasmer/libwasmer.so" ]
+  }
+}
+
 # This rule copies dill files to lib/_internal.
 copy("copy_vm_dill_files") {
   visibility = [ ":create_common_sdk" ]
@@ -902,6 +918,14 @@ group("create_common_sdk") {
   if (target_cpu == "x64") {
     public_deps += [ ":copy_libtensorflowlite_c" ]
   }
+
+  # CIPD only has versions of the Rust compiler for linux and mac x64 hosts.
+  # We also disallow cross-compialtion (it may be possible in future, but it
+  # isn't ready yet).
+  if (host_cpu == "x64" && (host_os == "linux" || host_os == "mac") &&
+      host_cpu == current_cpu && host_os == current_os && dart_enable_wasm) {
+    public_deps += [ ":copy_wasmer" ]
+  }
 }
 
 # Parts specific to the platform SDK.

third_party/wasmer/BUILD.gn

@@ -1,4 +1,4 @@
-import("//build/rust/rust.gni")
+import("../../build/rust/rust.gni")
 
 component("wasmer") {
   public = [ "wasmer.hh" ]
@@ -7,4 +7,5 @@ component("wasmer") {
 
 rust_library("wasmer_lib") {
   lib_name = "wasmer"
+  shared = true
 }

third_party/wasmer/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.17.1"
 
 [lib]
 name = "wasmer"
-crate-type = ["staticlib"]
+crate-type = ["dylib"]
 path = "wasmer.rs"
 
 [dependencies]