[vm] Toward deterministic builds.

- Remove random build-id.
 - Replace build time in embedded version string with commit time.
 - Remove timestamps from Observatory tarball.
 - Zero-initialize skipped bytes in snapshot streams.
 - Fix uninitialized fields in PatchClass, Script and Library.
 - Disable (under flag) random identity hashes and concurrent GC.

Bug: https://github.com/dart-lang/sdk/issues/31427
Change-Id: I3e95de679c8372841cd27ca60df78d9b00ffbfe1
Reviewed-on: https://dart-review.googlesource.com/22901
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Zach Anderson <zra@google.com>

build/config/compiler/BUILD.gn

@@ -234,6 +234,7 @@ config("compiler") {
       "-Wl,-z,noexecstack",
       "-Wl,-z,now",
       "-Wl,-z,relro",
+      "-Wl,--build-id=none",
     ]
   }
 

runtime/bin/BUILD.gn

@@ -232,10 +232,10 @@ template("build_libdart_builtin") {
   }
   static_library(target_name) {
     configs += [
-      "..:dart_arch_config",
-      "..:dart_config",
-      "..:dart_os_config",
-    ] + extra_configs
+                 "..:dart_arch_config",
+                 "..:dart_config",
+                 "..:dart_os_config",
+               ] + extra_configs
     if (is_fuchsia) {
       configs -= [ "//build/config:symbol_visibility_hidden" ]
     }
@@ -404,9 +404,7 @@ template("build_gen_snapshot_dart_io") {
       configs -= [ "//build/config:symbol_visibility_hidden" ]
     }
 
-    deps += [
-      "$dart_zlib_path",
-    ]
+    deps += [ "$dart_zlib_path" ]
 
     custom_sources_filter = [
       "*_test.cc",
@@ -459,10 +457,10 @@ template("dart_io") {
   }
   source_set(target_name) {
     configs += [
-      "..:dart_arch_config",
-      "..:dart_config",
-      "..:dart_os_config",
-    ] + extra_configs
+                 "..:dart_arch_config",
+                 "..:dart_config",
+                 "..:dart_os_config",
+               ] + extra_configs
     if (is_fuchsia) {
       configs -= [ "//build/config:symbol_visibility_hidden" ]
     }
@@ -569,6 +567,7 @@ compiled_action("generate_snapshot_bin") {
   ]
   tool = ":gen_snapshot"
   args = [
+    "--deterministic",
     "--snapshot_kind=" + dart_core_snapshot_kind,
     "--vm_snapshot_data=" + rebase_path(vm_snapshot_data, root_build_dir),
     "--vm_snapshot_instructions=" +
@@ -946,15 +945,14 @@ if (is_fuchsia) {
   package("dart_tests") {
     system_image = true
 
-    deps = [ ":hello_fuchsia" ]
-
-    binaries = [
-      {
-        name = "hello_fuchsia.dart"
-      },
+    deps = [
+      ":hello_fuchsia",
     ]
-  }
 
+    binaries = [ {
+          name = "hello_fuchsia.dart"
+        } ]
+  }
 }
 
 executable("process_test") {

runtime/tools/create_archive.py

@@ -14,11 +14,10 @@ from optparse import OptionParser
 from datetime import date
 import tarfile
 import tempfile
+import gzip
 
 def makeArchive(tar_path, client_root, compress, files):
   mode_string = 'w'
-  if compress:
-    mode_string = 'w:gz'
   tar = tarfile.open(tar_path, mode=mode_string)
   for input_file_name in files:
     # Chop off client_root.
@@ -30,9 +29,18 @@ def makeArchive(tar_path, client_root, compress, files):
       tarInfo = tarfile.TarInfo(name=archive_file_name)
       input_file.seek(0,2)
       tarInfo.size = input_file.tell()
+      tarInfo.mtime = 0  # For deterministic builds.
       input_file.seek(0)
       tar.addfile(tarInfo, fileobj=input_file)
   tar.close()
+  if compress:
+    with open(tar_path, "rb") as fin:
+      uncompressed = fin.read()
+    with open(tar_path, "wb") as fout:
+      # mtime=0 for deterministic builds.
+      gz = gzip.GzipFile(fileobj=fout, mode="wb", filename="", mtime=0)
+      gz.write(uncompressed)
+      gz.close()
 
 def writeCCFile(output_file,
                 outer_namespace,

runtime/vm/clustered_snapshot.h

@@ -186,7 +186,7 @@ class Serializer : public StackResource {
 
   void ReserveHeader() {
     // Make room for recording snapshot buffer size.
-    stream_.set_current(stream_.buffer() + Snapshot::kHeaderSize);
+    stream_.SetPosition(Snapshot::kHeaderSize);
   }
 
   void FillHeader(Snapshot::Kind kind) {

runtime/vm/compiler/jit/compiler.cc

@@ -116,6 +116,7 @@ static void PrecompilationModeHandler(bool value) {
 #endif
 
     FLAG_background_compilation = false;
+    FLAG_collect_code = false;
     FLAG_enable_mirrors = false;
     FLAG_fields_may_be_reset = true;
     FLAG_interpret_irregexp = true;
@@ -136,7 +137,6 @@ static void PrecompilationModeHandler(bool value) {
 #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
     // Set flags affecting runtime accordingly for dart_bootstrap.
     // These flags are constants with PRODUCT and DART_PRECOMPILED_RUNTIME.
-    FLAG_collect_code = false;
     FLAG_deoptimize_alot = false;  // Used in some tests.
     FLAG_deoptimize_every = 0;     // Used in some tests.
     FLAG_load_deferred_eagerly = true;

runtime/vm/datastream.h

@@ -74,7 +74,6 @@ class ReadStream : public ValueObject {
   intptr_t ReadUnsigned() { return Read<intptr_t>(kEndUnsignedByteMarker); }
 
   intptr_t Position() const { return current_ - buffer_; }
-
   void SetPosition(intptr_t value) {
     ASSERT((end_ - buffer_) > value);
     current_ = buffer_ + value;
@@ -312,12 +311,14 @@ class WriteStream : public ValueObject {
   void set_buffer(uint8_t* value) { *buffer_ = value; }
   intptr_t bytes_written() const { return current_ - *buffer_; }
 
-  void set_current(uint8_t* value) { current_ = value; }
+  intptr_t Position() const { return current_ - *buffer_; }
+  void SetPosition(intptr_t value) { current_ = *buffer_ + value; }
 
   void Align(intptr_t alignment) {
-    intptr_t position = current_ - *buffer_;
-    position = Utils::RoundUp(position, alignment);
-    current_ = *buffer_ + position;
+    intptr_t position_before = Position();
+    intptr_t position_after = Utils::RoundUp(position_before, alignment);
+    memset(current_, 0, position_after - position_before);
+    SetPosition(position_after);
   }
 
   template <int N, typename T>

runtime/vm/flag_list.h

@@ -53,11 +53,10 @@
   R(break_at_isolate_spawn, false, bool, false,                                \
     "Insert a one-time breakpoint at the entrypoint for all spawned isolates") \
   P(causal_async_stacks, bool, !USING_PRODUCT, "Improved async stacks")        \
-  C(collect_code, false, true, bool, true,                                     \
-    "Attempt to GC infrequently used code.")                                   \
+  P(collect_code, bool, true, "Attempt to GC infrequently used code.")         \
   P(collect_dynamic_function_names, bool, true,                                \
     "Collects all dynamic function names to identify unique targets")          \
-  R(concurrent_sweep, USING_MULTICORE, bool, USING_MULTICORE,                  \
+  P(concurrent_sweep, bool, USING_MULTICORE,                                   \
     "Concurrent sweep for old generation.")                                    \
   R(dedup_instructions, true, bool, false,                                     \
     "Canonicalize instructions when precompiling.")                            \

runtime/vm/isolate.cc

@@ -80,6 +80,26 @@ DEFINE_FLAG_HANDLER(CheckedModeHandler,
 DEFINE_FLAG_HANDLER(CheckedModeHandler, checked, "Enable checked mode.");
 #endif  // !defined(PRODUCT)
 
+static void DeterministicModeHandler(bool value) {
+  if (value) {
+    FLAG_background_compilation = false;
+    FLAG_collect_code = false;
+    // Parallel marking doesn't introduce non-determinism in the object
+    // iteration order.
+    FLAG_concurrent_sweep = false;
+    FLAG_random_seed = 0x44617274;  // "Dart"
+#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
+    FLAG_load_deferred_eagerly = true;
+#else
+    COMPILE_ASSERT(FLAG_load_deferred_eagerly);
+#endif
+  }
+}
+
+DEFINE_FLAG_HANDLER(DeterministicModeHandler,
+                    deterministic,
+                    "Enable deterministic mode.");
+
 // Quick access to the locally defined thread() and isolate() methods.
 #define T (thread())
 #define I (isolate())

runtime/vm/object.cc

@@ -2099,10 +2099,15 @@ RawClass* Class::New() {
   }
   FakeObject fake;
   result.set_handle_vtable(fake.vtable());
+  result.set_token_pos(TokenPosition::kNoSource);
   result.set_instance_size(FakeObject::InstanceSize());
+  result.set_type_arguments_field_offset_in_words(kNoTypeArguments);
   result.set_next_field_offset(FakeObject::NextFieldOffset());
   COMPILE_ASSERT((FakeObject::kClassId != kInstanceCid));
   result.set_id(FakeObject::kClassId);
+  result.set_num_type_arguments(0);
+  result.set_num_own_type_arguments(0);
+  result.set_num_native_fields(0);
   result.set_state_bits(0);
   if ((FakeObject::kClassId < kInstanceCid) ||
       (FakeObject::kClassId == kTypeArgumentsCid)) {
@@ -2114,11 +2119,6 @@ RawClass* Class::New() {
     // references, but do not recompute size.
     result.set_is_prefinalized();
   }
-  result.set_type_arguments_field_offset_in_words(kNoTypeArguments);
-  result.set_num_type_arguments(0);
-  result.set_num_own_type_arguments(0);
-  result.set_num_native_fields(0);
-  result.set_token_pos(TokenPosition::kNoSource);
   result.set_kernel_offset(-1);
   result.InitEmptyFields();
   Isolate::Current()->RegisterClass(result);
@@ -3240,15 +3240,15 @@ RawClass* Class::NewCommon(intptr_t index) {
   FakeInstance fake;
   ASSERT(fake.IsInstance());
   result.set_handle_vtable(fake.vtable());
+  result.set_token_pos(TokenPosition::kNoSource);
   result.set_instance_size(FakeInstance::InstanceSize());
+  result.set_type_arguments_field_offset_in_words(kNoTypeArguments);
   result.set_next_field_offset(FakeInstance::NextFieldOffset());
   result.set_id(index);
-  result.set_state_bits(0);
-  result.set_type_arguments_field_offset_in_words(kNoTypeArguments);
   result.set_num_type_arguments(kUnknownNumTypeArguments);
   result.set_num_own_type_arguments(kUnknownNumTypeArguments);
   result.set_num_native_fields(0);
-  result.set_token_pos(TokenPosition::kNoSource);
+  result.set_state_bits(0);
   result.InitEmptyFields();
   return result.raw();
 }
@@ -5212,6 +5212,7 @@ RawPatchClass* PatchClass::New(const Class& patched_class,
   result.set_patched_class(patched_class);
   result.set_origin_class(origin_class);
   result.set_script(Script::Handle(origin_class.script()));
+  result.set_library_kernel_offset(-1);
   return result.raw();
 }
 
@@ -5221,6 +5222,7 @@ RawPatchClass* PatchClass::New(const Class& patched_class,
   result.set_patched_class(patched_class);
   result.set_origin_class(patched_class);
   result.set_script(script);
+  result.set_library_kernel_offset(-1);
   return result.raw();
 }
 
@@ -9843,10 +9845,11 @@ RawScript* Script::New(const String& url,
   result.set_resolved_url(
       String::Handle(zone, Symbols::New(thread, resolved_url)));
   result.set_source(source);
+  result.SetLocationOffset(0, 0);
   result.set_kind(kind);
+  result.set_kernel_script_index(0);
   result.set_load_timestamp(
       FLAG_remove_script_timestamps_for_test ? 0 : OS::GetCurrentTimeMillis());
-  result.SetLocationOffset(0, 0);
   return result.raw();
 }
 
@@ -11099,6 +11102,7 @@ RawLibrary* Library::NewLibraryHelper(const String& url, bool import_core_lib) {
     result.set_debuggable(true);
   }
   result.set_is_dart_scheme(dart_scheme);
+  result.set_kernel_offset(-1);
   result.StoreNonPointer(&result.raw_ptr()->load_state_,
                          RawLibrary::kAllocated);
   result.StoreNonPointer(&result.raw_ptr()->index_, -1);

runtime/vm/object.h

@@ -3792,7 +3792,7 @@ class Library : public Object {
     return -1;
 #endif
   }
-  void set_kernel_offset(intptr_t offset) {
+  void set_kernel_offset(intptr_t offset) const {
     NOT_IN_PRECOMPILED(StoreNonPointer(&raw_ptr()->kernel_offset_, offset));
   }
 

runtime/vm/object_store.h

@@ -130,8 +130,6 @@ class ObjectPointerVisitor;
 
 // The object store is a per isolate instance which stores references to
 // objects used by the VM.
-// TODO(iposva): Move the actual store into the object heap for quick handling
-// by snapshots eventually.
 class ObjectStore {
  public:
   enum BootstrapLibraryId {

runtime/vm/random.cc

@@ -9,7 +9,10 @@
 
 namespace dart {
 
-DEFINE_FLAG(int, random_seed, 0, "Override the random seed for debugging.");
+DEFINE_FLAG(uint64_t,
+            random_seed,
+            0,
+            "Override the random seed for debugging.");
 
 Random::Random() {
   uint64_t seed = FLAG_random_seed;

runtime/vm/random.h

@@ -6,10 +6,13 @@
 #define RUNTIME_VM_RANDOM_H_
 
 #include "vm/allocation.h"
+#include "vm/flags.h"
 #include "vm/globals.h"
 
 namespace dart {
 
+DECLARE_FLAG(uint64_t, random_seed);
+
 class Random {
  public:
   Random();

runtime/vm/raw_object_snapshot.cc

@@ -12,8 +12,6 @@
 
 namespace dart {
 
-DECLARE_FLAG(bool, remove_script_timestamps_for_test);
-
 #define OFFSET_OF_FROM(obj)                                                    \
   obj.raw()->from() - reinterpret_cast<RawObject**>(obj.raw()->ptr())
 
@@ -1049,6 +1047,7 @@ RawScript* Script::ReadFrom(SnapshotReader* reader,
   script.StoreNonPointer(&script.raw_ptr()->kind_, reader->Read<int8_t>());
   script.StoreNonPointer(&script.raw_ptr()->kernel_script_index_,
                          reader->Read<int32_t>());
+  script.StoreNonPointer(&script.raw_ptr()->load_timestamp_, 0);
 
   *reader->StringHandle() ^= String::null();
   script.set_source(*reader->StringHandle());
@@ -1065,9 +1064,6 @@ RawScript* Script::ReadFrom(SnapshotReader* reader,
                         reader->PassiveObjectHandle()->raw());
   }
 
-  script.set_load_timestamp(
-      FLAG_remove_script_timestamps_for_test ? 0 : OS::GetCurrentTimeMillis());
-
   return script.raw();
 }
 

runtime/vm/snapshot.h

@@ -570,7 +570,7 @@ class BaseWriter : public StackResource {
 
   void ReserveHeader() {
     // Make room for recording snapshot buffer size.
-    stream_.set_current(stream_.buffer() + Snapshot::kHeaderSize);
+    stream_.SetPosition(Snapshot::kHeaderSize);
   }
 
   void FillHeader(Snapshot::Kind kind) {

runtime/vm/version_in.cc

@@ -30,7 +30,7 @@ const char* Version::CommitString() {
 }
 
 const char* Version::snapshot_hash_ = "{{SNAPSHOT_HASH}}";
-const char* Version::str_ = "{{VERSION_STR}} ({{BUILD_TIME}})";
+const char* Version::str_ = "{{VERSION_STR}} ({{COMMIT_TIME}})";
 const char* Version::commit_ = "{{VERSION_STR}}";
 
 }  // namespace dart

tools/make_version.py

@@ -61,8 +61,10 @@ def makeFile(quiet, output_file, input_file, ignore_svn_revision):
   version_string = makeVersionString(quiet, ignore_svn_revision)
   version_cc_text = version_cc_text.replace("{{VERSION_STR}}",
                                             version_string)
-  version_time = time.ctime(time.time())
-  version_cc_text = version_cc_text.replace("{{BUILD_TIME}}",
+  version_time = utils.GetGitTimestamp()
+  if version_time == None:
+    version_time = "Unknown timestamp"
+  version_cc_text = version_cc_text.replace("{{COMMIT_TIME}}",
                                             version_time)
   snapshot_hash = makeSnapshotHashString()
   version_cc_text = version_cc_text.replace("{{SNAPSHOT_HASH}}",

tools/utils.py

@@ -425,6 +425,16 @@ def GetGitRevision():
     return None
   return output
 
+def GetGitTimestamp():
+  p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%cd'],
+                       stdout = subprocess.PIPE,
+                       stderr = subprocess.STDOUT, shell=IsWindows(),
+                       cwd = DART_DIR)
+  output, _ = p.communicate()
+  if p.wait() != 0:
+    return None
+  return output
+
 # To eliminate clashing with older archived builds on bleeding edge we add
 # a base number bigger the largest svn revision (this also gives us an easy
 # way of seeing if an archive comes from git based or svn based commits).
@@ -545,6 +555,7 @@ def Main():
   print "IsWindows() -> ", IsWindows()
   print "GuessVisualStudioPath() -> ", GuessVisualStudioPath()
   print "GetGitRevision() -> ", GetGitRevision()
+  print "GetGitTimestamp() -> ", GetGitTimestamp()
   print "GetVersionFileContent() -> ", GetVersionFileContent()
   print "GetGitNumber() -> ", GetGitNumber()
 

utils/application_snapshot.gni

@@ -55,6 +55,7 @@ template("application_snapshot") {
     main_file = rebase_path(main_dart)
 
     args = [
+      "--deterministic",
       "--packages=$dot_packages",
       "--snapshot=$abs_output",
     ]