diff --git a/DEPS b/DEPS index 1b7ef2c8451..67aa6c2d4d2 100644 --- a/DEPS +++ b/DEPS @@ -512,7 +512,7 @@ deps = { "packages": [ { "package": "dart/cfe/dart2js_dills", - "version": "binary_version:42", + "version": "binary_version:43_2", } ], "dep_type": "cipd", diff --git a/pkg/front_end/test/binary_md_dill_reader.dart b/pkg/front_end/test/binary_md_dill_reader.dart index 01cf4d98d44..c0a4b5b7fbf 100644 --- a/pkg/front_end/test/binary_md_dill_reader.dart +++ b/pkg/front_end/test/binary_md_dill_reader.dart @@ -473,31 +473,33 @@ class BinaryMdDillReader { type = type.substring(0, type.indexOf("[")); type = _lookupGenericType(typeNames, type, types); - int intCount = -1; - if (vars[count] != null && vars[count] is int) { - intCount = vars[count]; - } else if (count.contains(".")) { - List countData = - count.split(regExpSplit).map((s) => s.trim()).toList(); - if (vars[countData[0]] != null) { - dynamic v = vars[countData[0]]; - if (v is Map && - countData[1] == "last" && - v["items"] is List && - v["items"].last is int) { - intCount = v["items"].last; - } else if (v is Map && v[countData[1]] != null) { - v = v[countData[1]]; - if (v is Map && v[countData[2]] != null) { - v = v[countData[2]]; - if (v is int) intCount = v; - } else if (v is int && - countData.length == 4 && - countData[2] == "+") { - intCount = v + int.parse(countData[3]); + int intCount = int.tryParse(count) ?? -1; + if (intCount == -1) { + if (vars[count] != null && vars[count] is int) { + intCount = vars[count]; + } else if (count.contains(".")) { + List countData = + count.split(regExpSplit).map((s) => s.trim()).toList(); + if (vars[countData[0]] != null) { + dynamic v = vars[countData[0]]; + if (v is Map && + countData[1] == "last" && + v["items"] is List && + v["items"].last is int) { + intCount = v["items"].last; + } else if (v is Map && v[countData[1]] != null) { + v = v[countData[1]]; + if (v is Map && v[countData[2]] != null) { + v = v[countData[2]]; + if (v is int) intCount = v; + } else if (v is int && + countData.length == 4 && + countData[2] == "+") { + intCount = v + int.parse(countData[3]); + } + } else { + throw "Unknown dot to int ($count)"; } - } else { - throw "Unknown dot to int ($count)"; } } } diff --git a/pkg/front_end/test/spell_checking_list_code.txt b/pkg/front_end/test/spell_checking_list_code.txt index a5651d0bf80..0dba39a9549 100644 --- a/pkg/front_end/test/spell_checking_list_code.txt +++ b/pkg/front_end/test/spell_checking_list_code.txt @@ -314,6 +314,7 @@ disallow disambiguator disjoint dispatched +distribute divided dmitryas doc @@ -326,6 +327,7 @@ downloaded downloading dq dquote +dsdk dst dummy dupdate @@ -437,12 +439,14 @@ futured futureor g gardening +gen generation gets getter1a getter1b getting gft +git github glb glob diff --git a/pkg/kernel/binary.md b/pkg/kernel/binary.md index 2b37b2c55e2..72e2057d80a 100644 --- a/pkg/kernel/binary.md +++ b/pkg/kernel/binary.md @@ -143,7 +143,8 @@ type CanonicalName { type ComponentFile { UInt32 magic = 0x90ABCDEF; - UInt32 formatVersion = 42; + UInt32 formatVersion = 43; + Byte[10] shortSdkHash; List problemsAsJson; // Described in problems.md. Library[] libraries; UriSource sourceMap; diff --git a/pkg/kernel/lib/binary/ast_from_binary.dart b/pkg/kernel/lib/binary/ast_from_binary.dart index 9a32677b497..b5e70b69d07 100644 --- a/pkg/kernel/lib/binary/ast_from_binary.dart +++ b/pkg/kernel/lib/binary/ast_from_binary.dart @@ -5,6 +5,7 @@ library kernel.ast_from_binary; import 'dart:core' hide MapEntry; import 'dart:developer'; +import 'dart:convert'; import 'dart:typed_data'; import '../ast.dart'; @@ -28,11 +29,22 @@ class InvalidKernelVersionError { InvalidKernelVersionError(this.version); String toString() { - return 'Unexpected Kernel version ${version} ' + return 'Unexpected Kernel Format Version ${version} ' '(expected ${Tag.BinaryFormatVersion}).'; } } +class InvalidKernelSdkVersionError { + final String version; + + InvalidKernelSdkVersionError(this.version); + + String toString() { + return 'Unexpected Kernel SDK Version ${version} ' + '(expected ${expectedSdkHash}).'; + } +} + class CompilationModeError { final String message; @@ -484,6 +496,13 @@ class BinaryBuilder { if (_bytes.length == 0) throw new StateError("Empty input given."); } + void _readAndVerifySdkHash() { + final sdkHash = ascii.decode(readBytes(sdkHashLength)); + if (!isValidSdkHash(sdkHash)) { + throw InvalidKernelSdkVersionError(sdkHash); + } + } + /// Deserializes a kernel component and stores it in [component]. /// /// When linking with a non-empty component, canonical names must have been @@ -511,6 +530,9 @@ class BinaryBuilder { if (version != Tag.BinaryFormatVersion) { throw InvalidKernelVersionError(version); } + + _readAndVerifySdkHash(); + _byteOffset = offset; List componentFileSizes = _indexComponents(); if (componentFileSizes.length > 1) { @@ -694,6 +716,8 @@ class BinaryBuilder { throw InvalidKernelVersionError(formatVersion); } + _readAndVerifySdkHash(); + // Read component index from the end of this ComponentFiles serialized data. _ComponentIndex index = _readComponentIndex(componentFileSize); @@ -718,6 +742,8 @@ class BinaryBuilder { throw InvalidKernelVersionError(formatVersion); } + _readAndVerifySdkHash(); + List problemsAsJson = readListOfStrings(); if (problemsAsJson != null) { component.problemsAsJson ??= []; diff --git a/pkg/kernel/lib/binary/ast_to_binary.dart b/pkg/kernel/lib/binary/ast_to_binary.dart index 57ccd2fd17e..dc3be1e21b9 100644 --- a/pkg/kernel/lib/binary/ast_to_binary.dart +++ b/pkg/kernel/lib/binary/ast_to_binary.dart @@ -4,6 +4,7 @@ library kernel.ast_to_binary; import 'dart:core' hide MapEntry; +import 'dart:convert'; import 'dart:developer'; import 'dart:io' show BytesBuilder; import 'dart:typed_data'; @@ -537,6 +538,7 @@ class BinaryPrinter implements Visitor, BinarySink { final componentOffset = getBufferOffset(); writeUInt32(Tag.ComponentFile); writeUInt32(Tag.BinaryFormatVersion); + writeBytes(ascii.encode(expectedSdkHash)); writeListOfStrings(component.problemsAsJson); indexLinkTable(component); _collectMetadata(component); diff --git a/pkg/kernel/lib/binary/tag.dart b/pkg/kernel/lib/binary/tag.dart index cec230824aa..79cea32fc6e 100644 --- a/pkg/kernel/lib/binary/tag.dart +++ b/pkg/kernel/lib/binary/tag.dart @@ -149,7 +149,7 @@ class Tag { /// Internal version of kernel binary format. /// Bump it when making incompatible changes in kernel binaries. /// Keep in sync with runtime/vm/kernel_binary.h, pkg/kernel/binary.md. - static const int BinaryFormatVersion = 42; + static const int BinaryFormatVersion = 43; } abstract class ConstantTag { @@ -169,3 +169,27 @@ abstract class ConstantTag { static const int UnevaluatedConstant = 12; // 13 is occupied by [SetConstant] } + +const int sdkHashLength = 10; // Bytes, a Git "short hash". + +const String sdkHashNull = '0000000000'; + +// Will be correct hash for Flutter SDK / Dart SDK we distribute. +// If non-null we will validate when consuming kernel, will use when producing +// kernel. +// If null, local development setting (e.g. run gen_kernel.dart from source), +// we put 0x00..00 into when producing, do not validate when consuming. +String get expectedSdkHash { + final sdkHash = + const String.fromEnvironment('sdk_hash', defaultValue: sdkHashNull); + if (sdkHash.length != sdkHashLength) { + throw '-Dsdk_hash= must be a ${sdkHashLength} byte string!'; + } + return sdkHash; +} + +bool isValidSdkHash(String sdkHash) { + return (sdkHash == sdkHashNull || + expectedSdkHash == sdkHashNull || + sdkHash == expectedSdkHash); +} diff --git a/runtime/bin/BUILD.gn b/runtime/bin/BUILD.gn index 6cdfad16447..7431b64f0a4 100644 --- a/runtime/bin/BUILD.gn +++ b/runtime/bin/BUILD.gn @@ -964,9 +964,11 @@ prebuilt_dart_action("gen_kernel_bytecode_dill") { abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) + vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", + "-Dsdk_hash=$sdk_hash", ] args = [ diff --git a/runtime/tests/vm/dart/sdk_hash_test.dart b/runtime/tests/vm/dart/sdk_hash_test.dart new file mode 100644 index 00000000000..6b05e96271f --- /dev/null +++ b/runtime/tests/vm/dart/sdk_hash_test.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:path/path.dart' as path; +import 'package:expect/expect.dart'; + +import 'snapshot_test_helper.dart'; + +// Keep in sync with pkg/kernel/lib/binary/tag.dart: +const tagComponentFile = [0x90, 0xAB, 0xCD, 0xEF]; +const tagBinaryFormatVersion = [0x00, 0x00, 0x00, 43]; + +Future main(List args) async { + if (args.length == 1 && args[0] == '--child') { + print('Hello, SDK Hash!'); + return; + } + + final String sourcePath = + path.join('runtime', 'tests', 'vm', 'dart_2', 'sdk_hash_test.dart'); + + await withTempDir((String tmp) async { + final String dillPath = path.join(tmp, 'test.dill'); + + { + final result = await Process.run(dart, [ + genKernel, + '--platform', + platformDill, + '-o', + dillPath, + sourcePath, + ]); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('', result.stdout); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!', result.stdout.trim()); + } + + // Invalidate the SDK hash in the kernel dill: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + // The SDK Hash is located after the ComponentFile and BinaryFormatVersion + // tags (both UInt32). + Expect.listEquals(tagComponentFile, bytes.sublist(0, 4)); + Expect.listEquals(tagBinaryFormatVersion, bytes.sublist(4, 8)); + // Flip the first byte in the hash: + bytes[8] ^= bytes[8]; + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals( + 'Can\'t load Kernel binary: Invalid SDK hash.', result.stderr.trim()); + Expect.equals(253, result.exitCode); + Expect.equals('', result.stdout); + } + + // Zero out the SDK hash in the kernel dill to disable the check: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + bytes.setRange(8, 18, ascii.encode('0000000000')); + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!', result.stdout.trim()); + } + }); +} diff --git a/runtime/tests/vm/dart/snapshot_test_helper.dart b/runtime/tests/vm/dart/snapshot_test_helper.dart index 853fc3388e2..974050e1806 100644 --- a/runtime/tests/vm/dart/snapshot_test_helper.dart +++ b/runtime/tests/vm/dart/snapshot_test_helper.dart @@ -48,6 +48,7 @@ final String executableSuffix = Platform.isWindows ? ".exe" : ""; final String buildDir = p.dirname(Platform.executable); final String platformDill = p.join(buildDir, "vm_platform_strong.dill"); final String genSnapshot = p.join(buildDir, "gen_snapshot${executableSuffix}"); +final String dart = p.join(buildDir, "dart${executableSuffix}"); final String dartPrecompiledRuntime = p.join(buildDir, "dart_precompiled_runtime${executableSuffix}"); final String genKernel = p.join("pkg", "vm", "bin", "gen_kernel.dart"); diff --git a/runtime/tests/vm/dart_2/sdk_hash_test.dart b/runtime/tests/vm/dart_2/sdk_hash_test.dart new file mode 100644 index 00000000000..6b05e96271f --- /dev/null +++ b/runtime/tests/vm/dart_2/sdk_hash_test.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:path/path.dart' as path; +import 'package:expect/expect.dart'; + +import 'snapshot_test_helper.dart'; + +// Keep in sync with pkg/kernel/lib/binary/tag.dart: +const tagComponentFile = [0x90, 0xAB, 0xCD, 0xEF]; +const tagBinaryFormatVersion = [0x00, 0x00, 0x00, 43]; + +Future main(List args) async { + if (args.length == 1 && args[0] == '--child') { + print('Hello, SDK Hash!'); + return; + } + + final String sourcePath = + path.join('runtime', 'tests', 'vm', 'dart_2', 'sdk_hash_test.dart'); + + await withTempDir((String tmp) async { + final String dillPath = path.join(tmp, 'test.dill'); + + { + final result = await Process.run(dart, [ + genKernel, + '--platform', + platformDill, + '-o', + dillPath, + sourcePath, + ]); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('', result.stdout); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!', result.stdout.trim()); + } + + // Invalidate the SDK hash in the kernel dill: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + // The SDK Hash is located after the ComponentFile and BinaryFormatVersion + // tags (both UInt32). + Expect.listEquals(tagComponentFile, bytes.sublist(0, 4)); + Expect.listEquals(tagBinaryFormatVersion, bytes.sublist(4, 8)); + // Flip the first byte in the hash: + bytes[8] ^= bytes[8]; + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals( + 'Can\'t load Kernel binary: Invalid SDK hash.', result.stderr.trim()); + Expect.equals(253, result.exitCode); + Expect.equals('', result.stdout); + } + + // Zero out the SDK hash in the kernel dill to disable the check: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + bytes.setRange(8, 18, ascii.encode('0000000000')); + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!', result.stdout.trim()); + } + }); +} diff --git a/runtime/tests/vm/dart_2/snapshot_test_helper.dart b/runtime/tests/vm/dart_2/snapshot_test_helper.dart index 9be810a28e8..b2754d0bf9d 100644 --- a/runtime/tests/vm/dart_2/snapshot_test_helper.dart +++ b/runtime/tests/vm/dart_2/snapshot_test_helper.dart @@ -48,6 +48,7 @@ final String executableSuffix = Platform.isWindows ? ".exe" : ""; final String buildDir = p.dirname(Platform.executable); final String platformDill = p.join(buildDir, "vm_platform_strong.dill"); final String genSnapshot = p.join(buildDir, "gen_snapshot${executableSuffix}"); +final String dart = p.join(buildDir, "dart${executableSuffix}"); final String dartPrecompiledRuntime = p.join(buildDir, "dart_precompiled_runtime${executableSuffix}"); final String genKernel = p.join("pkg", "vm", "bin", "gen_kernel.dart"); diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index 51fbdcc8df1..0ab4172ce68 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -338,11 +338,13 @@ cc/Profiler_TypedArrayAllocation: SkipByDesign cc/Service_Profile: SkipByDesign dart/isolates/thread_pool_test: SkipByDesign # Test uses dart:ffi which is not supported on simulators. dart/regress_41971_test: SkipByDesign # dart:ffi is not supported on simulator +dart/sdk_hash_test: SkipSlow # gen_kernel is slow to run on simarm dart/unboxed_param_args_descriptor_test: SkipByDesign # FFI helper not supported on simulator dart/unboxed_param_tear_off_test: SkipByDesign # FFI helper not supported on simulator dart/unboxed_param_test: SkipByDesign # FFI helper not supported on simulator dart_2/isolates/thread_pool_test: SkipByDesign # Test uses dart:ffi which is not supported on simulators. dart_2/regress_41971_test: SkipByDesign # dart:ffi is not supported on simulator +dart_2/sdk_hash_test: SkipSlow # gen_kernel is slow to run on simarm dart_2/unboxed_param_args_descriptor_test: SkipByDesign # FFI helper not supported on simulator dart_2/unboxed_param_tear_off_test: SkipByDesign # FFI helper not supported on simulator dart_2/unboxed_param_test: SkipByDesign # FFI helper not supported on simulator diff --git a/runtime/vm/compiler/frontend/kernel_translation_helper.cc b/runtime/vm/compiler/frontend/kernel_translation_helper.cc index 58407684162..93a9b490fca 100644 --- a/runtime/vm/compiler/frontend/kernel_translation_helper.cc +++ b/runtime/vm/compiler/frontend/kernel_translation_helper.cc @@ -2654,18 +2654,7 @@ TokenPosition KernelReaderHelper::ReadPosition() { } intptr_t KernelReaderHelper::SourceTableFieldCountFromFirstLibraryOffset() { - // translation_helper_.info() might not be initialized at this point so we - // can't use translation_helper_.info().kernel_binary_version(). - SetOffset(KernelFormatVersionOffset); - uint32_t formatVersion = reader_.ReadUInt32(); - intptr_t count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffsetPre41; - static_assert(kMinSupportedKernelFormatVersion < 41, "cleanup this code"); - if (formatVersion >= 41) { - count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffset41Plus; - } - return count_from_first_library_offset; + return SourceTableFieldCountFromFirstLibraryOffset41Plus; } intptr_t KernelReaderHelper::SourceTableSize() { diff --git a/runtime/vm/kernel_binary.cc b/runtime/vm/kernel_binary.cc index 3720fe23ac7..c34523e495d 100644 --- a/runtime/vm/kernel_binary.cc +++ b/runtime/vm/kernel_binary.cc @@ -15,6 +15,7 @@ #include "vm/kernel.h" #include "vm/object.h" #include "vm/os.h" +#include "vm/version.h" namespace dart { @@ -84,12 +85,17 @@ const char* kKernelInvalidBinaryFormatVersion = "Invalid kernel binary format version"; const char* kKernelInvalidSizeIndicated = "Invalid kernel binary: Indicated size is invalid"; +const char* kKernelInvalidSdkHash = "Invalid SDK hash"; + +const int kSdkHashSizeInBytes = 10; +const char* kSdkHashNull = "0000000000"; std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { - if (reader->size() < 60) { - // A kernel file (v41) currently contains at least the following: + if (reader->size() < 70) { + // A kernel file (v43) currently contains at least the following: // * Magic number (32) // * Kernel version (32) + // * SDK Hash (10 * 8) // * List of problems (8) // * Length of source map (32) // * Length of canonical name table (8) @@ -98,7 +104,7 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { // * Length of constant table (8) // * Component index (11 * 32) // - // so is at least 64 bytes. + // so is at least 74 bytes. // (Technically it will also contain an empty entry in both source map and // string table, taking up another 8 bytes.) if (error != nullptr) { @@ -124,6 +130,18 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { return nullptr; } + uint8_t sdkHash[kSdkHashSizeInBytes + 1]; + reader->ReadBytes(sdkHash, kSdkHashSizeInBytes); + sdkHash[kSdkHashSizeInBytes] = 0; // Null terminate. + if (strcmp(Version::SdkHash(), kSdkHashNull) != 0 && + strcmp((const char*)sdkHash, kSdkHashNull) != 0 && + strcmp((const char*)sdkHash, Version::SdkHash()) != 0) { + if (error != nullptr) { + *error = kKernelInvalidSdkHash; + } + return nullptr; + } + std::unique_ptr program(new Program()); program->binary_version_ = formatVersion; program->typed_data_ = reader->typed_data(); @@ -152,13 +170,8 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { // Read backwards at the end. program->library_count_ = reader->ReadFromIndexNoReset( reader->size_, LibraryCountFieldCountFromEnd, 1, 0); - static_assert(kMinSupportedKernelFormatVersion < 41, "cleanup this code"); intptr_t count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffsetPre41; - if (formatVersion >= 41) { - count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffset41Plus; - } + SourceTableFieldCountFromFirstLibraryOffset41Plus; program->source_table_offset_ = reader->ReadFromIndexNoReset( reader->size_, LibraryCountFieldCountFromEnd + 1 + program->library_count_ + 1 + @@ -171,13 +184,9 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { program->constant_table_offset_ = reader->ReadUInt32(); program->main_method_reference_ = NameIndex(reader->ReadUInt32() - 1); - if (formatVersion >= 41) { - NNBDCompiledMode compilation_mode = - static_cast(reader->ReadUInt32()); - program->compilation_mode_ = compilation_mode; - } else { - program->compilation_mode_ = NNBDCompiledMode::kDisabled; - } + NNBDCompiledMode compilation_mode = + static_cast(reader->ReadUInt32()); + program->compilation_mode_ = compilation_mode; return program; } diff --git a/runtime/vm/kernel_binary.h b/runtime/vm/kernel_binary.h index 11527eba3e4..33dc838a980 100644 --- a/runtime/vm/kernel_binary.h +++ b/runtime/vm/kernel_binary.h @@ -20,8 +20,8 @@ namespace kernel { static const uint32_t kMagicProgramFile = 0x90ABCDEFu; // Both version numbers are inclusive. -static const uint32_t kMinSupportedKernelFormatVersion = 29; -static const uint32_t kMaxSupportedKernelFormatVersion = 42; +static const uint32_t kMinSupportedKernelFormatVersion = 43; +static const uint32_t kMaxSupportedKernelFormatVersion = 43; // Keep in sync with package:kernel/lib/binary/tag.dart #define KERNEL_TAG_LIST(V) \ @@ -326,6 +326,12 @@ class Reader : public ValueObject { uint8_t PeekByte() { return buffer()[offset_]; } + void ReadBytes(uint8_t* buffer, uint8_t size) { + for (int i = 0; i < size; i++) { + buffer[i] = ReadByte(); + } + } + bool ReadBool() { return (ReadByte() & 1) == 1; } uint8_t ReadFlags() { return ReadByte(); } diff --git a/runtime/vm/kernel_loader.cc b/runtime/vm/kernel_loader.cc index 1d0d5437726..37a54697af0 100644 --- a/runtime/vm/kernel_loader.cc +++ b/runtime/vm/kernel_loader.cc @@ -297,7 +297,11 @@ Object& KernelLoader::LoadEntireProgram(Program* program, reader.set_raw_buffer(program->kernel_data() + subprogram_start); reader.set_size(subprogram_end - subprogram_start); reader.set_offset(0); - std::unique_ptr subprogram = Program::ReadFrom(&reader); + const char* error = nullptr; + std::unique_ptr subprogram = Program::ReadFrom(&reader, &error); + if (subprogram == nullptr) { + FATAL1("Failed to load kernel file: %s", error); + } ASSERT(subprogram->is_single_program()); KernelLoader loader(subprogram.get(), &uri_to_source_table); Object& load_result = Object::Handle(loader.LoadProgram(false)); @@ -894,7 +898,11 @@ void KernelLoader::FindModifiedLibraries(Program* program, reader.set_raw_buffer(program->kernel_data() + subprogram_start); reader.set_size(subprogram_end - subprogram_start); reader.set_offset(0); - std::unique_ptr subprogram = Program::ReadFrom(&reader); + const char* error = nullptr; + std::unique_ptr subprogram = Program::ReadFrom(&reader, &error); + if (subprogram == nullptr) { + FATAL1("Failed to load kernel file: %s", error); + } ASSERT(subprogram->is_single_program()); KernelLoader loader(subprogram.get(), /*uri_to_source_table=*/nullptr); loader.walk_incremental_kernel(modified_libs, is_empty_program, diff --git a/runtime/vm/version.h b/runtime/vm/version.h index fdc6bb0f5bc..a3769f6ef44 100644 --- a/runtime/vm/version.h +++ b/runtime/vm/version.h @@ -16,11 +16,13 @@ class Version : public AllStatic { static const char* CommitString(); static int CurrentAbiVersion(); static int OldestSupportedAbiVersion(); + static const char* SdkHash(); private: static const char* str_; static const char* snapshot_hash_; static const char* commit_; + static const char* git_short_hash_; }; } // namespace dart diff --git a/runtime/vm/version_in.cc b/runtime/vm/version_in.cc index b4f1e2d6e11..f63fe3fde8a 100644 --- a/runtime/vm/version_in.cc +++ b/runtime/vm/version_in.cc @@ -38,8 +38,13 @@ int Version::OldestSupportedAbiVersion() { return {{OLDEST_SUPPORTED_ABI_VERSION}}; } +const char* Version::SdkHash() { + return git_short_hash_; +} + const char* Version::snapshot_hash_ = "{{SNAPSHOT_HASH}}"; const char* Version::str_ = "{{VERSION_STR}} ({{CHANNEL}}) ({{COMMIT_TIME}})"; const char* Version::commit_ = "{{VERSION_STR}}"; +const char* Version::git_short_hash_ = "{{GIT_HASH}}"; } // namespace dart diff --git a/sdk_args.gni b/sdk_args.gni index 94e6b7a8eed..06dc3e8540b 100644 --- a/sdk_args.gni +++ b/sdk_args.gni @@ -19,4 +19,22 @@ declare_args() { # to out/ReleaseX64/args.gn. The path above can be extracted from the `.git` # file under the git worktree folder. default_git_folder = "$_dart_root/.git" + + # Whether to enable the SDK hash check that will prevent loading a kernel + # into a VM which was built with a different SDK. + verify_sdk_hash = true +} + +# The SDK hash to build into VM and kernels. +# The value 0000000000 signifies no hash is set, which will disable the check. +if (verify_sdk_hash) { + sdk_hash = exec_script("$_dart_root/tools/make_version.py", + [ "--format={{GIT_HASH}}" ], + "trim string", + [ + "$_dart_root/tools/VERSION", + "$_dart_root/tools/utils.py", + ]) +} else { + sdk_hash = "0000000000" } diff --git a/tools/gn.py b/tools/gn.py index 830c8b3c64b..4bd29b452a5 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -136,7 +136,7 @@ def UseSysroot(args, gn_args): return True -def ToGnArgs(args, mode, arch, target_os, sanitizer): +def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash): gn_args = {} host_os = HostOsForGn(HOST_OS) @@ -258,6 +258,8 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer): gn_args['dart_debug_optimization_level'] = args.debug_opt_level gn_args['debug_optimization_level'] = args.debug_opt_level + gn_args['verify_sdk_hash'] = verify_sdk_hash + return gn_args @@ -466,6 +468,10 @@ def parse_args(args): default=False, dest='use_qemu', action='store_true') + other_group.add_argument('--no-verify-sdk-hash', + help='Disable SDK hash checks.', + default=False, + action='store_true') options = parser.parse_args(args) if not ProcessOptions(options): @@ -506,7 +512,8 @@ def Main(argv): # See dartbug.com/32364 command = [gn, 'gen', out_dir] gn_args = ToCommandLine( - ToGnArgs(args, mode, arch, target_os, sanitizer)) + ToGnArgs(args, mode, arch, target_os, sanitizer, + not args.no_verify_sdk_hash)) gn_args += GetGNArgs(args) if args.verbose: print("gn gen --check in %s" % out_dir) diff --git a/tools/make_version.py b/tools/make_version.py index b0b795539ce..f8356f84982 100755 --- a/tools/make_version.py +++ b/tools/make_version.py @@ -7,19 +7,13 @@ from __future__ import print_function +import argparse import hashlib import os import sys import time -from optparse import OptionParser import utils - -def debugLog(message): - print(message, file=sys.stderr) - sys.stderr.flush() - - # When these files change, snapshots created by the VM are potentially no longer # backwards-compatible. VM_SNAPSHOT_FILES = [ @@ -45,142 +39,152 @@ VM_SNAPSHOT_FILES = [ ] -def MakeVersionString(quiet, no_git_hash, custom_for_pub=None): - channel = utils.GetChannel() - - if custom_for_pub and channel == 'be': - latest = utils.GetLatestDevTag() - if not latest: - # If grabbing the dev tag fails, then fall back on the VERSION file. - latest = utils.GetSemanticSDKVersion(no_git_hash=True) - if no_git_hash: - version_string = ("%s.%s" % (latest, custom_for_pub)) - else: - git_hash = utils.GetShortGitHash() - version_string = ("%s.%s-%s" % (latest, custom_for_pub, git_hash)) - else: - version_string = utils.GetSemanticSDKVersion(no_git_hash=no_git_hash) - if not quiet: - debugLog("Returning version string: %s " % version_string) - return version_string - - def MakeSnapshotHashString(): vmhash = hashlib.md5() for vmfilename in VM_SNAPSHOT_FILES: vmfilepath = os.path.join(utils.DART_DIR, 'runtime', 'vm', vmfilename) - with open(vmfilepath) as vmfile: - vmhash.update(vmfile.read().encode('utf-8')) + with open(vmfilepath, 'rb') as vmfile: + vmhash.update(vmfile.read()) return vmhash.hexdigest() -def MakeFile(quiet, - output_file, - input_file, - no_git_hash, - custom_for_pub, - version_file=None): - if version_file: - version_string = utils.GetVersion(no_git_hash, version_file) - else: - version_string = MakeVersionString(quiet, no_git_hash, custom_for_pub) +def GetSemanticVersionFormat(no_git_hash, custom_for_pub): + version_format = '{{SEMANTIC_SDK_VERSION}}' + + if custom_for_pub and utils.GetChannel() == 'be': + if no_git_hash: + version_format = '{{LATEST}}.{{PUB_CUSTOM}}' + else: + version_format = '{{LATEST}}.{{PUB_CUSTOM}}-{{GIT_HASH}}' + + return version_format + + +def FormatVersionString(version, + no_git_hash, + custom_for_pub, + version_file=None, + git_revision_file=None): + use_git_hash = not no_git_hash + + semantic_sdk_version = utils.GetSemanticSDKVersion(no_git_hash, + version_file, + git_revision_file) + semantic_version_format = GetSemanticVersionFormat(no_git_hash, + custom_for_pub) + version_str = (semantic_sdk_version + if version_file else semantic_version_format) + + version = version.replace('{{VERSION_STR}}', version_str) + + version = version.replace('{{SEMANTIC_SDK_VERSION}}', semantic_sdk_version) + + if custom_for_pub: + # LATEST is only used for custom_for_pub. + latest = None + if use_git_hash: + # If grabbing the dev tag fails, then fall back on the default VERSION file. + latest = utils.GetLatestDevTag() + if not latest: + latest = utils.GetSemanticSDKVersion(no_git_hash=True) + version = version.replace('{{LATEST}}', latest) + + version = version.replace('{{PUB_CUSTOM}}', custom_for_pub) + + git_hash = None + if use_git_hash: + git_hash = utils.GetShortGitHash() + if git_hash is None or len(git_hash) != 10: + git_hash = '0000000000' + version = version.replace('{{GIT_HASH}}', git_hash) - version_cc_text = open(input_file).read() - version_cc_text = version_cc_text.replace("{{VERSION_STR}}", version_string) channel = utils.GetChannel() - version_cc_text = version_cc_text.replace("{{CHANNEL}}", channel) - version_time = utils.GetGitTimestamp() - if no_git_hash or version_time == None: - version_time = "Unknown timestamp" - version_cc_text = version_cc_text.replace("{{COMMIT_TIME}}", - version_time.decode("utf-8")) + version = version.replace('{{CHANNEL}}', channel) + + version_time = None + if use_git_hash: + version_time = utils.GetGitTimestamp() + if version_time == None: + version_time = 'Unknown timestamp' + version = version.replace('{{COMMIT_TIME}}', version_time.decode('utf-8')) + abi_version = utils.GetAbiVersion(version_file) - version_cc_text = version_cc_text.replace("{{ABI_VERSION}}", abi_version) + version = version.replace('{{ABI_VERSION}}', abi_version) + oldest_supported_abi_version = utils.GetOldestSupportedAbiVersion( version_file) - version_cc_text = version_cc_text.replace( - "{{OLDEST_SUPPORTED_ABI_VERSION}}", oldest_supported_abi_version) + version = version.replace('{{OLDEST_SUPPORTED_ABI_VERSION}}', + oldest_supported_abi_version) + snapshot_hash = MakeSnapshotHashString() - version_cc_text = version_cc_text.replace("{{SNAPSHOT_HASH}}", - snapshot_hash) - open(output_file, 'w').write(version_cc_text) - return True + version = version.replace('{{SNAPSHOT_HASH}}', snapshot_hash) + + return version -def main(args): +def main(): try: # Parse input. - parser = OptionParser() - parser.add_option( - "--custom_for_pub", - action="store", - type="string", - help=("Generates a version string that works with pub that includes" - "the given string. This is silently ignored on channels other" - "than be")) - parser.add_option( - "--input", - action="store", - type="string", - help="input template file.") - parser.add_option( - "--no_git_hash", - action="store_true", + parser = argparse.ArgumentParser() + parser.add_argument( + '--custom_for_pub', + help=('Generates a version string that works with pub that ' + 'includes the given string. This is silently ignored on ' + 'channels other than be.')) + parser.add_argument('--input', help='Input template file.') + parser.add_argument( + '--no_git_hash', + action='store_true', default=False, - help="Don't try to determine svn revision") - parser.add_option( - "--output", action="store", type="string", help="output file name") - parser.add_option( - "-q", - "--quiet", - action="store_true", - default=False, - help="disable console output") - parser.add_option( - "--version-file", - action="store", - type="string", - default=None, - help="Version file") + help=('Don\'t try to call git to derive things like ' + 'git revision hash.')) + parser.add_argument('--output', help='output file name') + parser.add_argument('-q', + '--quiet', + action='store_true', + default=False, + help='DEPRECATED: Does nothing!') + parser.add_argument('--version-file', help='Path to the VERSION file.') + parser.add_argument('--git-revision-file', + help='Path to the GIT_REVISION file.') + parser.add_argument( + '--format', + default='{{VERSION_STR}}', + help='Version format used if no input template is given.') - (options, args) = parser.parse_args() + args = parser.parse_args() # If there is no input template, then write the bare version string to - # options.output. If there is no options.output, then write the version + # args.output. If there is no args.output, then write the version # string to stdout. - if not options.input: - version_string = MakeVersionString( - options.quiet, options.no_git_hash, options.custom_for_pub) - if options.output: - open(options.output, 'w').write(version_string) - else: - print(version_string) - return 0 - if not options.output: - sys.stderr.write('--output not specified\n') - return -1 - if not options.input: - sys.stderr.write('--input not specified\n') - return -1 + version_template = '' + if args.input: + version_template = open(args.input).read() + elif not args.format is None: + version_template = args.format + else: + raise 'No version template given! Set either --input or --format.' - files = [] - for arg in args: - files.append(arg) + version = FormatVersionString(version_template, args.no_git_hash, + args.custom_for_pub, args.version_file, + args.git_revision_file) - if not MakeFile(options.quiet, options.output, options.input, - options.no_git_hash, options.custom_for_pub, - options.version_file): - return -1 + if args.output: + with open(args.output, 'w') as fh: + fh.write(version) + else: + sys.stdout.write(version) return 0 + except Exception as inst: sys.stderr.write('make_version.py exception\n') sys.stderr.write(str(inst)) sys.stderr.write('\n') + return -1 if __name__ == '__main__': - exit_code = main(sys.argv) - sys.exit(exit_code) + sys.exit(main()) diff --git a/tools/utils.py b/tools/utils.py index dda4389971f..42a7db417a9 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -349,34 +349,36 @@ def GetBuildSdkBin(host_os, mode=None, arch=None, target_os=None): return os.path.join(build_root, 'dart-sdk', 'bin') -def GetShortVersion(): - version = ReadVersionFile() +def GetShortVersion(version_file=None): + version = ReadVersionFile(version_file) return ('{}.{}.{}.{}.{}'.format(version.major, version.minor, version.patch, version.prerelease, version.prerelease_patch)) -def GetSemanticSDKVersion(no_git_hash=False, version_file=None): +def GetSemanticSDKVersion(no_git_hash=False, + version_file=None, + git_revision_file=None): version = ReadVersionFile(version_file) if not version: return None + suffix = '' if version.channel == 'be': - postfix = '-edge' if no_git_hash else '-edge.{}'.format( - GetGitRevision()) + suffix = '-edge' if no_git_hash else '-edge.{}'.format( + GetGitRevision(git_revision_file)) elif version.channel in ('beta', 'dev'): - postfix = '-{}.{}.{}'.format(version.prerelease, - version.prerelease_patch, version.channel) + suffix = '-{}.{}.{}'.format(version.prerelease, + version.prerelease_patch, version.channel) else: assert version.channel == 'stable' - postfix = '' return '{}.{}.{}{}'.format(version.major, version.minor, version.patch, - postfix) + suffix) -def GetVersion(no_git_hash=False, version_file=None): - return GetSemanticSDKVersion(no_git_hash, version_file) +def GetVersion(no_git_hash=False, version_file=None, git_revision_file=None): + return GetSemanticSDKVersion(no_git_hash, version_file, git_revision_file) # The editor used to produce the VERSION file put on gcs. We now produce this @@ -396,8 +398,8 @@ def GetVersionFileContent(): return json.dumps(result, indent=2) -def GetChannel(): - version = ReadVersionFile() +def GetChannel(version_file=None): + version = ReadVersionFile(version_file) return version.channel @@ -466,8 +468,8 @@ def ReadVersionFile(version_file=None): # We only use numbers on the master branch (bleeding edge). On branches # we use the version number instead for archiving purposes. # The number on master is the count of commits on the master branch. -def GetArchiveVersion(): - version = ReadVersionFile() +def GetArchiveVersion(version_file=None): + version = ReadVersionFile(version_file=None) if not version: raise 'Could not get the archive version, parsing the version file failed' if version.channel in ['be', 'integration']: @@ -475,9 +477,10 @@ def GetArchiveVersion(): return GetSemanticSDKVersion() -def GetGitRevision(repo_path=DART_DIR): +def GetGitRevision(git_revision_file=None, repo_path=DART_DIR): # When building from tarball use tools/GIT_REVISION - git_revision_file = os.path.join(repo_path, 'tools', 'GIT_REVISION') + if git_revision_file is None: + git_revision_file = os.path.join(repo_path, 'tools', 'GIT_REVISION') try: with open(git_revision_file) as fd: return fd.read().decode('utf-8').strip() @@ -493,7 +496,8 @@ def GetGitRevision(repo_path=DART_DIR): # We expect a full git hash if len(revision) != 40: print('Warning: Could not parse git commit, output was {}'.format( - revision)) + revision), + file=sys.stderr) return None return revision @@ -512,12 +516,14 @@ def GetShortGitHash(repo_path=DART_DIR): def GetLatestDevTag(repo_path=DART_DIR): + # We used the old, pre-git2.13 refname:strip here since lstrip will fail on + # older git versions. strip is an alias for lstrip in later versions. cmd = [ 'git', 'for-each-ref', 'refs/tags/*dev*', '--sort=-taggerdate', - "--format=%(refname:lstrip=2)", + "--format=%(refname:strip=2)", '--count=1', ] p = subprocess.Popen(cmd, @@ -529,7 +535,8 @@ def GetLatestDevTag(repo_path=DART_DIR): tag = output.decode('utf-8').strip() if p.wait() != 0: print('Warning: Could not get the most recent dev branch tag {}'.format( - tag)) + tag), + file=sys.stderr) return None return tag @@ -560,7 +567,8 @@ def GetGitNumber(repo_path=DART_DIR): return number + GIT_NUMBER_BASE except: print( - 'Warning: Could not parse git count, output was {}'.format(number)) + 'Warning: Could not parse git count, output was {}'.format(number), + file=sys.stderr) return None diff --git a/utils/application_snapshot.gni b/utils/application_snapshot.gni index 10743175143..8f3f8822730 100644 --- a/utils/application_snapshot.gni +++ b/utils/application_snapshot.gni @@ -98,9 +98,11 @@ template("_application_snapshot") { depfile = "$output.d" abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) + vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", + "-Dsdk_hash=$sdk_hash", ] script = gen_kernel_script diff --git a/utils/compile_platform.gni b/utils/compile_platform.gni index a2e94b3cd61..447ed644a04 100644 --- a/utils/compile_platform.gni +++ b/utils/compile_platform.gni @@ -3,6 +3,7 @@ # BSD-style license that can be found in the LICENSE file. import("../build/dart/dart_action.gni") +import("../sdk_args.gni") _dart_root = get_path_info("..", "abspath") @@ -46,6 +47,8 @@ template("compile_platform") { outputs = invoker.outputs + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + inputs = [] deps = [] args = [] diff --git a/utils/dartanalyzer/BUILD.gn b/utils/dartanalyzer/BUILD.gn index f1d9b014c7a..48c9dcf8668 100644 --- a/utils/dartanalyzer/BUILD.gn +++ b/utils/dartanalyzer/BUILD.gn @@ -44,6 +44,7 @@ prebuilt_dart_action("generate_summary_strong") { inputs = sdk_lib_files + analyzer_files output = "$root_gen_dir/strong.sum" outputs = [ output ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] args = [ "build", rebase_path(output), diff --git a/utils/dartdevc/BUILD.gn b/utils/dartdevc/BUILD.gn index 15bfe16d828..c9ff6555035 100644 --- a/utils/dartdevc/BUILD.gn +++ b/utils/dartdevc/BUILD.gn @@ -82,6 +82,8 @@ template("dart2js_compile") { packages = "../../.packages" + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + args = [ "$abs_main", "-m", @@ -118,6 +120,8 @@ prebuilt_dart_action("dartdevc_patch_sdk") { # TODO(rnystrom): List the outputs more precisely? outputs = [ "$patched_sdk_dir/version" ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + args = [ "--libraries", rebase_path("$sdk_root/lib/libraries.json"), @@ -230,6 +234,8 @@ template("dartdevc_kernel_compile") { "$js_gen_dir/$module.js", ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + args = [ "-k", "--dart-sdk-summary=$sdk_path", @@ -384,6 +390,8 @@ template("dartdevc_sdk_js") { "$js_gen_dir/legacy/dart_sdk.js.map", ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + script = "../../pkg/dev_compiler/bin/dartdevc.dart" args = [ diff --git a/utils/kernel-service/BUILD.gn b/utils/kernel-service/BUILD.gn index 062c3ff108b..a01bb1b5411 100644 --- a/utils/kernel-service/BUILD.gn +++ b/utils/kernel-service/BUILD.gn @@ -85,9 +85,11 @@ template("kernel_service_dill") { depfile = "$root_gen_dir/kernel_service" + invoker.target_name + "_dill.d" abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) + vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", + "-Dsdk_hash=$sdk_hash", ] script = gen_kernel_script