diff --git a/DEPS b/DEPS index d6be5ede4b8..ac2c4ce09ed 100644 --- a/DEPS +++ b/DEPS @@ -66,8 +66,8 @@ vars = { # The list of revisions for these tools comes from Fuchsia, here: # https://fuchsia.googlesource.com/integration/+/HEAD/toolchain # If there are problems with the toolchain, contact fuchsia-toolchain@. - "clang_revision": "c2592c374e469f343ecea82d6728609650924259", - "gn_revision": "d7c2209cebcfe37f46dba7be4e1a7000ffc342fb", + "clang_revision": "60d276923902051192eba692e5312e605c9d9f65", + "gn_revision": "0bcd37bd2b83f1a9ee17088037ebdfe6eab6d31a", # Ninja, runs the build based on files generated by GN. "ninja_tag": "version:2@1.11.1.chromium.4", @@ -466,8 +466,7 @@ deps = { "version": "git_revision:" + Var("clang_revision"), }, ], - # TODO(https://fxbug.dev/73385): Use arm64 toolchain on arm64 when it exists. - "condition": "host_cpu == x64 and host_os == mac or host_cpu == arm64 and host_os == mac", + "condition": "host_os == mac", # On ARM64 Macs too because Goma doesn't support the host-arm64 toolchain. "dep_type": "cipd", }, Var("dart_root") + "/buildtools/win-x64/clang": { @@ -490,6 +489,16 @@ deps = { "condition": "host_os == 'linux' and host_cpu == 'arm64'", "dep_type": "cipd", }, + Var("dart_root") + "/buildtools/mac-arm64/clang": { + "packages": [ + { + "package": "fuchsia/third_party/clang/mac-arm64", + "version": "git_revision:" + Var("clang_revision"), + }, + ], + "condition": "host_os == 'mac' and host_cpu == 'arm64'", + "dep_type": "cipd", + }, Var("dart_root") + "/third_party/webdriver/chrome": { "packages": [ diff --git a/build/config/clang/clang.gni b/build/config/clang/clang.gni index c2ba3f7ac31..49cc5097ac0 100644 --- a/build/config/clang/clang.gni +++ b/build/config/clang/clang.gni @@ -2,9 +2,11 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("../../toolchain/goma.gni") + _toolchain_cpu = host_cpu -if (host_os == "mac") { - # TODO(https://fxbug.dev/73385): Use arm64 toolchain on arm64 when it exists. +if (host_os == "mac" && use_goma) { + # Goma does not support ARM64. _toolchain_cpu = "x64" } diff --git a/build/config/compiler/BUILD.gn b/build/config/compiler/BUILD.gn index c2b1643d77d..4e2a79debf4 100644 --- a/build/config/compiler/BUILD.gn +++ b/build/config/compiler/BUILD.gn @@ -377,7 +377,10 @@ config("compiler") { } else if (is_linux) { toolchain_stamp_file = "//buildtools/linux-x64/clang/.versions/clang.cipd_version" - } else { + } else if (is_mac && host_cpu == "arm64") { + toolchain_stamp_file = + "//buildtools/mac-arm64/clang/.versions/clang.cipd_version" + } else if (is_mac) { toolchain_stamp_file = "//buildtools/mac-x64/clang/.versions/clang.cipd_version" } @@ -548,6 +551,7 @@ if (is_win) { "-Wno-microsoft-unqualified-friend", "-Wno-unknown-argument", # icu "-Wno-unused-value", # crashpad + "-Wno-deprecated-non-prototype", # zlib ] } else { default_warning_flags += [ @@ -579,6 +583,7 @@ if (is_win) { default_warning_flags += [ "-Wno-tautological-constant-compare", "-Wno-unused-but-set-variable", # icu + "-Wno-deprecated-non-prototype", # zlib ] } else { default_warning_flags += diff --git a/build/toolchain/mac/BUILD.gn b/build/toolchain/mac/BUILD.gn index 2b9c42608fd..1dd0cdac08e 100644 --- a/build/toolchain/mac/BUILD.gn +++ b/build/toolchain/mac/BUILD.gn @@ -20,6 +20,15 @@ if (use_goma) { goma_prefix = "" } +# Goma doesn't support the host-arm64 toolchain, so continue using Rosetta. +if (host_cpu == "arm64" && !use_goma) { + rebased_clang_dir = + rebase_path("//buildtools/mac-arm64/clang/bin", root_build_dir) +} else { + rebased_clang_dir = + rebase_path("//buildtools/mac-x64/clang/bin", root_build_dir) +} + # Shared toolchain definition. Invocations should set toolchain_os to set the # build args in this definition. template("mac_toolchain") { @@ -211,28 +220,10 @@ template("mac_toolchain") { } } -# Toolchain used for Mac host targets. mac_toolchain("clang_x64") { toolchain_cpu = "x64" toolchain_os = "mac" - prefix = rebase_path("//buildtools/mac-x64/clang/bin", root_build_dir) - cc = "${goma_prefix}$prefix/clang" - cxx = "${goma_prefix}$prefix/clang++" - ar = "${prefix}/llvm-ar" - ld = cxx - strip = "strip" - is_clang = true - if (mac_enable_relative_sdk_path) { - mac_sdk_path = rebase_path(mac_sdk_path, root_build_dir) - } - sysroot_flags = "-isysroot $mac_sdk_path -mmacosx-version-min=$mac_sdk_min" -} - -# Toolchain used for Mac host (i386) targets. -mac_toolchain("clang_x86") { - toolchain_cpu = "i386" - toolchain_os = "mac" - prefix = rebase_path("//buildtools/mac-x64/clang/bin", root_build_dir) + prefix = rebased_clang_dir cc = "${goma_prefix}$prefix/clang" cxx = "${goma_prefix}$prefix/clang++" ar = "${prefix}/llvm-ar" @@ -248,7 +239,7 @@ mac_toolchain("clang_x86") { mac_toolchain("clang_arm64") { toolchain_cpu = "arm64" toolchain_os = "mac" - prefix = rebase_path("//buildtools/mac-x64/clang/bin", root_build_dir) + prefix = rebased_clang_dir cc = "${goma_prefix}$prefix/clang" cxx = "${goma_prefix}$prefix/clang++" ar = "${prefix}/llvm-ar" diff --git a/pkg/dart2native/lib/dart2native_macho.dart b/pkg/dart2native/lib/dart2native_macho.dart index 701e3c3448c..fe240c352cf 100644 --- a/pkg/dart2native/lib/dart2native_macho.dart +++ b/pkg/dart2native/lib/dart2native_macho.dart @@ -6,7 +6,6 @@ import 'dart:io'; import 'dart:math'; import './macho.dart'; -import './macho_utils.dart'; // Simplifies casting so we get null values back instead of exceptions. T? cast(x) => x is T ? x : null; @@ -66,60 +65,69 @@ class _MacOSVersion { // compatible with MachO executables. Future writeAppendedMachOExecutable( String dartaotruntimePath, String payloadPath, String outputPath) async { - File originalExecutableFile = File(dartaotruntimePath); + final aotRuntimeFile = File(dartaotruntimePath); - MachOFile machOFile = MachOFile.fromFile(originalExecutableFile); - final oldLinkEditSegmentFileOffset = machOFile.linkEditSegment?.fileOffset; - if (oldLinkEditSegmentFileOffset == null) { + final aotRuntimeHeaders = MachOFile.fromFile(aotRuntimeFile); + final oldLinkEdit = aotRuntimeHeaders.linkEditSegment; + if (oldLinkEdit == null) { throw FormatException("__LINKEDIT segment not found"); } - // Insert the new segment that contains our snapshot data. - File newSegmentFile = File(payloadPath); - // Get the length of the contents of the section to be added. - final payloadLength = newSegmentFile.lengthSync(); - // We only need the original offset of the __LINKEDIT segment from the - // original headers, which we've already retrieved. Thus, use the new - // MachOFile from inserting the snapshot segment from here on. - machOFile = machOFile.insertSegmentLoadCommand( - payloadLength, snapshotSegmentName, snapshotSectionName); + final snapshotFile = File(payloadPath); + final payloadLength = snapshotFile.lengthSync(); - // Write out the new executable, with the same contents except the new header. - File outputFile = File(outputPath); - RandomAccessFile stream = await outputFile.open(mode: FileMode.write); + // Add the header information for where the snapshot will live, and retrieve + // the needed parts back out of the new headers. + final outputHeaders = + aotRuntimeHeaders.adjustHeaderForSnapshot(payloadLength); + final snapshotNote = outputHeaders.snapshotNote!; + final newLinkEdit = outputHeaders.linkEditSegment!; - // Write the MachO header. - machOFile.writeSync(stream); - final int headerBytesWritten = stream.positionSync(); + final output = await File(outputPath).open(mode: FileMode.write); - RandomAccessFile newSegmentFileStream = await newSegmentFile.open(); - RandomAccessFile originalFileStream = await originalExecutableFile.open(); - await originalFileStream.setPosition(headerBytesWritten); + void addPadding(int start, int end) { + assert(end >= start); + output.writeFromSync(List.filled(end - start, 0)); + } - // Write the unchanged data from the original file up to the __LINKEDIT - // segment contents, so we can insert the snapshot there. - await pipeStream(originalFileStream, stream, - numToWrite: oldLinkEditSegmentFileOffset - headerBytesWritten); + // First, write the new headers. + outputHeaders.writeSync(output); + // If the newer headers are smaller, add appropriate padding to fit. + // + // TODO(49783): Once linker flags are in place in g3, this check should always + // succeed and should be removed. + if (outputHeaders.size <= aotRuntimeHeaders.size) { + addPadding(outputHeaders.size, aotRuntimeHeaders.size); + } + // TODO(49783): Once linker flags are in place in g3, this should always be + // aotRuntimeHeaders.size, but for now allow for the possibility of + // overwriting part of the original contents with the header as before. + final originalStart = max(aotRuntimeHeaders.size, outputHeaders.size); - void addPadding(int size) => stream.writeFromSync(List.filled(size, 0)); + // Now write the original contents from the header to the __LINKEDIT segment + // contents. + final aotRuntimeStream = await aotRuntimeFile.open(); + await aotRuntimeStream.setPosition(originalStart); + await pipeStream(aotRuntimeStream, output, + numToWrite: oldLinkEdit.fileOffset - originalStart); - final snapshotSegment = machOFile.snapshotSegment!; + // Now insert the snapshot contents at this position in the file. // There may be additional padding needed between the old __LINKEDIT file // offset and the start of the new snapshot. - addPadding(snapshotSegment.fileOffset - oldLinkEditSegmentFileOffset); - - // Write the inserted section data, ensuring that the data is padded to the - // segment size. - await pipeStream(newSegmentFileStream, stream); - addPadding(align(payloadLength, segmentAlignment) - payloadLength); + addPadding(oldLinkEdit.fileOffset, snapshotNote.fileOffset); + final snapshotStream = await snapshotFile.open(); + await pipeStream(snapshotStream, output); + // Now add appropriate padding after the snapshot to reach the expected offset + // of the __LINKEDIT segment in the new file. + final snapshotEnd = snapshotNote.fileOffset + snapshotNote.fileSize; + addPadding(snapshotEnd, newLinkEdit.fileOffset); // Copy the rest of the file from the original to the new one. - await pipeStream(originalFileStream, stream); + await pipeStream(aotRuntimeStream, output); + await output.close(); - await stream.close(); - - if (machOFile.hasCodeSignature) { + if (outputHeaders.hasCodeSignature) { if (!Platform.isMacOS) { throw 'Cannot sign MachO binary on non-macOS platform'; } diff --git a/pkg/dart2native/lib/macho.dart b/pkg/dart2native/lib/macho.dart index 3919d92d808..f8dac8323fb 100644 --- a/pkg/dart2native/lib/macho.dart +++ b/pkg/dart2native/lib/macho.dart @@ -9,7 +9,6 @@ // ignore_for_file: non_constant_identifier_names, constant_identifier_names import 'dart:io'; -import 'dart:math'; import 'dart:typed_data'; import './macho_utils.dart'; @@ -291,13 +290,9 @@ abstract class MachOLoadCommand { /// command. bool get mustBeUnderstood => (code & LoadCommandType._reqDyld) != 0; - // Overwrite in subclasses that are used in the minimumFileOffset calculation. - int? get minimumFileOffset => null; - - // Returns a version of the load command with any file offsets appropriately - // adjusted as needed. Should be overloaded by any load commands that contain - // file offsets. - MachOLoadCommand adjust(OffsetsAdjuster adjuster) => this; + /// Returns a version of the load command with any file offsets appropriately + /// adjusted as needed. + MachOLoadCommand adjust(OffsetsAdjuster adjuster); void writeSync(MachOWriter stream) { stream @@ -306,9 +301,9 @@ abstract class MachOLoadCommand { writeContentsSync(stream); } - // Subclasses need to implement this serializer, which should NOT - // attempt to serialize the cmd and the cmdsize to the stream. That - // logic is handled by the parent class automatically. + /// Subclasses need to implement this serializer, which should NOT + /// attempt to serialize the cmd and the cmdsize to the stream. That + /// logic is handled by the parent class automatically. void writeContentsSync(MachOWriter stream); } @@ -318,6 +313,9 @@ class MachOGenericLoadCommand extends MachOLoadCommand { MachOGenericLoadCommand(cmd, cmdsize, this.contents) : super(cmd, cmdsize); + @override + MachOLoadCommand adjust(OffsetsAdjuster adjuster) => this; + @override void writeContentsSync(MachOWriter stream) { stream.writeBytes(contents); @@ -429,6 +427,10 @@ class MachOHeader { bool get is64Bit => reserved != null; Endian get endian => magicEndian(magic); + // The size of the header when written to disk. Seven 32-bit fields, plus + // an extra 32-bit field for 64-bit headers to align it to word size. + int get size => 7 * 4 + (is64Bit ? 4 : 0); + void writeSync(RandomAccessFile original) { // Like reading, first we write the magic value using host endianness, // and then write the rest of the value using the detected endianness. @@ -561,22 +563,18 @@ class MachOSegmentCommand extends MachOLoadCommand { LoadCommandType.fromCode(code) == LoadCommandType.segment64 ? 8 : 4; @override - int? get minimumFileOffset { - if (sections.isEmpty) { - // Don't use the file offset of this segment if the segment is empty. - return fileSize != 0 ? fileOffset : null; - } - int? minOffset; - for (final section in sections) { - // Skip zero fill sections. - if (section.isZeroFill) continue; - assert(section.fileOffset != 0 && section.size != 0); - minOffset = minOffset == null - ? section.fileOffset - : min(minOffset, section.fileOffset); - } - return minOffset; - } + MachOSegmentCommand adjust(OffsetsAdjuster adjuster) => MachOSegmentCommand( + code, + size, + name, + memoryAddress, + memorySize, + adjuster.adjust(fileOffset), + fileSize, + maxProtection, + initialProtection, + flags, + sections.map((s) => s.adjust(adjuster)).toList()); @override void writeContentsSync(MachOWriter stream) { @@ -757,6 +755,20 @@ class MachOSection { cachedType == SectionType.threadLocalZeroFill; } + MachOSection adjust(OffsetsAdjuster adjuster) => MachOSection( + name, + segmentName, + memoryAddress, + size, + adjuster.adjust(fileOffset), + alignment, + adjuster.adjust(relocationsFileOffset), + relocationsCount, + flags, + reserved1, + reserved2, + reserved3); + void writeContentsSync(MachOWriter stream) { final int wordSize = is64Bit ? 8 : 4; stream.writeFixedLengthNullTerminatedString(name, _nameLength); @@ -1373,8 +1385,6 @@ class MachONoteCommand extends MachOLoadCommand { int code, int size, this.dataOwner, this.fileOffset, this.fileSize) : super(code, size); - static const _dataOwnerLength = 16; - static MachONoteCommand fromStream(int code, int size, MachOReader stream) { final dataOwner = stream.readFixedLengthNullTerminatedString(_dataOwnerLength); @@ -1383,6 +1393,21 @@ class MachONoteCommand extends MachOLoadCommand { return MachONoteCommand(code, size, dataOwner, fileOffset, fileSize); } + /// Constructs a note load command given the content of the note-specific + /// fields, using the default values for the code and size fields. + static MachONoteCommand fromFields( + String dataOwner, int fileOffset, int fileSize) => + MachONoteCommand(LoadCommandType.note.code, _defaultSize, dataOwner, + fileOffset, fileSize); + + /// The maximum size of the dataOwner field contents. + static const _dataOwnerLength = 16; + + // The total size of any given note load command. A note load command contains + // the 4-byte cmd and cmdsize fields that start every load command, a 16 byte + // name field, and then two additional 8-byte fields. + static const _defaultSize = 4 + 4 + 16 + 8 + 8; + @override MachONoteCommand adjust(OffsetsAdjuster adjuster) => MachONoteCommand( code, size, dataOwner, adjuster.adjust(fileOffset), fileSize); @@ -1456,15 +1481,10 @@ class MachOFile { /// non-header contents of the MachO file. final List commands; - /// A cached version of the minimum file offset of any segments or sections, - /// which we use to determine post-header padding. - final int minimumFileOffset; - /// Whether or not the parsed MachO file has a code signature. final bool hasCodeSignature; - MachOFile._(this.header, this.commands, this.minimumFileOffset, - this.hasCodeSignature); + MachOFile._(this.header, this.commands, this.hasCodeSignature); static MachOFile fromFile(File file) { // Ensure the file is long enough to contain the magic bytes. @@ -1487,89 +1507,43 @@ class MachOFile { final commands = List.generate( header.loadCommandsCount, (_) => MachOLoadCommand.fromStream(reader)); - // Set the max header offset to the maximum file size so that when we read - // in the header we can correctly set the total header size. - final minimumFileOffset = commands.fold((1 << 63) - 1, (i, c) { - final minFileOffset = c.minimumFileOffset; - return minFileOffset != null ? min(i, minFileOffset) : i; - }); + final size = _totalSize(header, commands); + assert(size == stream.positionSync()); + final hasCodeSignature = commands.any((c) => c.type == LoadCommandType.codeSignature); - return MachOFile._(header, commands, minimumFileOffset, hasCodeSignature); + return MachOFile._(header, commands, hasCodeSignature); } - /// Returns a new MachOFile that is like the input, but with a new - /// segment load command with a single section inserted prior to the - /// __LINKEDIT segment load command. Any file offsets in other load commands - /// that reference the __LINKEDIT segment are adjusted appropriately. - MachOFile insertSegmentLoadCommand( - int segmentLength, String segmentName, String sectionName) { + /// Returns a new MachOFile that is like the input, but with the empty segment + /// used to reserve header space dropped and with a new note load command + /// inserted prior to the __LINKEDIT segment load command. Any file offsets + /// in other load commands that reference the __LINKEDIT segment are adjusted + /// appropriately. + MachOFile adjustHeaderForSnapshot(int snapshotSize) { + // This is not an idempotent operation. + if (snapshotNote != null) { + throw FormatException( + "The executable already has a Dart snapshot inserted"); + } + + final reserved = reservedSegment; + // TODO(49783): Once linker flags are in place in g3, we should throw a + // FormatException if the segment used to reserve header space is not found. + final linkedit = linkEditSegment; if (linkedit == null) { throw FormatException("__LINKEDIT segment not found"); } - // Create the new segment. + // We insert the contents of the snapshot where the old linkedit segment + // started in the original executable, aligned appropriately. final int fileOffset = align(linkedit.fileOffset, segmentAlignment); - final int fileSize = align(segmentLength, segmentAlignment); - final int memoryAddress = linkedit.memoryAddress; - final int memorySize = fileSize; - final int maxProtection = VirtualMemoryProtection.read.code; - final int initialProtection = maxProtection; + final int fileSize = snapshotSize; - final int sectionFlags = - MachOSection.combineIntoFlags(SectionType.regular, 0); - - final loadCommandDefinitionSize = 4 * 2; - final sectionDefinitionSize = 16 * 2 + 8 * 2 + 4 * 8; - final segmentDefinitionSize = 16 + 8 * 4 + 4 * 4; - final commandSize = loadCommandDefinitionSize + - segmentDefinitionSize + - sectionDefinitionSize; - - final section = MachOSection( - sectionName, - segmentName, - memoryAddress, - fileSize, - fileOffset, - segmentAlignmentLog2, - 0, - 0, - sectionFlags, - 0, - 0, - 0); - - final segment = MachOSegmentCommand( - LoadCommandType.segment64.code, - commandSize, - segmentName, - memoryAddress, - memorySize, - fileOffset, - fileSize, - maxProtection, - initialProtection, - 0, - [section]); - - // Setup the new linkedit command. - final shiftedLinkeditMemoryAddress = memoryAddress + segment.memorySize; - final shiftedLinkeditFileOffset = fileOffset + segment.fileSize; - final shiftedLinkedit = MachOSegmentCommand( - linkedit.code, - linkedit.size, - linkedit.name, - shiftedLinkeditMemoryAddress, - linkedit.memorySize, - shiftedLinkeditFileOffset, - linkedit.fileSize, - linkedit.maxProtection, - linkedit.initialProtection, - linkedit.flags, - linkedit.sections); + final note = + MachONoteCommand.fromFields(snapshotNoteName, fileOffset, fileSize); // Now we need to build the new header from these modified pieces. final newHeader = MachOHeader( @@ -1577,36 +1551,65 @@ class MachOFile { header.cpu, header.machine, header.type, - header.loadCommandsCount + 1, - header.loadCommandsSize + segment.size, + // If the reserved section exists, we remove it and replace it with + // the note. + // + // TODO(49783): Once linker flags are in place in g3, reserved should + // never be null. + header.loadCommandsCount + (reserved == null ? 1 : 0), + header.loadCommandsSize - (reserved?.size ?? 0) + note.size, header.flags, header.reserved); - final adjuster = OffsetsAdjuster(); - adjuster.add( - linkedit.fileOffset, shiftedLinkeditFileOffset - linkedit.fileOffset); + // We'll want the __LINKEDIT segment to start at the next aligned file + // offset after the end of the snapshot, so we'll need to adjust all + // file offsets pointing into it (including its own segment) accordingly. + final snapshotEnd = + align(note.fileOffset + note.fileSize, segmentAlignment); + final adjuster = OffsetsAdjuster() + ..add(linkedit.fileOffset, snapshotEnd - linkedit.fileOffset); final newCommands = []; for (final command in commands) { - if (command == linkedit) { - // Insert the new segment prior to the __LINKEDIT segment. - newCommands.add(segment); - // Replace the old __LINKEDIT segment with the new one. - newCommands.add(shiftedLinkedit); - } else { - // Shift the offsets of any other load command that fall within the - // __LINKEDIT segment appropriately. - newCommands.add(command.adjust(adjuster)); + if (command == reserved) { + // Drop the reserved segment on the floor, as we only add it so there's + // enough header space to add the note. + continue; } + if (command == linkedit) { + // Insert the new note prior to the __LINKEDIT segment. + newCommands.add(note); + } + newCommands.add(command.adjust(adjuster)); } - return MachOFile._( - newHeader, newCommands, minimumFileOffset, hasCodeSignature); + final newFile = MachOFile._(newHeader, newCommands, hasCodeSignature); + + // TODO(49783): Once linker flags are in place in g3, we should throw a + // FormatException if [newFile.size] is greater than [size]. + + return newFile; } /// The name of the segment containing all the structs created and maintained /// by the link editor. static const _linkEditSegmentName = "__LINKEDIT"; + /// Retrieves the segment load command used to reserve header space for the + /// snapshot information. Returns null if not found or if it is of an + /// unexpected form. + MachOSegmentCommand? get reservedSegment { + final reservedIndex = commands.indexWhere( + (c) => c is MachOSegmentCommand && c.name == reservedSegmentName); + if (reservedIndex < 0) { + return null; + } + final reserved = commands[reservedIndex] as MachOSegmentCommand; + assert(reserved.fileSize == 0); + assert(reserved.sections.single.name == reservedSectionName); + return reserved; + } + + /// Retrieves the __LINKEDIT segment load command. Returns null if not found. MachOSegmentCommand? get linkEditSegment { final linkEditIndex = commands.indexWhere( (c) => c is MachOSegmentCommand && c.name == _linkEditSegmentName); @@ -1621,16 +1624,25 @@ class MachOFile { return commands[linkEditIndex] as MachOSegmentCommand; } - MachOSegmentCommand? get snapshotSegment { + /// Retrieves the note load command that points to the snapshot contents in + /// the executable. Returns null if not found. + MachONoteCommand? get snapshotNote { final snapshotIndex = commands.indexWhere( - (c) => c is MachOSegmentCommand && c.name == snapshotSegmentName); + (c) => c is MachONoteCommand && c.dataOwner == snapshotNoteName); if (snapshotIndex < 0) { return null; } - - return commands[snapshotIndex] as MachOSegmentCommand; + return commands[snapshotIndex] as MachONoteCommand; } + static bool containsSnapshot(File file) => + MachOFile.fromFile(file).snapshotNote != null; + + static int _totalSize(MachOHeader header, List commands) => + commands.fold(header.size, (i, c) => i + c.size); + + int get size => _totalSize(header, commands); + /// Writes the MachO file to the given [RandomAccessFile] stream. void writeSync(RandomAccessFile stream) { header.writeSync(stream); @@ -1641,11 +1653,5 @@ class MachOFile { for (var command in commands) { command.writeSync(writer); } - - // Pad the header according to the offset. - final int paddingAmount = minimumFileOffset - stream.positionSync(); - if (paddingAmount > 0) { - stream.writeFromSync(List.filled(paddingAmount, 0)); - } } } diff --git a/pkg/dart2native/lib/macho_utils.dart b/pkg/dart2native/lib/macho_utils.dart index b9ff8360b6c..e975ecb93ff 100644 --- a/pkg/dart2native/lib/macho_utils.dart +++ b/pkg/dart2native/lib/macho_utils.dart @@ -6,11 +6,14 @@ import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; -// Note that these two values MUST match the ones in -// runtime/bin/snapshot_utils.cc, which looks specifically for the snapshot in -// this segment and section. -const String snapshotSegmentName = "__CUSTOM"; -const String snapshotSectionName = "__dart_app_snap"; +// Note that these values MUST match the arguments to -add_empty_section in +// runtime/BUILD.gn. +const String reservedSegmentName = "__CUSTOM"; +const String reservedSectionName = "__space_for_note"; + +// Note that this value MUST match runtime/bin/snapshot_utils.cc, which looks +// specifically for the snapshot in this note. +const String snapshotNoteName = "__dart_app_snap"; /// The page size for aligning segments in MachO files. X64 MacOS uses 4k pages, /// and ARM64 MacOS uses 16k pages, so we use 16k here. diff --git a/pkg/dartdev/test/commands/compile_test.dart b/pkg/dartdev/test/commands/compile_test.dart index e42927acf10..11deb0ec85e 100644 --- a/pkg/dartdev/test/commands/compile_test.dart +++ b/pkg/dartdev/test/commands/compile_test.dart @@ -3,7 +3,9 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:io'; +import 'dart:math'; +import 'package:dart2native/dart2native_macho.dart' show pipeStream; import 'package:dart2native/macho.dart'; import 'package:path/path.dart' as path; import 'package:test/test.dart'; @@ -46,15 +48,9 @@ void defineCompileTests() { expect(File(outFile).existsSync(), true, reason: 'File not found: $outFile'); - // Ensure the file contains the __CUSTOM segment. - final machOFile = MachOFile.fromFile(File(outFile)); - - // Throws an exception (and thus the test fails) if the segment doesn't - // exist. - machOFile.commands - .where((segment) => - segment is MachOSegmentCommand && segment.name == '__CUSTOM') - .first; + if (!MachOFile.containsSnapshot(File(outFile))) { + throw FormatException('Snapshot not found in standalone executable'); + } // Ensure that the exe can be signed. final codeSigningProcess = await Process.start('codesign', [ @@ -68,6 +64,88 @@ void defineCompileTests() { final signingResult = await codeSigningProcess.exitCode; expect(signingResult, 0); }, skip: isRunningOnIA32); + + test('Changing snapshot contents fails to validate', () async { + final p = project(mainSrc: '''void main() {}'''); + final inFile = + path.canonicalize(path.join(p.dirPath, p.relativeFilePath)); + final outFile = path.canonicalize(path.join(p.dirPath, 'myexe')); + final corruptedFile = + path.canonicalize(path.join(p.dirPath, 'myexe-corrupted')); + + var result = await p.run( + [ + 'compile', + 'exe', + '-o', + outFile, + inFile, + ], + ); + + expect(result.stdout, contains(soundNullSafetyMessage)); + expect(result.stderr, isEmpty); + expect(result.exitCode, 0); + expect(File(outFile).existsSync(), true, + reason: 'File not found: $outFile'); + + final macho = MachOFile.fromFile(File(outFile)); + final snapshotNote = macho.snapshotNote; + if (snapshotNote == null) { + throw FormatException('Snapshot not found in standalone executable'); + } + + if (macho.hasCodeSignature) { + // Verify the resulting executable using codesign. + result = Process.runSync('codesign', [ + '-v', + outFile, + ]); + + expect(result.stderr, isEmpty); + expect(result.exitCode, 0); + } else { + // Sign the executable first. + final codeSigningProcess = await Process.start('codesign', [ + '-o', + 'runtime', + '-s', + '-', + outFile, + ]); + + final signingResult = await codeSigningProcess.exitCode; + expect(signingResult, 0); + } + + // Pick a random range of bytes within the snapshot. + final rand = Random(); + final offset1 = rand.nextInt(snapshotNote.fileSize); + final offset2 = rand.nextInt(snapshotNote.fileSize); + final int start = snapshotNote.fileOffset + min(offset1, offset2); + final int size = max(offset1, offset2) - min(offset1, offset2); + + // Write the corrupted version of the executable, corrupting the bytes in + // the calculated range by incrementing them (modulo 256). + final original = File(outFile).openSync(); + final corrupted = File(corruptedFile).openSync(mode: FileMode.write); + await pipeStream(original, corrupted, numToWrite: start); + final bytesToCorrupt = original.readSync(size); + for (int i = 0; i < bytesToCorrupt.length; i++) { + bytesToCorrupt[i] = (bytesToCorrupt[i] + 1) % 256; + } + corrupted.writeFromSync(bytesToCorrupt); + await pipeStream(original, corrupted); + + // (Fail to) verify the resulting executable using codesign. + result = Process.runSync('codesign', [ + '-v', + corruptedFile, + ]); + + expect(result.stderr, isNotEmpty); + expect(result.exitCode, 1); + }, skip: isRunningOnIA32); } // *** NOTE ***: These tests *must* be run with the `--use-sdk` option diff --git a/runtime/BUILD.gn b/runtime/BUILD.gn index 3e6c317b746..d894483fe0f 100644 --- a/runtime/BUILD.gn +++ b/runtime/BUILD.gn @@ -59,6 +59,21 @@ config("dart_precompiled_runtime_config") { "DART_PRECOMPILED_RUNTIME", "EXCLUDE_CFE_AND_KERNEL_PLATFORM", ] + if (is_mac) { + # We create an empty __space_for_note section in a __CUSTOM segment to + # reserve the header space needed for inserting a snapshot into the + # executable when creating standalone executables. This segment and section + # is removed in standalone executables, replaced with a note that points to + # the snapshot in the file. + # + # (A segment load command with a single section is 132 bytes in 32-bit + # executables and 152 bytes in 64-bit ones, and a note load command is + # always 40 bytes.) + # + # Keep this in sync with the constants reservedSegmentName and + # reservedSectionName in pkg/dart2native/lib/macho_utils.dart. + ldflags = [ "-Wl,-add_empty_section,__CUSTOM,__space_for_note" ] + } } # Controls DART_PRECOMPILER #define. diff --git a/runtime/bin/snapshot_utils.cc b/runtime/bin/snapshot_utils.cc index fd0d4dc46c4..dd418817ac0 100644 --- a/runtime/bin/snapshot_utils.cc +++ b/runtime/bin/snapshot_utils.cc @@ -29,9 +29,7 @@ namespace bin { static const int64_t kAppSnapshotHeaderSize = 5 * kInt64Size; static const int64_t kAppSnapshotPageSize = 16 * KB; -static const char kMachOAppSnapshotSegmentName[] DART_UNUSED = "__CUSTOM"; -static const char kMachOAppSnapshotSectionName[] DART_UNUSED = - "__dart_app_snap"; +static const char kMachOAppSnapshotNoteName[] DART_UNUSED = "__dart_app_snap"; class MappedAppSnapshot : public AppSnapshot { public: @@ -246,104 +244,58 @@ static AppSnapshot* TryReadAppSnapshotElf( #if defined(DART_TARGET_OS_MACOS) AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElfFromMachO( const char* container_path) { + // Ensure file is actually MachO-formatted. + if (!IsMachOFormattedBinary(container_path)) { + Syslog::PrintErr("Expected a Mach-O binary.\n"); + return nullptr; + } + File* file = File::Open(NULL, container_path, File::kRead); if (file == nullptr) { return nullptr; } RefCntReleaseScope rs(file); - // Ensure file is actually MachO-formatted. - if (!IsMachOFormattedBinary(container_path)) { + // Read in the Mach-O header. Note that the 64-bit header is the same layout + // as the 32-bit header, just with an extra field for alignment, so we can + // safely load a 32-bit header to get all the information we need. + mach_o::mach_header header; + file->ReadFully(&header, sizeof(header)); + + if (header.magic == mach_o::MH_CIGAM || header.magic == mach_o::MH_CIGAM_64) { Syslog::PrintErr( - "Attempted load target was not formatted as expected: " - "expected Mach-O binary.\n"); + "Expected a host endian header but found a byte-swapped header.\n"); return nullptr; } - // Parse the first 4bytes and extract the magic number. - uint32_t magic; - file->SetPosition(0); - file->ReadFully(&magic, sizeof(uint32_t)); - - const bool is64Bit = - magic == mach_o::MH_MAGIC_64 || magic == mach_o::MH_CIGAM_64; - const bool isByteSwapped = - magic == mach_o::MH_CIGAM || magic == mach_o::MH_CIGAM_64; - - if (isByteSwapped) { - Syslog::PrintErr( - "Dart snapshot contained an unexpected binary file layout. " - "Expected non-byte swapped header but found a byte-swapped header.\n"); - return nullptr; + if (header.magic == mach_o::MH_MAGIC_64) { + // Set the file position as if we had read a 64-bit header. + file->SetPosition(sizeof(mach_o::mach_header_64)); } - file->SetPosition(0); + // Now we search through the load commands to find our snapshot note, which + // has a data_owner field of kMachOAppSnapshotNoteName. + for (uint32_t i = 0; i < header.ncmds; ++i) { + mach_o::load_command command; + file->ReadFully(&command, sizeof(mach_o::load_command)); - // Read in the Mach-O header, which will contain information about all of the - // segments in the binary. - // - // From the header we determine where our special segment is located. This - // segment must be named according to the convention captured by - // kMachOAppSnapshotSegmentType and kMachOAppSnapshotSegmentName. - if (!is64Bit) { - Syslog::PrintErr( - "Dart snapshot compiled with 32bit architecture. " - "Currently only 64bit architectures are supported.\n"); - return nullptr; - } else { - mach_o::mach_header_64 header; - file->ReadFully(&header, sizeof(header)); - - for (uint32_t i = 0; i < header.ncmds; ++i) { - mach_o::load_command command; - file->ReadFully(&command, sizeof(mach_o::load_command)); - - file->SetPosition(file->Position() - sizeof(command)); - if (command.cmd != mach_o::LC_SEGMENT && - command.cmd != mach_o::LC_SEGMENT_64) { - file->SetPosition(file->Position() + command.cmdsize); - continue; - } - - mach_o::segment_command_64 segment; - file->ReadFully(&segment, sizeof(segment)); - - for (uint32_t j = 0; j < segment.nsects; ++j) { - mach_o::section_64 section; - file->ReadFully(§ion, sizeof(section)); - - if (segment.cmd == mach_o::LC_SEGMENT_64 && - strcmp(section.segname, kMachOAppSnapshotSegmentName) == 0 && - strcmp(section.sectname, kMachOAppSnapshotSectionName) == 0) { - // We have to do the loading "by-hand" because we need to set the - // snapshot length to a specific length instead of the "rest of the - // file", which is the assumption that TryReadAppSnapshotElf makes. - const char* error = nullptr; - const uint8_t* vm_data_buffer = nullptr; - const uint8_t* vm_instructions_buffer = nullptr; - const uint8_t* isolate_data_buffer = nullptr; - const uint8_t* isolate_instructions_buffer = nullptr; - - std::unique_ptr snapshot(new uint8_t[section.size]); - file->SetPosition(section.offset); - file->ReadFully(snapshot.get(), sizeof(uint8_t) * section.size); - - Dart_LoadedElf* handle = Dart_LoadELF_Memory( - snapshot.get(), section.size, &error, &vm_data_buffer, - &vm_instructions_buffer, &isolate_data_buffer, - &isolate_instructions_buffer); - - if (handle == nullptr) { - Syslog::PrintErr("Loading failed: %s\n", error); - return nullptr; - } - - return new ElfAppSnapshot(handle, vm_data_buffer, - vm_instructions_buffer, isolate_data_buffer, - isolate_instructions_buffer); - } - } + file->SetPosition(file->Position() - sizeof(command)); + if (command.cmd != mach_o::LC_NOTE) { + file->SetPosition(file->Position() + command.cmdsize); + continue; } + + mach_o::note_command note; + file->ReadFully(¬e, sizeof(note)); + + if (strcmp(note.data_owner, kMachOAppSnapshotNoteName) != 0) { + file->SetPosition(file->Position() + command.cmdsize); + continue; + } + + // A note with the correct name was found, so we assume that the + // file contents for that note contains an ELF snapshot. + return TryReadAppSnapshotElf(container_path, note.offset); } return nullptr; @@ -549,18 +501,29 @@ bool Snapshot::IsMachOFormattedBinary(const char* filename) { } RefCntReleaseScope rs(file); - // Ensure the file is long enough to even contain the magic bytes. - if (file->Length() < 4) { + const uint64_t size = file->Length(); + // Parse the first 4 bytes and check the magic numbers. + uint32_t magic; + if (size < sizeof(magic)) { + // The file isn't long enough to contain the magic bytes. return false; } - - // Parse the first 4bytes and check the magic numbers. - uint32_t magic; file->SetPosition(0); - file->Read(&magic, sizeof(uint32_t)); + file->ReadFully(&magic, sizeof(magic)); - return magic == mach_o::MH_MAGIC_64 || magic == mach_o::MH_CIGAM_64 || - magic == mach_o::MH_MAGIC || magic == mach_o::MH_CIGAM; + // Depending on the magic numbers, check that the size of the file is + // large enough for either a 32-bit or 64-bit header. + switch (magic) { + case mach_o::MH_MAGIC: + case mach_o::MH_CIGAM: + return size >= sizeof(mach_o::mach_header); + case mach_o::MH_MAGIC_64: + case mach_o::MH_CIGAM_64: + return size >= sizeof(mach_o::mach_header_64); + default: + // Not a Mach-O formatted file. + return false; + } } #endif // defined(DART_TARGET_OS_MACOS) diff --git a/runtime/platform/mach_o.h b/runtime/platform/mach_o.h index d5bf913fe60..377331cf80e 100644 --- a/runtime/platform/mach_o.h +++ b/runtime/platform/mach_o.h @@ -49,50 +49,13 @@ struct load_command { uint32_t cmdsize; }; -static const uint32_t LC_SEGMENT = 0x1; -static const uint32_t LC_SEGMENT_64 = 0x19; - -struct section { - char sectname[16]; - char segname[16]; - uint32_t addr; - uint32_t size; - uint32_t offset; - uint32_t align; - uint32_t reloff; - uint32_t nreloc; - uint32_t flags; - uint32_t reserved1; - uint32_t reserved2; -}; - -struct section_64 { - char sectname[16]; - char segname[16]; - uint64_t addr; - uint64_t size; - uint32_t offset; - uint32_t align; - uint32_t reloff; - uint32_t nreloc; - uint32_t flags; - uint32_t reserved1; - uint32_t reserved2; - uint32_t reserved3; -}; - -struct segment_command_64 { +static const uint32_t LC_NOTE = 0x31; +struct note_command { uint32_t cmd; uint32_t cmdsize; - char segname[16]; - uint64_t vmaddr; - uint64_t vmsize; - uint64_t fileoff; - uint64_t filesize; - vm_prot_t maxprot; - vm_prot_t initprot; - uint32_t nsects; - uint32_t flags; + char data_owner[16]; + uint64_t offset; + uint64_t size; }; #pragma pack(pop) diff --git a/runtime/vm/malloc_hooks_ia32.cc b/runtime/vm/malloc_hooks_ia32.cc index 2b7a3710a79..0572ac03400 100644 --- a/runtime/vm/malloc_hooks_ia32.cc +++ b/runtime/vm/malloc_hooks_ia32.cc @@ -11,9 +11,9 @@ namespace dart { #if defined(DEBUG) -const intptr_t kSkipCount = 6; +const intptr_t kSkipCount = 7; #elif !(defined(PRODUCT) || defined(DEBUG)) -const intptr_t kSkipCount = 5; +const intptr_t kSkipCount = 6; #endif } // namespace dart