MacOS-Signable Compiled Dart Scripts

This CL implements a slightly altered compilation process for `dart
compile` on MacOS. Instead of the traditional approach of
concatenating a dart snapshot with the dart runtime, this new
implementation uses a new MachO segment to store the snapshot. Doing
so ensure users can properly sign the resulting binaries.

The dart runtime has also been updated to look for the snapshots in
this new segment.

There are new unit tests ensuring the dart runtime can correctly
identify MachO-formatted executables and an end-to-end test ensuring
`dart compile` produces code that has the new segment and produces
code that can be signed.

TEST=pkg/dartdev/test/commands/compile_test

Change-Id: Ibdce80455e02bc2b6fd345d80be8d75f989a9c28
Cq-Include-Trybots: luci.dart.try:analyzer-mac-release-try,dart-sdk-mac-arm64-try,dart-sdk-mac-try,pkg-mac-release-arm64-try,pkg-mac-release-try,vm-kernel-mac-debug-x64-try,vm-kernel-mac-product-x64-try,vm-kernel-mac-release-arm64-try,vm-kernel-mac-release-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/228080
Reviewed-by: Martin Kustermann <kustermann@google.com>
Reviewed-by: Tess Strickland <sstrickl@google.com>
Commit-Queue: Tess Strickland <sstrickl@google.com>
This commit is contained in:
Tess Strickland 2022-03-09 18:02:34 +00:00 committed by Commit Bot
parent 35c5713d8c
commit 6ef426d7df
18 changed files with 3549 additions and 24 deletions

View file

@ -5,6 +5,9 @@
import 'dart:io';
import 'dart:typed_data';
import 'package:dart2native/dart2native_macho.dart'
show writeAppendedMachOExecutable;
// Maximum page size across all supported architectures (arm64 macOS has 16K
// pages, some arm64 Linux distributions have 64K pages).
const elfPageSize = 65536;
@ -14,6 +17,11 @@ enum Kind { aot, exe }
Future writeAppendedExecutable(
String dartaotruntimePath, String payloadPath, String outputPath) async {
if (Platform.isMacOS) {
return await writeAppendedMachOExecutable(
dartaotruntimePath, payloadPath, outputPath);
}
final dartaotruntime = File(dartaotruntimePath);
final int dartaotruntimeLength = dartaotruntime.lengthSync();

View file

@ -0,0 +1,334 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';
import 'package:dart2native/macho.dart';
import 'package:dart2native/macho_parser.dart';
const String kSnapshotSegmentName = "__CUSTOM";
const String kSnapshotSectionName = "__dart_app_snap";
const int kMinimumSegmentSize = 0x4000;
// Since arm64 macOS has 16K pages, which is larger than the 4K pages on x64
// macOS, we use this larger page size to ensure the MachO file is aligned
// properly on all architectures.
const int kSegmentAlignment = 0x4000;
int align(int size, int base) {
final int over = size % base;
if (over != 0) {
return size + (base - over);
}
return size;
}
// Utility for aligning parts of MachO headers to the defined sizes.
int vmSizeAlign(int size) {
return align(max(size, kMinimumSegmentSize), kSegmentAlignment);
}
// Returns value + amount only if the original value is within the bounds
// defined by [withinStart, withinStart + withinSize).
Uint32 addIfWithin(
Uint32 value, Uint64 amount, Uint64 withinStart, Uint64 withinSize) {
final intWithinStart = withinStart.asInt();
final intWithinSize = withinSize.asInt();
if (value >= intWithinStart && value < (intWithinStart + intWithinSize)) {
return (value.asUint64() + amount).asUint32();
} else {
return value;
}
}
// Trims a bytestring that an arbitrary number of null characters on the end of
// it.
String trimmedBytestring(Uint8List bytestring) {
return String.fromCharCodes(bytestring.takeWhile((value) => value != 0));
}
// Simplifies casting so we get null values back instead of exceptions.
T? cast<T>(x) => x is T ? x : null;
// Inserts a segment definition into a MachOFile. This does NOT insert the
// actual segment into the file. It only inserts the definition of that segment
// into the MachO header.
//
// In addition to simply specifying the definition for the segment, this
// function also moves the existing __LINKEDIT segment to the end of the header
// definition as is required by the MachO specification (or at least MacOS's
// implementation of it). In doing so there are several offsets in the original
// __LINKEDIT segment that must be updated to point to their new location
// because the __LINKEDIT segment and sections are now in a different
// place. This function takes care of those shifts as well.
//
// Returns the original, unmodified, __LINKEDIT segment.
Future<MachOSegmentCommand64> insertSegmentDefinition(MachOFile file,
File segment, String segmentName, String sectionName) async {
// Load in the data to be inserted.
final segmentData = await segment.readAsBytes();
// Find the existing __LINKEDIT segment
final linkedit = cast<MachOSegmentCommand64>(file.commands
.where((segment) =>
segment.asType() is MachOSegmentCommand64 &&
MachOConstants.SEG_LINKEDIT ==
trimmedBytestring((segment as MachOSegmentCommand64).segname))
.first);
final linkeditIndex = file.commands.indexWhere((segment) =>
segment.asType() is MachOSegmentCommand64 &&
MachOConstants.SEG_LINKEDIT ==
trimmedBytestring((segment as MachOSegmentCommand64).segname));
if (linkedit == null) {
throw FormatException(
"Could not find a __LINKEDIT section in the specified binary.");
} else {
// Create the new segment.
final Uint8List segname = Uint8List(16);
segname.setRange(0, segmentName.length, ascii.encode(segmentName));
segname.fillRange(segmentName.length, 16, 0);
final Uint64 vmaddr = linkedit.vmaddr;
final Uint64 vmsize = Uint64(vmSizeAlign(segmentData.length));
final Uint64 fileoff = linkedit.fileoff;
final Uint64 filesize = vmsize;
final Int32 maxprot = MachOConstants.VM_PROT_READ;
final Int32 initprot = maxprot;
final Uint32 nsects = Uint32(1);
final Uint8List sectname = Uint8List(16);
sectname.setRange(0, sectionName.length, ascii.encode(sectionName));
sectname.fillRange(sectionName.length, 16, 0);
final Uint64 addr = vmaddr;
final Uint64 size = Uint64(segmentData.length);
final Uint32 offset = fileoff.asUint32();
final Uint32 flags = MachOConstants.S_REGULAR;
final Uint32 zero = Uint32(0);
final loadCommandDefinitionSize = 4 * 2;
final sectionDefinitionSize = 16 * 2 + 8 * 2 + 4 * 8;
final segmentDefinitionSize = 16 + 8 * 4 + 4 * 4;
final commandSize = loadCommandDefinitionSize +
segmentDefinitionSize +
sectionDefinitionSize;
final loadCommand =
MachOLoadCommand(MachOConstants.LC_SEGMENT_64, Uint32(commandSize));
final section = MachOSection64(sectname, segname, addr, size, offset, zero,
zero, zero, flags, zero, zero, zero);
final segment = MachOSegmentCommand64(Uint32(commandSize), segname, vmaddr,
vmsize, fileoff, filesize, maxprot, initprot, nsects, zero, [section]);
// Setup the new linkedit command.
final shiftedLinkeditVmaddr = linkedit.vmaddr + segment.vmsize;
final shiftedLinkeditFileoff = linkedit.fileoff + segment.filesize;
final shiftedLinkedit = MachOSegmentCommand64(
linkedit.cmdsize,
linkedit.segname,
shiftedLinkeditVmaddr,
linkedit.vmsize,
shiftedLinkeditFileoff,
linkedit.filesize,
linkedit.maxprot,
linkedit.initprot,
linkedit.nsects,
linkedit.flags,
linkedit.sections);
// Shift all of the related commands that need to reference the new file
// position of the linkedit segment.
for (var i = 0; i < file.commands.length; i++) {
final command = file.commands[i];
final offsetAmount = segment.filesize;
final withinStart = linkedit.fileoff;
final withinSize = linkedit.filesize;
// For the specific command that we need to adjust, we need to move the
// commands' various offsets forward by the new segment's size in the file
// (segment.filesize). However, we need to ensure that when we move the
// offset forward, we exclude cases where the offset was originally
// outside of the linkedit segment (i.e. offset < linkedit.fileoff or
// offset >= linkedit.fileoff + linkedit.filesize). The DRY-ing function
// addIfWithin takes care of that repeated logic.
if (command is MachODyldInfoCommand) {
file.commands[i] = MachODyldInfoCommand(
command.cmd,
command.cmdsize,
addIfWithin(
command.rebase_off, offsetAmount, withinStart, withinSize),
command.rebase_size,
addIfWithin(
command.bind_off, offsetAmount, withinStart, withinSize),
command.bind_size,
addIfWithin(
command.weak_bind_off, offsetAmount, withinStart, withinSize),
command.weak_bind_size,
addIfWithin(
command.lazy_bind_off, offsetAmount, withinStart, withinSize),
command.lazy_bind_size,
addIfWithin(
command.export_off, offsetAmount, withinStart, withinSize),
command.export_size);
} else if (command is MachOSymtabCommand) {
file.commands[i] = MachOSymtabCommand(
command.cmdsize,
addIfWithin(command.symoff, offsetAmount, withinStart, withinSize),
command.nsyms,
addIfWithin(command.stroff, offsetAmount, withinStart, withinSize),
command.strsize);
} else if (command is MachODysymtabCommand) {
file.commands[i] = MachODysymtabCommand(
command.cmdsize,
command.ilocalsym,
command.nlocalsym,
command.iextdefsym,
command.nextdefsym,
command.iundefsym,
command.nundefsym,
addIfWithin(command.tocoff, offsetAmount, withinStart, withinSize),
command.ntoc,
addIfWithin(
command.modtaboff, offsetAmount, withinStart, withinSize),
command.nmodtab,
addIfWithin(
command.extrefsymoff, offsetAmount, withinStart, withinSize),
command.nextrefsyms,
addIfWithin(
command.indirectsymoff, offsetAmount, withinStart, withinSize),
command.nindirectsyms,
addIfWithin(
command.extreloff, offsetAmount, withinStart, withinSize),
command.nextrel,
addIfWithin(
command.locreloff, offsetAmount, withinStart, withinSize),
command.nlocrel);
} else if (command is MachOLinkeditDataCommand) {
file.commands[i] = MachOLinkeditDataCommand(
command.cmd,
command.cmdsize,
addIfWithin(command.dataoff, offsetAmount, withinStart, withinSize),
command.datasize);
}
}
// Now we need to build the new header from these modified pieces.
file.header = MachOHeader(
file.header!.magic,
file.header!.cputype,
file.header!.cpusubtype,
file.header!.filetype,
file.header!.ncmds + Uint32(1),
file.header!.sizeofcmds + loadCommand.cmdsize,
file.header!.flags,
file.header!.reserved);
file.commands[linkeditIndex] = shiftedLinkedit;
file.commands.insert(linkeditIndex, segment);
}
return linkedit;
}
// Pipe from one file stream into another. We do this in chunks to avoid
// excessive memory load.
Future<int> pipeStream(RandomAccessFile from, RandomAccessFile to,
{int? numToWrite, int chunkSize = 1 << 30}) async {
int numWritten = 0;
final int fileLength = from.lengthSync();
while (from.positionSync() != fileLength) {
final int availableBytes = fileLength - from.positionSync();
final int numToRead = numToWrite == null
? min(availableBytes, chunkSize)
: min(numToWrite - numWritten, min(availableBytes, chunkSize));
final buffer = await from.read(numToRead);
await to.writeFrom(buffer);
numWritten += numToRead;
if (numToWrite != null && numWritten >= numToWrite) {
break;
}
}
return numWritten;
}
// Writes an "appended" dart runtime + script snapshot file in a format
// compatible with MachO executables.
Future writeAppendedMachOExecutable(
String dartaotruntimePath, String payloadPath, String outputPath) async {
File originalExecutableFile = File(dartaotruntimePath);
MachOFile machOFile = MachOFile();
await machOFile.loadFromFile(originalExecutableFile);
// Insert the new segment that contains our snapshot data.
File newSegmentFile = File(payloadPath);
// Note that these two values MUST match the ones in
// runtime/bin/snapshot_utils.cc, which looks specifically for the snapshot in
// this segment/section.
final linkeditCommand = await insertSegmentDefinition(
machOFile, newSegmentFile, kSnapshotSegmentName, kSnapshotSectionName);
// Write out the new executable, with the same contents except the new header.
File outputFile = File(outputPath);
RandomAccessFile stream = await outputFile.open(mode: FileMode.write);
// Write the MachO header.
machOFile.writeSync(stream);
final int headerBytesWritten = stream.positionSync();
RandomAccessFile newSegmentFileStream = await newSegmentFile.open();
RandomAccessFile originalFileStream = await originalExecutableFile.open();
await originalFileStream.setPosition(headerBytesWritten);
// Write the unchanged data from the original file.
await pipeStream(originalFileStream, stream,
numToWrite: linkeditCommand.fileoff.asInt() - headerBytesWritten);
// Write the inserted section data, ensuring that the data is padded to the
// segment size.
await pipeStream(newSegmentFileStream, stream);
final int newSegmentLength = newSegmentFileStream.lengthSync();
final int alignedSegmentSize = vmSizeAlign(newSegmentLength);
await stream.writeFrom(List.filled(alignedSegmentSize - newSegmentLength, 0));
// Copy the rest of the file from the original to the new one.
await pipeStream(originalFileStream, stream);
await stream.close();
if (machOFile.hasCodeSignature) {
// After writing the modified file, we perform ad-hoc signing (no identity)
// similar to the linker (the linker-signed option flag) to ensure that any
// LC_CODE_SIGNATURE block has the correct CD hashes. This is necessary for
// platforms where signature verification is always on (e.g., OS X on M1).
final signingProcess = await Process.run(
'codesign', ['-o', 'linker-signed', '-s', '-', outputPath]);
if (signingProcess.exitCode != 0) {
print('Subcommand terminated with exit code ${signingProcess.exitCode}.');
if (signingProcess.stdout.isNotEmpty) {
print('Subcommand stdout:');
print(signingProcess.stdout);
}
if (signingProcess.stderr.isNotEmpty) {
print('Subcommand stderr:');
print(signingProcess.stderr);
}
throw 'Could not sign the new executable';
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,376 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';
import './macho.dart';
extension ByteReader on RandomAccessFile {
Uint32 readUint32() {
Uint8List rawBytes = readSync(4);
var byteView = ByteData.view(rawBytes.buffer);
return Uint32(byteView.getUint32(0, Endian.little));
}
Uint64 readUint64() {
Uint8List rawBytes = readSync(8);
var byteView = ByteData.view(rawBytes.buffer);
return Uint64(byteView.getUint64(0, Endian.little));
}
Int32 readInt32() {
Uint8List rawBytes = readSync(4);
var byteView = ByteData.view(rawBytes.buffer);
return Int32(byteView.getInt32(0, Endian.little));
}
}
class MachOFile {
IMachOHeader? header;
// The headerMaxOffset is set during parsing based on the maximum offset for
// segment offsets. Assuming the header start at byte 0 (that seems to always
// be the case), this number represents the total size of the header, which
// often includes a significant amount of zero-padding.
int headerMaxOffset = 0;
// We keep track on whether a code signature was seen so we can recreate it
// in the case that the binary has a CD hash that nededs updating.
bool hasCodeSignature = false;
// This wil contain all of the "load commands" in this MachO file. A load
// command is really a typed schema that indicates various parts of the MachO
// file (e.g. where to find the TEXT and DATA sections).
List<IMachOLoadCommand> commands =
List<IMachOLoadCommand>.empty(growable: true);
MachOFile();
// Returns the number of bytes read from the file.
Future<int> loadFromFile(File file) async {
// Ensure the file is long enough to contain the magic bytes.
final int fileLength = await file.length();
if (fileLength < 4) {
throw FormatException(
"File was not formatted properly. Length was too short: $fileLength");
}
// Read the first 4 bytes to see what type of MachO file this is.
var stream = await file.open();
var magic = stream.readUint32();
bool is64Bit = magic == MachOConstants.MH_MAGIC_64 ||
magic == MachOConstants.MH_CIGAM_64;
await stream.setPosition(0);
// Set the max header offset to the maximum file size so that when we read
// in the header we can correctly set the total header size.
headerMaxOffset = (1 << 63) - 1;
header = await _headerFromStream(stream, is64Bit);
if (header == null) {
throw FormatException(
"Could not parse a MachO header from the file: ${file.path}");
} else {
commands = await _commandsFromStream(stream, header!);
}
return stream.positionSync();
}
Future<MachOSymtabCommand> parseSymtabFromStream(
final Uint32 cmdsize, RandomAccessFile stream) async {
final symoff = stream.readUint32();
final nsyms = stream.readUint32();
final stroff = stream.readUint32();
final strsize = stream.readUint32();
return MachOSymtabCommand(cmdsize, symoff, nsyms, stroff, strsize);
}
Future<MachODysymtabCommand> parseDysymtabFromStream(
final Uint32 cmdsize, RandomAccessFile stream) async {
final ilocalsym = stream.readUint32();
final nlocalsym = stream.readUint32();
final iextdefsym = stream.readUint32();
final nextdefsym = stream.readUint32();
final iundefsym = stream.readUint32();
final nundefsym = stream.readUint32();
final tocoff = stream.readUint32();
final ntoc = stream.readUint32();
final modtaboff = stream.readUint32();
final nmodtab = stream.readUint32();
final extrefsymoff = stream.readUint32();
final nextrefsyms = stream.readUint32();
final indirectsymoff = stream.readUint32();
final nindirectsyms = stream.readUint32();
final extreloff = stream.readUint32();
final nextrel = stream.readUint32();
final locreloff = stream.readUint32();
final nlocrel = stream.readUint32();
return MachODysymtabCommand(
cmdsize,
ilocalsym,
nlocalsym,
iextdefsym,
nextdefsym,
iundefsym,
nundefsym,
tocoff,
ntoc,
modtaboff,
nmodtab,
extrefsymoff,
nextrefsyms,
indirectsymoff,
nindirectsyms,
extreloff,
nextrel,
locreloff,
nlocrel);
}
Future<MachOLinkeditDataCommand> parseLinkeditDataCommand(
final Uint32 cmd, final Uint32 cmdsize, RandomAccessFile stream) async {
final dataoff = stream.readUint32();
final datasize = stream.readUint32();
return MachOLinkeditDataCommand(
cmd,
cmdsize,
dataoff,
datasize,
);
}
Future<MachODyldInfoCommand> parseDyldInfoFromStream(
final Uint32 cmd, final Uint32 cmdsize, RandomAccessFile stream) async {
// Note that we're relying on the fact that the mirror returns the list of
// fields in the same order they're defined ni the class definition.
final rebaseOff = stream.readUint32();
final rebaseSize = stream.readUint32();
final bindOff = stream.readUint32();
final bindSize = stream.readUint32();
final weakBindOff = stream.readUint32();
final weakBindSize = stream.readUint32();
final lazyBindOff = stream.readUint32();
final lazyBindSize = stream.readUint32();
final exportOff = stream.readUint32();
final exportSize = stream.readUint32();
return MachODyldInfoCommand(
cmd,
cmdsize,
rebaseOff,
rebaseSize,
bindOff,
bindSize,
weakBindOff,
weakBindSize,
lazyBindOff,
lazyBindSize,
exportOff,
exportSize);
}
Future<MachOSegmentCommand64> parseSegmentCommand64FromStream(
final Uint32 cmdsize, RandomAccessFile stream) async {
final Uint8List segname = await stream.read(16);
final vmaddr = stream.readUint64();
final vmsize = stream.readUint64();
final fileoff = stream.readUint64();
final filesize = stream.readUint64();
final maxprot = stream.readInt32();
final initprot = stream.readInt32();
final nsects = stream.readUint32();
final flags = stream.readUint32();
if (nsects.asInt() == 0 && filesize.asInt() != 0) {
headerMaxOffset = min(headerMaxOffset, fileoff.asInt());
}
final sections = List.filled(nsects.asInt(), 0).map((_) {
final Uint8List sectname = stream.readSync(16);
final Uint8List segname = stream.readSync(16);
final addr = stream.readUint64();
final size = stream.readUint64();
final offset = stream.readUint32();
final align = stream.readUint32();
final reloff = stream.readUint32();
final nreloc = stream.readUint32();
final flags = stream.readUint32();
final reserved1 = stream.readUint32();
final reserved2 = stream.readUint32();
final reserved3 = stream.readUint32();
final notZerofill =
(flags & MachOConstants.S_ZEROFILL) != MachOConstants.S_ZEROFILL;
if (offset > 0 && size > 0 && notZerofill) {
headerMaxOffset = min(headerMaxOffset, offset.asInt());
}
return MachOSection64(sectname, segname, addr, size, offset, align,
reloff, nreloc, flags, reserved1, reserved2, reserved3);
}).toList();
return MachOSegmentCommand64(cmdsize, segname, vmaddr, vmsize, fileoff,
filesize, maxprot, initprot, nsects, flags, sections);
}
Future<IMachOHeader> _headerFromStream(
RandomAccessFile stream, bool is64Bit) async {
final magic = stream.readUint32();
final cputype = stream.readUint32();
final cpusubtype = stream.readUint32();
final filetype = stream.readUint32();
final ncmds = stream.readUint32();
final sizeofcmds = stream.readUint32();
final flags = stream.readUint32();
if (is64Bit) {
final reserved = stream.readUint32();
return MachOHeader(magic, cputype, cpusubtype, filetype, ncmds,
sizeofcmds, flags, reserved);
} else {
return MachOHeader32(
magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags);
}
}
void writeLoadCommandToStream(
IMachOLoadCommand command, RandomAccessFile stream) {
command.writeSync(stream);
}
void writeSync(RandomAccessFile stream) {
// Write the header.
stream.writeUint32(header!.magic);
stream.writeUint32(header!.cputype);
stream.writeUint32(header!.cpusubtype);
stream.writeUint32(header!.filetype);
stream.writeUint32(header!.ncmds);
stream.writeUint32(header!.sizeofcmds);
stream.writeUint32(header!.flags);
if (header is MachOHeader) {
stream.writeUint32(header!.reserved);
}
// Write all of the commands.
commands.forEach((command) {
writeLoadCommandToStream(command, stream);
});
// Pad the header according to the offset.
final int paddingAmount = headerMaxOffset - stream.positionSync();
if (paddingAmount > 0) {
stream.writeFromSync(List.filled(paddingAmount, 0));
}
}
Future<List<IMachOLoadCommand>> _commandsFromStream(
RandomAccessFile stream, IMachOHeader header) async {
final loadCommands = List<MachOLoadCommand>.empty(growable: true);
for (int i = 0; i < header.ncmds.asInt(); i++) {
final cmd = stream.readUint32();
final cmdsize = stream.readUint32();
// We need to read cmdsize bytes to get to the next command definition,
// but the cmdsize does includes the 2 bytes we just read (cmd +
// cmdsize) so we need to subtract those.
await stream
.setPosition((await stream.position()) + cmdsize.asInt() - 2 * 4);
loadCommands.add(MachOLoadCommand(cmd, cmdsize));
}
// Un-read all the bytes we just read.
var loadCommandsOffset = loadCommands
.map((command) => command.cmdsize)
.reduce((value, element) => value + element);
await stream
.setPosition((await stream.position()) - loadCommandsOffset.asInt());
final commands = List<IMachOLoadCommand>.empty(growable: true);
for (int i = 0; i < header.ncmds.asInt(); i++) {
final cmd = stream.readUint32();
final cmdsize = stream.readUint32();
// TODO(sarietta): Handle all MachO load command types. For now, since
// this implementation is exclusively being used to handle generating
// MacOS-compatible MachO executables for compiled dart scripts, only the
// load commands that are currently implemented are strictly necessary. It
// may be useful to handle all cases and pull this functionality out to a
// separate MachO library.
if (cmd == MachOConstants.LC_SEGMENT_64) {
commands.add(await parseSegmentCommand64FromStream(cmdsize, stream));
} else if (cmd == MachOConstants.LC_DYLD_INFO_ONLY ||
cmd == MachOConstants.LC_DYLD_INFO) {
commands.add(await parseDyldInfoFromStream(cmd, cmdsize, stream));
} else if (cmd == MachOConstants.LC_SYMTAB) {
commands.add(await parseSymtabFromStream(cmdsize, stream));
} else if (cmd == MachOConstants.LC_DYSYMTAB) {
commands.add(await parseDysymtabFromStream(cmdsize, stream));
} else if (cmd == MachOConstants.LC_CODE_SIGNATURE ||
cmd == MachOConstants.LC_SEGMENT_SPLIT_INFO ||
cmd == MachOConstants.LC_FUNCTION_STARTS ||
cmd == MachOConstants.LC_DATA_IN_CODE ||
cmd == MachOConstants.LC_DYLIB_CODE_SIGN_DRS) {
if (cmd == MachOConstants.LC_CODE_SIGNATURE) {
hasCodeSignature = true;
}
commands.add(await parseLinkeditDataCommand(cmd, cmdsize, stream));
} else if (cmd == MachOConstants.LC_SEGMENT ||
cmd == MachOConstants.LC_SYMSEG ||
cmd == MachOConstants.LC_THREAD ||
cmd == MachOConstants.LC_UNIXTHREAD ||
cmd == MachOConstants.LC_LOADFVMLIB ||
cmd == MachOConstants.LC_IDFVMLIB ||
cmd == MachOConstants.LC_IDENT ||
cmd == MachOConstants.LC_FVMFILE ||
cmd == MachOConstants.LC_PREPAGE ||
cmd == MachOConstants.LC_LOAD_DYLIB ||
cmd == MachOConstants.LC_ID_DYLIB ||
cmd == MachOConstants.LC_LOAD_DYLINKER ||
cmd == MachOConstants.LC_ID_DYLINKER ||
cmd == MachOConstants.LC_PREBOUND_DYLIB ||
cmd == MachOConstants.LC_ROUTINES ||
cmd == MachOConstants.LC_SUB_FRAMEWORK ||
cmd == MachOConstants.LC_SUB_UMBRELLA ||
cmd == MachOConstants.LC_SUB_CLIENT ||
cmd == MachOConstants.LC_SUB_LIBRARY ||
cmd == MachOConstants.LC_TWOLEVEL_HINTS ||
cmd == MachOConstants.LC_PREBIND_CKSUM ||
cmd == MachOConstants.LC_LOAD_WEAK_DYLIB ||
cmd == MachOConstants.LC_ROUTINES_64 ||
cmd == MachOConstants.LC_UUID ||
cmd == MachOConstants.LC_RPATH ||
cmd == MachOConstants.LC_REEXPORT_DYLIB ||
cmd == MachOConstants.LC_LAZY_LOAD_DYLIB ||
cmd == MachOConstants.LC_ENCRYPTION_INFO ||
cmd == MachOConstants.LC_LOAD_UPWARD_DYLIB ||
cmd == MachOConstants.LC_VERSION_MIN_MACOSX ||
cmd == MachOConstants.LC_VERSION_MIN_IPHONEOS ||
cmd == MachOConstants.LC_DYLD_ENVIRONMENT ||
cmd == MachOConstants.LC_MAIN ||
cmd == MachOConstants.LC_SOURCE_VERSION ||
cmd == MachOConstants.LC_BUILD_VERSION) {
// cmdsize includes the size of the contents + cmd + cmdsize
final contents = await stream.read(cmdsize.asInt() - 2 * 4);
commands.add(MachOGenericLoadCommand(cmd, cmdsize, contents));
} else {
// cmdsize includes the size of the contents + cmd + cmdsize
final contents = await stream.read(cmdsize.asInt() - 2 * 4);
commands.add(MachOGenericLoadCommand(cmd, cmdsize, contents));
final cmdString = "0x${cmd.asInt().toRadixString(16)}";
print("Found unknown MachO load command: $cmdString");
}
}
return commands;
}
}

View file

@ -4,6 +4,8 @@
import 'dart:io';
import 'package:dart2native/macho.dart';
import 'package:dart2native/macho_parser.dart';
import 'package:path/path.dart' as path;
import 'package:test/test.dart';
@ -21,6 +23,61 @@ const String unsoundNullSafetyMessage =
void defineCompileTests() {
final isRunningOnIA32 = Platform.version.contains('ia32');
if (Platform.isMacOS) {
test('Compile exe for MacOS signing', () async {
final p = project(mainSrc: '''void main() {}''');
final inFile =
path.canonicalize(path.join(p.dirPath, p.relativeFilePath));
final outFile = path.canonicalize(path.join(p.dirPath, 'myexe'));
var result = await p.run(
[
'compile',
'exe',
'-o',
outFile,
inFile,
],
);
expect(result.stdout, contains(soundNullSafetyMessage));
expect(result.stderr, isEmpty);
expect(result.exitCode, 0);
expect(File(outFile).existsSync(), true,
reason: 'File not found: $outFile');
// Ensure the file contains the __CUSTOM segment.
final machOFile = MachOFile();
await machOFile.loadFromFile(File(outFile));
// Throws an exception (and thus the test fails) if the segment doesn't
// exist.
machOFile.commands.where((segment) {
if (segment.asType() is MachOSegmentCommand64) {
final segmentName = (segment as MachOSegmentCommand64).segname;
final segmentNameTrimmed = String.fromCharCodes(
segmentName.takeWhile((value) => value != 0));
return segmentNameTrimmed == '__CUSTOM';
} else {
return false;
}
}).first;
// Ensure that the exe can be signed.
final codeSigningProcess = await Process.start('codesign', [
'-o',
'runtime',
'-s',
'-',
outFile,
]);
final signingResult = await codeSigningProcess.exitCode;
expect(signingResult, 0);
}, skip: isRunningOnIA32);
}
// *** NOTE ***: These tests *must* be run with the `--use-sdk` option
// as they depend on a fully built SDK to resolve various snapshot files
// used by compilation.
@ -153,9 +210,9 @@ void defineCompileTests() {
[],
);
expect(result.stdout, contains('I love executables'));
expect(result.stderr, isEmpty);
expect(result.exitCode, 0);
expect(result.stdout, contains('I love executables'));
}, skip: isRunningOnIA32);
test('Compile to executable disabled on IA32', () async {
@ -220,9 +277,9 @@ void defineCompileTests() {
[],
);
expect(result.stdout, contains('42'));
expect(result.stderr, isEmpty);
expect(result.exitCode, 0);
expect(result.stdout, contains('42'));
}, skip: isRunningOnIA32);
test('Compile and run aot snapshot', () async {
@ -443,9 +500,9 @@ void main() {}
[],
);
expect(result.stdout, contains('sound'));
expect(result.stderr, isEmpty);
expect(result.exitCode, 0);
expect(result.stdout, contains('sound'));
}, skip: isRunningOnIA32);
test('Compile and run exe with --no-sound-null-safety', () async {

View file

@ -72,4 +72,6 @@ builtin_impl_tests = [
"file_test.cc",
"hashmap_test.cc",
"priority_heap_test.cc",
"snapshot_utils_test.cc",
"test_utils.cc",
]

View file

@ -88,6 +88,7 @@ MappedMemory* File::Map(MapType type,
switch (type) {
case kReadOnly:
prot = PROT_READ;
map_flags |= MAP_RESILIENT_CODESIGN;
break;
case kReadExecute:
// Try to allocate near the VM's binary.

View file

@ -2,28 +2,18 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "bin/file.h"
#include "bin/dartutils.h"
#include "bin/directory.h"
#include "bin/file.h"
#include "bin/test_utils.h"
#include "platform/assert.h"
#include "platform/globals.h"
#include "vm/unit_test.h"
namespace dart {
// Helper method to be able to run the test from the runtime
// directory, or the top directory.
static const char* GetFileName(const char* name) {
if (bin::File::Exists(NULL, name)) {
return name;
} else {
static const int kRuntimeLength = strlen("runtime/");
return name + kRuntimeLength;
}
}
TEST_CASE(Read) {
const char* kFilename = GetFileName("runtime/bin/file_test.cc");
const char* kFilename = bin::test::GetFileName("runtime/bin/file_test.cc");
bin::File* file = bin::File::Open(NULL, kFilename, bin::File::kRead);
EXPECT(file != NULL);
char buffer[16];
@ -36,7 +26,7 @@ TEST_CASE(Read) {
}
TEST_CASE(OpenUri_RelativeFilename) {
const char* kFilename = GetFileName("runtime/bin/file_test.cc");
const char* kFilename = bin::test::GetFileName("runtime/bin/file_test.cc");
char* encoded = reinterpret_cast<char*>(bin::DartUtils::ScopedCString(
strlen(kFilename) * 3 + 1));
char* t = encoded;
@ -63,7 +53,8 @@ TEST_CASE(OpenUri_RelativeFilename) {
}
TEST_CASE(OpenUri_AbsoluteFilename) {
const char* kRelativeFilename = GetFileName("runtime/bin/file_test.cc");
const char* kRelativeFilename =
bin::test::GetFileName("runtime/bin/file_test.cc");
const char* kFilename = bin::File::GetCanonicalPath(NULL, kRelativeFilename);
EXPECT_NOTNULL(kFilename);
char* encoded = reinterpret_cast<char*>(bin::DartUtils::ScopedCString(
@ -100,7 +91,8 @@ static const char* Concat(const char* a, const char* b) {
}
TEST_CASE(OpenUri_ValidUri) {
const char* kRelativeFilename = GetFileName("runtime/bin/file_test.cc");
const char* kRelativeFilename =
bin::test::GetFileName("runtime/bin/file_test.cc");
const char* kAbsoluteFilename = bin::File::GetCanonicalPath(NULL,
kRelativeFilename);
EXPECT_NOTNULL(kAbsoluteFilename);
@ -132,7 +124,8 @@ TEST_CASE(OpenUri_ValidUri) {
}
TEST_CASE(OpenUri_UriWithSpaces) {
const char* kRelativeFilename = GetFileName("runtime/bin/file_test.cc");
const char* kRelativeFilename =
bin::test::GetFileName("runtime/bin/file_test.cc");
const char* strSystemTemp = bin::Directory::SystemTemp(NULL);
EXPECT_NOTNULL(strSystemTemp);
const char* kTempDir = Concat(strSystemTemp, "/foo bar");
@ -175,7 +168,7 @@ TEST_CASE(OpenUri_UriWithSpaces) {
}
TEST_CASE(OpenUri_InvalidUriPercentEncoding) {
const char* kFilename = GetFileName("runtime/bin/file_test.cc");
const char* kFilename = bin::test::GetFileName("runtime/bin/file_test.cc");
char* encoded = reinterpret_cast<char*>(bin::DartUtils::ScopedCString(
strlen(kFilename) * 3 + 1));
char* t = encoded;
@ -195,7 +188,7 @@ TEST_CASE(OpenUri_InvalidUriPercentEncoding) {
}
TEST_CASE(OpenUri_TruncatedUriPercentEncoding) {
const char* kFilename = GetFileName("runtime/bin/file_test.cc");
const char* kFilename = bin::test::GetFileName("runtime/bin/file_test.cc");
char* encoded = reinterpret_cast<char*>(bin::DartUtils::ScopedCString(
strlen(kFilename) * 3 + 1));
char* t = encoded;
@ -216,7 +209,7 @@ TEST_CASE(OpenUri_TruncatedUriPercentEncoding) {
TEST_CASE(FileLength) {
const char* kFilename =
GetFileName("runtime/tests/vm/data/fixed_length_file");
bin::test::GetFileName("runtime/tests/vm/data/fixed_length_file");
bin::File* file = bin::File::Open(NULL, kFilename, bin::File::kRead);
EXPECT(file != NULL);
EXPECT_EQ(42, file->Length());
@ -226,7 +219,7 @@ TEST_CASE(FileLength) {
TEST_CASE(FilePosition) {
char buf[42];
const char* kFilename =
GetFileName("runtime/tests/vm/data/fixed_length_file");
bin::test::GetFileName("runtime/tests/vm/data/fixed_length_file");
bin::File* file = bin::File::Open(NULL, kFilename, bin::File::kRead);
EXPECT(file != NULL);
EXPECT(file->ReadFully(buf, 12));

View file

@ -13,6 +13,9 @@
#include "bin/file.h"
#include "bin/platform.h"
#include "include/dart_api.h"
#if defined(DART_TARGET_OS_MACOS)
#include <platform/mach_o.h>
#endif
#include "platform/utils.h"
#define LOG_SECTION_BOUNDARIES false
@ -23,6 +26,11 @@ namespace bin {
static const int64_t kAppSnapshotHeaderSize = 5 * kInt64Size;
static const int64_t kAppSnapshotPageSize = 16 * KB;
static const char kMachOAppSnapshotSegmentName[] __attribute__((unused)) =
"__CUSTOM";
static const char kMachOAppSnapshotSectionName[] __attribute__((unused)) =
"__dart_app_snap";
class MappedAppSnapshot : public AppSnapshot {
public:
MappedAppSnapshot(MappedMemory* vm_snapshot_data,
@ -233,8 +241,121 @@ static AppSnapshot* TryReadAppSnapshotElf(
return nullptr;
}
#if defined(DART_TARGET_OS_MACOS)
AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElfFromMachO(
const char* container_path) {
File* file = File::Open(NULL, container_path, File::kRead);
if (file == nullptr) {
return nullptr;
}
RefCntReleaseScope<File> rs(file);
// Ensure file is actually MachO-formatted.
if (!IsMachOFormattedBinary(container_path)) {
Syslog::PrintErr(
"Attempted load target was not formatted as expected: "
"expected Mach-O binary.\n");
return nullptr;
}
// Parse the first 4bytes and extract the magic number.
uint32_t magic;
file->SetPosition(0);
file->Read(&magic, sizeof(uint32_t));
const bool is64Bit =
magic == mach_o::MH_MAGIC_64 || magic == mach_o::MH_CIGAM_64;
const bool isByteSwapped =
magic == mach_o::MH_CIGAM || magic == mach_o::MH_CIGAM_64;
if (isByteSwapped) {
Syslog::PrintErr(
"Dart snapshot contained an unexpected binary file layout. "
"Expected non-byte swapped header but found a byte-swapped header.\n");
return nullptr;
}
file->SetPosition(0);
// Read in the Mach-O header, which will contain information about all of the
// segments in the binary.
//
// From the header we determine where our special segment is located. This
// segment must be named according to the convention captured by
// kMachOAppSnapshotSegmentType and kMachOAppSnapshotSegmentName.
if (!is64Bit) {
Syslog::PrintErr(
"Dart snapshot compiled with 32bit architecture. "
"Currently only 64bit architectures are supported.\n");
return nullptr;
} else {
mach_o::mach_header_64 header;
file->Read(&header, sizeof(header));
for (uint32_t i = 0; i < header.ncmds; ++i) {
mach_o::load_command command;
file->Read(&command, sizeof(mach_o::load_command));
file->SetPosition(file->Position() - sizeof(command));
if (command.cmd != mach_o::LC_SEGMENT &&
command.cmd != mach_o::LC_SEGMENT_64) {
file->SetPosition(file->Position() + command.cmdsize);
continue;
}
mach_o::segment_command_64 segment;
file->Read(&segment, sizeof(segment));
for (uint32_t j = 0; j < segment.nsects; ++j) {
mach_o::section_64 section;
file->Read(&section, sizeof(section));
if (segment.cmd == mach_o::LC_SEGMENT_64 &&
strcmp(section.segname, kMachOAppSnapshotSegmentName) == 0 &&
strcmp(section.sectname, kMachOAppSnapshotSectionName) == 0) {
// We have to do the loading "by-hand" because we need to set the
// snapshot length to a specific length instead of the "rest of the
// file", which is the assumption that TryReadAppSnapshotElf makes.
const char* error = nullptr;
const uint8_t* vm_data_buffer = nullptr;
const uint8_t* vm_instructions_buffer = nullptr;
const uint8_t* isolate_data_buffer = nullptr;
const uint8_t* isolate_instructions_buffer = nullptr;
std::unique_ptr<uint8_t[]> snapshot(new uint8_t[section.size]);
file->SetPosition(section.offset);
file->Read(snapshot.get(), sizeof(uint8_t) * section.size);
Dart_LoadedElf* handle = Dart_LoadELF_Memory(
snapshot.get(), section.size, &error, &vm_data_buffer,
&vm_instructions_buffer, &isolate_data_buffer,
&isolate_instructions_buffer);
if (handle == nullptr) {
Syslog::PrintErr("Loading failed: %s\n", error);
return nullptr;
}
return new ElfAppSnapshot(handle, vm_data_buffer,
vm_instructions_buffer, isolate_data_buffer,
isolate_instructions_buffer);
}
}
}
}
return nullptr;
}
#endif // defined(DART_TARGET_OS_MACOS)
AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElf(
const char* container_path) {
#if defined(DART_TARGET_OS_MACOS)
if (IsMachOFormattedBinary(container_path)) {
return TryReadAppendedAppSnapshotElfFromMachO(container_path);
}
#endif
File* file = File::Open(NULL, container_path, File::kRead);
if (file == nullptr) {
return nullptr;
@ -329,6 +450,29 @@ static AppSnapshot* TryReadAppSnapshotDynamicLibrary(const char* script_name) {
#endif // defined(DART_PRECOMPILED_RUNTIME)
#if defined(DART_TARGET_OS_MACOS)
bool Snapshot::IsMachOFormattedBinary(const char* filename) {
File* file = File::Open(NULL, filename, File::kRead);
if (file == nullptr) {
return false;
}
RefCntReleaseScope<File> rs(file);
// Ensure the file is long enough to even contain the magic bytes.
if (file->Length() < 4) {
return false;
}
// Parse the first 4bytes and check the magic numbers.
uint32_t magic;
file->SetPosition(0);
file->Read(&magic, sizeof(uint32_t));
return magic == mach_o::MH_MAGIC_64 || magic == mach_o::MH_CIGAM_64 ||
magic == mach_o::MH_MAGIC || magic == mach_o::MH_CIGAM;
}
#endif // defined(DART_TARGET_OS_MACOS)
AppSnapshot* Snapshot::TryReadAppSnapshot(const char* script_uri,
bool force_load_elf_from_memory,
bool decode_uri) {

View file

@ -38,6 +38,10 @@ class Snapshot {
// an ELF binary). May report false negatives.
static bool IsAOTSnapshot(const char* snapshot_filename);
#if defined(DART_TARGET_OS_MACOS)
static bool IsMachOFormattedBinary(const char* container_path);
#endif
static AppSnapshot* TryReadAppendedAppSnapshotElf(const char* container_path);
static AppSnapshot* TryReadAppSnapshot(
const char* script_uri,
@ -54,6 +58,11 @@ class Snapshot {
intptr_t isolate_instructions_size);
private:
#if defined(DART_TARGET_OS_MACOS)
static AppSnapshot* TryReadAppendedAppSnapshotElfFromMachO(
const char* container_path);
#endif
DISALLOW_ALLOCATION();
DISALLOW_IMPLICIT_CONSTRUCTORS(Snapshot);
};

View file

@ -0,0 +1,38 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "bin/snapshot_utils.h"
#include "bin/file.h"
#include "bin/test_utils.h"
#include "platform/assert.h"
#include "platform/globals.h"
#include "vm/unit_test.h"
namespace dart {
#if defined(DART_TARGET_OS_MACOS)
TEST_CASE(CanDetectMachOFiles) {
const char* kMachO32BitLittleEndianFilename =
bin::test::GetFileName("runtime/tests/vm/data/macho_32bit_little_endian");
const char* kMachO64BitLittleEndianFilename =
bin::test::GetFileName("runtime/tests/vm/data/macho_64bit_little_endian");
const char* kMachO32BitBigEndianFilename =
bin::test::GetFileName("runtime/tests/vm/data/macho_32bit_big_endian");
const char* kMachO64BitBigEndianFilename =
bin::test::GetFileName("runtime/tests/vm/data/macho_64bit_big_endian");
EXPECT(
bin::Snapshot::IsMachOFormattedBinary(kMachO32BitLittleEndianFilename));
EXPECT(
bin::Snapshot::IsMachOFormattedBinary(kMachO64BitLittleEndianFilename));
EXPECT(bin::Snapshot::IsMachOFormattedBinary(kMachO32BitBigEndianFilename));
EXPECT(bin::Snapshot::IsMachOFormattedBinary(kMachO64BitBigEndianFilename));
const char* kFilename =
bin::test::GetFileName("runtime/bin/snapshot_utils_test.cc");
EXPECT(!bin::Snapshot::IsMachOFormattedBinary(kFilename));
}
#endif
} // namespace dart

23
runtime/bin/test_utils.cc Normal file
View file

@ -0,0 +1,23 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "bin/test_utils.h"
#include "bin/file.h"
namespace dart {
namespace bin {
namespace test {
const char* GetFileName(const char* name) {
if (bin::File::Exists(NULL, name)) {
return name;
} else {
static const int kRuntimeLength = strlen("runtime/");
return name + kRuntimeLength;
}
}
} // namespace test
} // namespace bin
} // namespace dart

20
runtime/bin/test_utils.h Normal file
View file

@ -0,0 +1,20 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_BIN_TEST_UTILS_H_
#define RUNTIME_BIN_TEST_UTILS_H_
namespace dart {
namespace bin {
namespace test {
// Helper method to be able to run the test from the runtime
// directory, or the top directory.
const char* GetFileName(const char* name);
} // namespace test
} // namespace bin
} // namespace dart
#endif // RUNTIME_BIN_TEST_UTILS_H_

104
runtime/platform/mach_o.h Normal file
View file

@ -0,0 +1,104 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_PLATFORM_MACH_O_H_
#define RUNTIME_PLATFORM_MACH_O_H_
#include <platform/globals.h>
namespace dart {
namespace mach_o {
#pragma pack(push, 1)
typedef int cpu_type_t;
typedef int cpu_subtype_t;
typedef int vm_prot_t;
struct mach_header {
uint32_t magic;
cpu_type_t cputype;
cpu_subtype_t cpusubtype;
uint32_t filetype;
uint32_t ncmds;
uint32_t sizeofcmds;
uint32_t flags;
};
static const uint32_t MH_MAGIC = 0xfeedface;
static const uint32_t MH_CIGAM = 0xcefaedfe;
struct mach_header_64 {
uint32_t magic;
cpu_type_t cputype;
cpu_subtype_t cpusubtype;
uint32_t filetype;
uint32_t ncmds;
uint32_t sizeofcmds;
uint32_t flags;
uint32_t reserved;
};
static const uint32_t MH_MAGIC_64 = 0xfeedfacf;
static const uint32_t MH_CIGAM_64 = 0xcffaedfe;
struct load_command {
uint32_t cmd;
uint32_t cmdsize;
};
static const uint32_t LC_SEGMENT = 0x1;
static const uint32_t LC_SEGMENT_64 = 0x19;
struct section {
char sectname[16];
char segname[16];
uint32_t addr;
uint32_t size;
uint32_t offset;
uint32_t align;
uint32_t reloff;
uint32_t nreloc;
uint32_t flags;
uint32_t reserved1;
uint32_t reserved2;
};
struct section_64 {
char sectname[16];
char segname[16];
uint64_t addr;
uint64_t size;
uint32_t offset;
uint32_t align;
uint32_t reloff;
uint32_t nreloc;
uint32_t flags;
uint32_t reserved1;
uint32_t reserved2;
uint32_t reserved3;
};
struct segment_command_64 {
uint32_t cmd;
uint32_t cmdsize;
char segname[16];
uint64_t vmaddr;
uint64_t vmsize;
uint64_t fileoff;
uint64_t filesize;
vm_prot_t maxprot;
vm_prot_t initprot;
uint32_t nsects;
uint32_t flags;
};
#pragma pack(pop)
} // namespace mach_o
} // namespace dart
#endif // RUNTIME_PLATFORM_MACH_O_H_

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.