languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-15 21:40:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extract mmap support from dart2js into package:mmap

Browse source 
The purpose of the CL is to enable re-use of the mmap support in dart2js
in other tools (e.g. package:heapsnapshot & package:kernel).

There's a small refactoring to remove zero-termination logic out of the
general mmap support.

Change-Id: I7a9889acea43d5ce0ab1eb10dcefbfa74c44bf93
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/279348
Reviewed-by: Jens Johansen <jensj@google.com>
Reviewed-by: Nate Biggs <natebiggs@google.com>
Commit-Queue: Martin Kustermann <kustermann@google.com>
This commit is contained in:
Martin Kustermann 2023-01-23 09:10:23 +00:00 committed by Commit Queue
parent ba15a61271
commit 44ec0ae07d
7 changed files with 348 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

174
pkg/compiler/lib/src/io/mapped_file.dart
View file

@ -6,172 +6,24 @@
// SDK is rolled.
// ignore_for_file: deprecated_member_use
import 'dart:ffi';
import 'dart:io';
import 'dart:typed_data';
import 'package:compiler/src/source_file_provider.dart';
import 'package:ffi/ffi.dart';
// int open(const char *path, int oflag, ...);
@FfiNative<Int32 Function(Pointer<Utf8>, Int32)>("open")
external int open(Pointer<Utf8> filename, int flags);
// int close(int fd);
@FfiNative<Int32 Function(Int32)>("close")
external int close(int fd);
// void* mmap(void* addr, size_t length,
// int prot, int flags,
// int fd, off_t offset)
@FfiNative<
Pointer<Uint8> Function(
Pointer<Uint8>, IntPtr, Int32, Int32, Int32, IntPtr)>("mmap")
external Pointer<Uint8> mmap(
Pointer<Uint8> address, int len, int prot, int flags, int fd, int offset);
// int munmap(void *addr, size_t length)
@FfiNative<IntPtr Function(Pointer<Uint8> address, IntPtr len)>("munmap")
external int munmap(Pointer<Uint8> address, int len);
final processSymbols = DynamicLibrary.process();
final munmapNative = processSymbols.lookup<Void>('munmap');
final closeNative = processSymbols.lookup<Void>('close');
final freeNative = processSymbols.lookup<Void>('free');
// int mprotect(void *addr, size_t len, int prot)
@FfiNative<Int32 Function(Pointer<Uint8>, IntPtr, Int32)>("mprotect")
external int mprotect(Pointer<Uint8> addr, int len, int prot);
// DART_EXPORT Dart_Handle
// Dart_NewExternalTypedDataWithFinalizer(Dart_TypedData_Type type,
// void* data,
// intptr_t length,
// void* peer,
// intptr_t external_allocation_size,
// Dart_HandleFinalizer callback)
typedef Dart_NewExternalTypedDataWithFinalizerNative = Handle Function(
Int32, Pointer<Void>, IntPtr, Pointer<Void>, IntPtr, Pointer<Void>);
typedef Dart_NewExternalTypedDataWithFinalizerDart = Object Function(
int, Pointer<Void>, int, Pointer<Void>, int, Pointer<Void>);
final Dart_NewExternalTypedDataWithFinalizer = processSymbols.lookupFunction<
Dart_NewExternalTypedDataWithFinalizerNative,
Dart_NewExternalTypedDataWithFinalizerDart>(
'Dart_NewExternalTypedDataWithFinalizer');
const int kPageSize = 4096;
const int kProtRead = 1;
const int kProtWrite = 2;
const int kProtExec = 4;
const int kMapPrivate = 2;
const int kMapAnon = 0x20;
const int kMapFailed = -1;
// We need to attach the finalizer which calls close() and
final finalizerAddress = () {
final finalizerStub = mmap(nullptr, kPageSize, kProtRead | kProtWrite,
kMapPrivate | kMapAnon, -1, 0);
finalizerStub.cast<Uint8>().asTypedList(kPageSize).setAll(0, <int>[
// Regenerate by running dart mmap.dart gen
// ASM_START
// #include <cstddef>
// #include <cstdint>
//
// struct PeerData {
// int (*close)(int);
// int (*munmap)(void*, size_t);
// int (*free)(void*);
// void* mapping;
// intptr_t size;
// intptr_t fd;
// };
//
// extern "C" void finalizer(void* callback_data, void* peer) {
// auto data = static_cast<PeerData*>(peer);
// data->munmap(data->mapping, data->size);
// data->close(data->fd);
// data->free(peer);
// }
//
0x55, 0x48, 0x89, 0xf5, 0x48, 0x8b, 0x76, 0x20, 0x48, 0x8b, 0x7d, 0x18, //
0xff, 0x55, 0x08, 0x8b, 0x7d, 0x28, 0xff, 0x55, 0x00, 0x48, 0x8b, 0x45, //
0x10, 0x48, 0x89, 0xef, 0x5d, 0xff, 0xe0, //
// ASM_END
]);
if (mprotect(finalizerStub, kPageSize, kProtRead | kProtExec) != 0) {
throw 'Failed to write executable code to the memory.';
}
return finalizerStub.cast<Void>();
}();
class PeerData extends Struct {
external Pointer<Void> close;
external Pointer<Void> munmap;
external Pointer<Void> free;
external Pointer<Uint8> mapping;
@IntPtr()
external int size;
@IntPtr()
external int fd;
}
Uint8List toExternalDataWithFinalizer(
Pointer<Uint8> memory, int size, int length, int fd) {
final peer = malloc.allocate<PeerData>(sizeOf<PeerData>());
peer.ref.close = closeNative;
peer.ref.munmap = munmapNative;
peer.ref.free = freeNative;
peer.ref.mapping = memory;
peer.ref.size = size;
peer.ref.fd = fd;
return Dart_NewExternalTypedDataWithFinalizer(
/*Dart_TypedData_kUint8*/ 2,
memory.cast(),
length,
peer.cast(),
size,
finalizerAddress,
) as Uint8List;
}
import 'package:mmap/mmap.dart';
Uint8List viewOfFile(String filename, bool zeroTerminated) {
final cfilename = filename.toNativeUtf8();
final int fd = open(cfilename, 0);
malloc.free(cfilename);
if (fd == 0) throw 'failed to open';
try {
final length = File(filename).lengthSync();
int lengthRoundedUp = (length + kPageSize - 1) & ~(kPageSize - 1);
final result =
mmap(nullptr, lengthRoundedUp, kProtRead, kMapPrivate, fd, 0);
if (result.address == kMapFailed) throw 'failed to map';
try {
if (zeroTerminated) {
if (length == lengthRoundedUp) {
// In the rare case we need a zero-terminated list and the file size
// is exactly page-aligned we need to allocate a new list with extra
// room for the terminating 0.
return Uint8List(length + 1)
..setRange(
0,
length,
toExternalDataWithFinalizer(
result, lengthRoundedUp, length, fd));
}
return toExternalDataWithFinalizer(
result, lengthRoundedUp, length + 1, fd);
}
return toExternalDataWithFinalizer(result, lengthRoundedUp, length, fd);
} catch (_) {
munmap(result, lengthRoundedUp);
rethrow;
}
} catch (e) {
close(fd);
rethrow;
final mappedFile = mmapFile(filename);
if (!zeroTerminated) {
return mappedFile.fileBytes;
}
if (mappedFile.hasZeroPadding) {
return mappedFile.fileBytesZeroTerminated;
}
// In the rare case we need a zero-terminated list and the file size
// is exactly page-aligned we need to allocate a new list with extra
// room for the terminating 0.
return Uint8List(mappedFile.fileLength + 1)
..setRange(0, mappedFile.fileLength, mappedFile.fileBytes);
}
class MemoryMapSourceFileByteReader implements SourceFileByteReader {
@ -179,7 +31,7 @@ class MemoryMapSourceFileByteReader implements SourceFileByteReader {
@override
Uint8List getBytes(String filename, {bool zeroTerminated = true}) {
if (Platform.isLinux) {
if (supportsMMap) {
try {
return viewOfFile(filename, zeroTerminated);
} catch (e) {

2
pkg/compiler/pubspec.yaml
View file

@ -13,12 +13,12 @@ dependencies:
collection: any
crypto: any
dart2js_info: any
ffi: any
front_end: any
js_ast: any
js_runtime: any
js_shared: any
kernel: any
mmap: any
vm_service: any
# Use 'any' constraints here; we get our versions from the DEPS file.

84
pkg/mmap/lib/mmap.dart Normal file
View file

@ -0,0 +1,84 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:ffi';
import 'dart:io';
import 'dart:typed_data';
import 'package:ffi/ffi.dart';
import 'src/mmap_impl.dart';
/// Whether memory mapping files is supported.
///
/// Currently we only support linux-x64.
final bool supportsMMap = Platform.isLinux && sizeOf<IntPtr>() == 8;
class MemoryMappedFile {
/// The memory mapped region of the address space as bytes.
///
/// The [Uint8List]'s length is a multiple of page size and as such may be
/// larger than the file length. The extra bytes are guaranteed to be zero.
///
/// Once this list becomes unreachable the underlying mapping will be freed &
/// file descriptor will be closed.
final Uint8List mappedBytes;
/// The length of the file that was mapped.
final int fileLength;
MemoryMappedFile(this.mappedBytes, this.fileLength);
/// Whether the [mappedBytes] contain extra zeros after the file content.
bool get hasZeroPadding => (fileLength % kPageSize) != 0;
/// The bytes representing the file.
Uint8List get fileBytes => Uint8List.sublistView(mappedBytes, 0, fileLength);
/// The bytes representing the file plus an extra zero byte.
Uint8List get fileBytesZeroTerminated {
if (!hasZeroPadding) throw 'The mapped file is page aligned.';
return Uint8List.sublistView(mappedBytes, 0, fileLength + 1);
}
}
/// Maps the given [filename] into the virtual address space.
///
/// Notice this only works on platforms where [supportsMMap] returns `true`.
/// Notice that files of length 0 cannot be mapped.
MemoryMappedFile mmapFile(String filename) {
if (!supportsMMap) throw 'MMap not supported';
final Pointer<Utf8> cfilename = filename.toNativeUtf8();
final int fd = open(cfilename, 0);
malloc.free(cfilename);
if (fd == 0) throw 'failed to open';
try {
final int length = File(filename).lengthSync();
final int lengthRoundedUp = (length + kPageSize - 1) & ~(kPageSize - 1);
final Pointer<Uint8> result =
mmap(nullptr, lengthRoundedUp, kProtRead, kMapPrivate, fd, 0);
if (result.address == kMapFailed) throw 'failed to map';
try {
final Uint8List bytes = toExternalDataWithFinalizer(
result, lengthRoundedUp, lengthRoundedUp, fd);
return MemoryMappedFile(bytes, length);
} catch (_) {
munmap(result, lengthRoundedUp);
rethrow;
}
} catch (e) {
close(fd);
rethrow;
}
}
Uint8List mmapOrReadFileSync(String filename) {
if (supportsMMap) {
try {
return mmapFile(filename).fileBytes;
} catch (_) {}
}
return File(filename).readAsBytesSync();
}

135
pkg/mmap/lib/src/mmap_impl.dart Normal file
View file

@ -0,0 +1,135 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// File is compiled with checked in SDK, update [FfiNative]s to [Native] when
// SDK is rolled.
// ignore_for_file: deprecated_member_use
import 'dart:ffi';
import 'dart:typed_data';
import 'package:ffi/ffi.dart';
// int open(const char *path, int oflag, ...);
@FfiNative<Int32 Function(Pointer<Utf8>, Int32)>("open")
external int open(Pointer<Utf8> filename, int flags);
// int close(int fd);
@FfiNative<Int32 Function(Int32)>("close")
external int close(int fd);
// void* mmap(void* addr, size_t length,
// int prot, int flags,
// int fd, off_t offset)
@FfiNative<
Pointer<Uint8> Function(
Pointer<Uint8>, IntPtr, Int32, Int32, Int32, IntPtr)>("mmap")
external Pointer<Uint8> mmap(
Pointer<Uint8> address, int len, int prot, int flags, int fd, int offset);
// int munmap(void *addr, size_t length)
@FfiNative<IntPtr Function(Pointer<Uint8> address, IntPtr len)>("munmap")
external int munmap(Pointer<Uint8> address, int len);
final DynamicLibrary processSymbols = DynamicLibrary.process();
final munmapNative = processSymbols.lookup<Void>('munmap');
final closeNative = processSymbols.lookup<Void>('close');
final freeNative = processSymbols.lookup<Void>('free');
// int mprotect(void *addr, size_t len, int prot)
@FfiNative<Int32 Function(Pointer<Uint8>, IntPtr, Int32)>("mprotect")
external int mprotect(Pointer<Uint8> addr, int len, int prot);
// DART_EXPORT Dart_Handle
// Dart_NewExternalTypedDataWithFinalizer(Dart_TypedData_Type type,
// void* data,
// intptr_t length,
// void* peer,
// intptr_t external_allocation_size,
// Dart_HandleFinalizer callback)
typedef Dart_NewExternalTypedDataWithFinalizerNative = Handle Function(
Int32, Pointer<Void>, IntPtr, Pointer<Void>, IntPtr, Pointer<Void>);
typedef Dart_NewExternalTypedDataWithFinalizerDart = Object Function(
int, Pointer<Void>, int, Pointer<Void>, int, Pointer<Void>);
final Dart_NewExternalTypedDataWithFinalizer = processSymbols.lookupFunction<
Dart_NewExternalTypedDataWithFinalizerNative,
Dart_NewExternalTypedDataWithFinalizerDart>(
'Dart_NewExternalTypedDataWithFinalizer');
const int kPageSize = 4096;
const int kProtRead = 1;
const int kProtWrite = 2;
const int kProtExec = 4;
const int kMapPrivate = 2;
const int kMapAnon = 0x20;
const int kMapFailed = -1;
// We need to attach the finalizer which calls close() and
final finalizerAddress = () {
final Pointer<Uint8> finalizerStub = mmap(nullptr, kPageSize,
kProtRead | kProtWrite, kMapPrivate | kMapAnon, -1, 0);
finalizerStub.cast<Uint8>().asTypedList(kPageSize).setAll(0, <int>[
// Regenerate by running dart mmap.dart gen
// ASM_START
// #include <cstddef>
// #include <cstdint>
//
// struct PeerData {
// int (*close)(int);
// int (*munmap)(void*, size_t);
// int (*free)(void*);
// void* mapping;
// intptr_t size;
// intptr_t fd;
// };
//
// extern "C" void finalizer(void* callback_data, void* peer) {
// auto data = static_cast<PeerData*>(peer);
// data->munmap(data->mapping, data->size);
// data->close(data->fd);
// data->free(peer);
// }
//
0x55, 0x48, 0x89, 0xf5, 0x48, 0x8b, 0x76, 0x20, 0x48, 0x8b, 0x7d, 0x18, //
0xff, 0x55, 0x08, 0x8b, 0x7d, 0x28, 0xff, 0x55, 0x00, 0x48, 0x8b, 0x45, //
0x10, 0x48, 0x89, 0xef, 0x5d, 0xff, 0xe0, //
// ASM_END
]);
if (mprotect(finalizerStub, kPageSize, kProtRead | kProtExec) != 0) {
throw 'Failed to write executable code to the memory.';
}
return finalizerStub.cast<Void>();
}();
class PeerData extends Struct {
external Pointer<Void> close;
external Pointer<Void> munmap;
external Pointer<Void> free;
external Pointer<Uint8> mapping;
@IntPtr()
external int size;
@IntPtr()
external int fd;
}
Uint8List toExternalDataWithFinalizer(
Pointer<Uint8> memory, int size, int length, int fd) {
final Pointer<PeerData> peer = malloc.allocate<PeerData>(sizeOf<PeerData>());
peer.ref.close = closeNative;
peer.ref.munmap = munmapNative;
peer.ref.free = freeNative;
peer.ref.mapping = memory;
peer.ref.size = size;
peer.ref.fd = fd;
return Dart_NewExternalTypedDataWithFinalizer(
/*Dart_TypedData_kUint8*/ 2,
memory.cast(),
length,
peer.cast(),
size,
finalizerAddress,
) as Uint8List;
}

16
pkg/mmap/pubspec.yaml Normal file
View file

@ -0,0 +1,16 @@
name: mmap
# This package is not intended for consumption on pub.dev. DO NOT publish.
publish_to: none
environment:
sdk: '>=2.14.0 <3.0.0'
# Use 'any' constraints here; we get our versions from the DEPS file.
dependencies:
ffi: any
# Use 'any' constraints here; we get our versions from the DEPS file.
dev_dependencies:
expect: any
path: any

97
pkg/mmap/test/mmap_test.dart Normal file
View file

@ -0,0 +1,97 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'dart:typed_data';
import 'package:expect/expect.dart';
import 'package:path/path.dart' as path;
import 'package:mmap/mmap.dart';
import 'package:mmap/src/mmap_impl.dart' show kPageSize;
final sizesToTest = [
0,
kPageSize - 1,
kPageSize,
2 * kPageSize - 1,
2 * kPageSize
];
main() {
final tempDir = Directory.systemTemp.createTempSync('mmap_test');
try {
testMmapOrReadFile(tempDir);
if (!supportsMMap) {
testUnsupported();
} else {
testSupported(tempDir);
}
} finally {
tempDir.deleteSync(recursive: true);
}
}
void testUnsupported() {
Expect.throws(() => mmapFile(Platform.executable));
}
void testSupported(Directory tempDir) {
for (final size in sizesToTest) {
final testFile = path.join(tempDir.path, 'file.bin');
File(testFile).writeAsBytesSync(initBytes(Uint8List(size)));
final fileLength = File(testFile).lengthSync();
Expect.equals(size, fileLength);
if (size == 0) {
Expect.throws(() => mmapFile(testFile));
continue;
}
final mapping = mmapFile(testFile);
Expect.equals(size, mapping.fileLength);
Expect.equals(size, mapping.fileBytes.length);
verifyBytes(mapping.fileBytes);
if (mapping.hasZeroPadding) {
verifyBytes(mapping.fileBytesZeroTerminated, 1);
verifyBytes(mapping.mappedBytes, kPageSize - (size % kPageSize));
} else {
verifyBytes(mapping.mappedBytes);
Expect.throws(() => mapping.fileBytesZeroTerminated);
}
}
}
void testMmapOrReadFile(Directory tempDir) {
for (final size in sizesToTest) {
final testFile = path.join(tempDir.path, 'file.bin');
File(testFile).writeAsBytesSync(initBytes(Uint8List(size)));
final fileLength = File(testFile).lengthSync();
Expect.equals(size, fileLength);
final bytes = mmapOrReadFileSync(testFile);
Expect.equals(size, bytes.length);
verifyBytes(bytes);
}
}
Uint8List initBytes(Uint8List bytes) {
for (int i = 0; i < bytes.length; ++i) {
bytes[i] = i % 23;
}
return bytes;
}
void verifyBytes(Uint8List bytes, [int zeroBytes = 0]) {
for (int i = 0; i < bytes.length - zeroBytes; ++i) {
Expect.equals(i % 23, bytes[i]);
}
for (int i = bytes.length - zeroBytes; i < bytes.length; ++i) {
Expect.equals(0, bytes[i]);
}
}

3
pkg/pkg.status
View file

@ -255,5 +255,6 @@ vm/test/modular_kernel_plus_aot_test: SkipByDesign # This test should only run i
front_end/test/fasta/*: Skip
front_end/tool/_fasta/*: Skip
[ $runtime == chrome || $runtime == ff || $runtime == firefox || $runtime == safari || $jscl ]
[ $browser || $jscl ]
compiler/test/*: Skip # dart2js uses #import('dart:io'); and it is not self-hosted (yet).
mmap/*: SkipByDesign # Only meant to run on vm