[modular_test] no longer generate .packages within modular test suites

Since the null-safety test migration we've been generating both a .packages file
and a package_config.json file. This deletes the use of the old .packages file
and updates all references to use the new file.

Change-Id: Iecef64ac8ed8579338795ad5327765118d643236
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/240650
Reviewed-by: Joshua Litt <joshualitt@google.com>
This commit is contained in:
Sigmund Cherem 2022-04-09 00:35:13 +00:00 committed by Commit Bot
parent dbc91a01ad
commit 441a762352
8 changed files with 94 additions and 224 deletions

View file

@ -463,7 +463,7 @@
"name": "modular_test",
"rootUri": "../pkg/modular_test",
"packageUri": "lib/",
"languageVersion": "2.2"
"languageVersion": "2.10"
},
{
"name": "native_stack_traces",

View file

@ -35,7 +35,6 @@ dev_dependencies:
http: any
js:
path: ../js
package_config: any
path: any
source_maps: any
# Unpublished packages that can be used via path dependency

View file

@ -14,11 +14,11 @@ import 'package:compiler/src/commandline_options.dart';
import 'package:compiler/src/kernel/dart2js_target.dart';
import 'package:front_end/src/compute_platform_binaries_location.dart'
show computePlatformBinariesLocation;
import 'package:modular_test/src/create_package_config.dart';
import 'package:modular_test/src/io_pipeline.dart';
import 'package:modular_test/src/pipeline.dart';
import 'package:modular_test/src/suite.dart';
import 'package:modular_test/src/runner.dart';
import 'package:package_config/package_config.dart';
import 'package:modular_test/src/suite.dart';
String packageConfigJsonPath = ".dart_tool/package_config.json";
Uri sdkRoot = Platform.script.resolve("../../../");
@ -42,17 +42,6 @@ const jsId = DataId("js");
const txtId = DataId("txt");
const fakeRoot = 'dev-dart-app:/';
String _packageConfigEntry(String name, Uri root,
{Uri packageRoot, LanguageVersion version}) {
var fields = [
'"name": "${name}"',
'"rootUri": "$root"',
if (packageRoot != null) '"packageUri": "$packageRoot"',
if (version != null) '"languageVersion": "$version"'
];
return '{${fields.join(',')}}';
}
String getRootScheme(Module module) {
// We use non file-URI schemes for representeing source locations in a
// root-agnostic way. This allows us to refer to file across modules and
@ -81,63 +70,6 @@ List<String> getSources(Module module) {
return module.sources.map((uri) => sourceToImportUri(module, uri)).toList();
}
void writePackageConfig(
Module module, Set<Module> transitiveDependencies, Uri root) async {
// TODO(joshualitt): Figure out a way to support package configs in
// tests/modular.
var packageConfig = await loadPackageConfigUri(packageConfigUri);
// We create both a .packages and package_config.json file which defines
// the location of this module if it is a package. The CFE requires that
// if a `package:` URI of a dependency is used in an import, then we need
// that package entry in the associated file. However, after it checks that
// the definition exists, the CFE will not actually use the resolved URI if
// a library for the import URI is already found in one of the provide
// .dill files of the dependencies. For that reason, and to ensure that
// a step only has access to the files provided in a module, we generate a
// config file with invalid folders for other packages.
// TODO(sigmund): follow up with the CFE to see if we can remove the need
// for these dummy entries..
// TODO(joshualitt): Generate just the json file.
var packagesJson = [];
var packagesContents = StringBuffer();
if (module.isPackage) {
packagesContents.write('${module.name}:${module.packageBase}\n');
packagesJson.add(_packageConfigEntry(
module.name, Uri.parse('../${module.packageBase}')));
}
int unusedNum = 0;
for (Module dependency in transitiveDependencies) {
if (dependency.isPackage) {
// rootUri should be ignored for dependent modules, so we pass in a
// bogus value.
var rootUri = Uri.parse('unused$unusedNum');
unusedNum++;
var dependentPackage = packageConfig[dependency.name];
var packageJson = dependentPackage == null
? _packageConfigEntry(dependency.name, rootUri)
: _packageConfigEntry(dependentPackage.name, rootUri,
version: dependentPackage.languageVersion);
packagesJson.add(packageJson);
packagesContents.write('${dependency.name}:$rootUri\n');
}
}
if (module.isPackage) {
await File.fromUri(root.resolve(packageConfigJsonPath))
.create(recursive: true);
await File.fromUri(root.resolve(packageConfigJsonPath)).writeAsString('{'
' "configVersion": ${packageConfig.version},'
' "packages": [ ${packagesJson.join(',')} ]'
'}');
}
await File.fromUri(root.resolve('.packages'))
.writeAsString('$packagesContents');
}
abstract class CFEStep extends IOModularStep {
final String stepName;
@ -158,11 +90,14 @@ abstract class CFEStep extends IOModularStep {
if (_options.verbose) print("\nstep: $stepName on $module");
Set<Module> transitiveDependencies = computeTransitiveDependencies(module);
writePackageConfig(module, transitiveDependencies, root);
await writePackageConfig(module, transitiveDependencies, root);
String rootScheme = getRootScheme(module);
List<String> sources;
List<String> extraArgs = ['--packages-file', '$rootScheme:/.packages'];
List<String> extraArgs = [
'--packages-file',
'$rootScheme:/$packageConfigJsonPath'
];
if (module.isSdk) {
// When no flags are passed, we can skip compilation and reuse the
// platform.dill created by build.py.
@ -318,21 +253,21 @@ class ModularAnalysisStep extends IOModularStep {
List<String> sources = [];
List<String> extraArgs = [];
if (!module.isSdk) {
writePackageConfig(module, transitiveDependencies, root);
await writePackageConfig(module, transitiveDependencies, root);
String rootScheme = getRootScheme(module);
sources = getSources(module);
dillDependencies = transitiveDependencies
.map((m) => '${toUri(m, dillSummaryId)}')
.toList();
extraArgs = [
'--packages=${root.resolve('.packages')}',
'--packages=${root.resolve(packageConfigJsonPath)}',
'--multi-root=$root',
'--multi-root-scheme=$rootScheme',
];
}
List<String> args = [
'--packages=${sdkRoot.toFilePath()}/.packages',
'--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
_dart2jsScript,
'--no-sound-null-safety',
if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
@ -400,7 +335,7 @@ class ConcatenateDillsStep extends IOModularStep {
.toList();
dataDependencies.add('${toUri(module, modularDataId)}');
List<String> args = [
'--packages=${sdkRoot.toFilePath()}/.packages',
'--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
_dart2jsScript,
// TODO(sigmund): remove this dependency on libraries.json
if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
@ -461,7 +396,7 @@ class ComputeClosedWorldStep extends IOModularStep {
.toList();
dataDependencies.add('${toUri(module, modularDataId)}');
List<String> args = [
'--packages=${sdkRoot.toFilePath()}/.packages',
'--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
_dart2jsScript,
// TODO(sigmund): remove this dependency on libraries.json
if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
@ -508,7 +443,7 @@ class GlobalAnalysisStep extends IOModularStep {
List<String> flags) async {
if (_options.verbose) print("\nstep: dart2js global analysis on $module");
List<String> args = [
'--packages=${sdkRoot.toFilePath()}/.packages',
'--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
_dart2jsScript,
// TODO(sigmund): remove this dependency on libraries.json
if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
@ -562,7 +497,7 @@ class Dart2jsCodegenStep extends IOModularStep {
List<String> flags) async {
if (_options.verbose) print("\nstep: dart2js backend on $module");
List<String> args = [
'--packages=${sdkRoot.toFilePath()}/.packages',
'--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
_dart2jsScript,
if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
'${Flags.entryUri}=$fakeRoot${module.mainSource}',
@ -615,7 +550,7 @@ class Dart2jsEmissionStep extends IOModularStep {
List<String> flags) async {
if (_options.verbose) print("step: dart2js backend on $module");
List<String> args = [
'--packages=${sdkRoot.toFilePath()}/.packages',
'--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
_dart2jsScript,
if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
'${Flags.entryUri}=$fakeRoot${module.mainSource}',

View file

@ -33,7 +33,6 @@ dev_dependencies:
lints: any
modular_test:
path: ../modular_test
package_config: any
sourcemap_testing:
path: ../sourcemap_testing
stack_trace: any

View file

@ -9,11 +9,11 @@
/// This is a shell that runs multiple tests, one per folder under `data/`.
import 'dart:io';
import 'package:modular_test/src/create_package_config.dart';
import 'package:modular_test/src/io_pipeline.dart';
import 'package:modular_test/src/pipeline.dart';
import 'package:modular_test/src/runner.dart';
import 'package:modular_test/src/suite.dart';
import 'package:package_config/package_config.dart';
String packageConfigJsonPath = '.dart_tool/package_config.json';
Uri sdkRoot = Platform.script.resolve('../../../');
@ -22,13 +22,8 @@ Options _options;
String _dartdevcScript;
String _kernelWorkerScript;
// TODO(joshualitt): Figure out a way to support package configs in
// tests/modular.
PackageConfig _packageConfig;
void main(List<String> args) async {
_options = Options.parse(args);
_packageConfig = await loadPackageConfigUri(packageConfigUri);
await _resolveScripts();
await runSuite(
sdkRoot.resolve('tests/modular/'),
@ -45,17 +40,6 @@ const dillId = DataId('dill');
const jsId = DataId('js');
const txtId = DataId('txt');
String _packageConfigEntry(String name, Uri root,
{Uri packageRoot, LanguageVersion version}) {
var fields = [
'"name": "$name"',
'"rootUri": "$root"',
if (packageRoot != null) '"packageUri": "$packageRoot"',
if (version != null) '"languageVersion": "$version"'
];
return '{${fields.join(',')}}';
}
class SourceToSummaryDillStep implements IOModularStep {
@override
List<DataId> get resultData => const [dillId];
@ -96,7 +80,7 @@ class SourceToSummaryDillStep implements IOModularStep {
_sourceToImportUri(module, rootScheme, relativeUri);
var transitiveDependencies = computeTransitiveDependencies(module);
await _createPackagesFile(module, root, transitiveDependencies);
await writePackageConfig(module, transitiveDependencies, root);
List<String> sources;
List<String> extraArgs;
@ -180,7 +164,7 @@ class DDCStep implements IOModularStep {
if (_options.verbose) print('\nstep: ddc on $module');
var transitiveDependencies = computeTransitiveDependencies(module);
await _createPackagesFile(module, root, transitiveDependencies);
await writePackageConfig(module, transitiveDependencies, root);
var rootScheme = module.isSdk ? 'dev-dart-sdk' : 'dev-dart-app';
List<String> sources;
@ -333,57 +317,6 @@ String get _d8executable {
throw UnsupportedError('Unsupported platform.');
}
Future<void> _createPackagesFile(
Module module, Uri root, Set<Module> transitiveDependencies) async {
// We create both a .packages and package_config.json file which defines
// the location of this module if it is a package. The CFE requires that
// if a `package:` URI of a dependency is used in an import, then we need
// that package entry in the associated file. However, after it checks that
// the definition exists, the CFE will not actually use the resolved URI if
// a library for the import URI is already found in one of the provide
// .dill files of the dependencies. For that reason, and to ensure that
// a step only has access to the files provided in a module, we generate a
// config file with invalid folders for other packages.
// TODO(sigmund): follow up with the CFE to see if we can remove the need
// for these dummy entries..
// TODO(joshualitt): Generate just the json file.
var packagesJson = [];
var packagesContents = StringBuffer();
if (module.isPackage) {
packagesContents.write('${module.name}:${module.packageBase}\n');
packagesJson.add(_packageConfigEntry(
module.name, Uri.parse('../${module.packageBase}')));
}
var unusedNum = 0;
for (var dependency in transitiveDependencies) {
if (dependency.isPackage) {
// rootUri should be ignored for dependent modules, so we pass in a
// bogus value.
var rootUri = Uri.parse('unused$unusedNum');
unusedNum++;
var dependentPackage = _packageConfig[dependency.name];
var packageJson = dependentPackage == null
? _packageConfigEntry(dependency.name, rootUri)
: _packageConfigEntry(dependentPackage.name, rootUri,
version: dependentPackage.languageVersion);
packagesJson.add(packageJson);
packagesContents.write('${dependency.name}:$rootUri\n');
}
}
if (module.isPackage) {
await File.fromUri(root.resolve(packageConfigJsonPath))
.create(recursive: true);
await File.fromUri(root.resolve(packageConfigJsonPath)).writeAsString('{'
' "configVersion": ${_packageConfig.version},'
' "packages": [ ${packagesJson.join(',')} ]'
'}');
}
await File.fromUri(root.resolve('.packages'))
.writeAsString('$packagesContents');
}
String _sourceToImportUri(Module module, String rootScheme, Uri relativeUri) {
if (module.isPackage) {
var basePath = module.packageBase.path;

View file

@ -9,11 +9,11 @@
/// This is a shell that runs multiple tests, one per folder under `data/`.
import 'dart:io';
import 'package:modular_test/src/create_package_config.dart';
import 'package:modular_test/src/io_pipeline.dart';
import 'package:modular_test/src/pipeline.dart';
import 'package:modular_test/src/runner.dart';
import 'package:modular_test/src/suite.dart';
import 'package:package_config/package_config.dart';
String packageConfigJsonPath = '.dart_tool/package_config.json';
Uri sdkRoot = Platform.script.resolve('../../../');
@ -22,13 +22,8 @@ Options _options;
String _dartdevcScript;
String _kernelWorkerScript;
// TODO(joshualitt): Figure out a way to support package configs in
// tests/modular.
PackageConfig _packageConfig;
void main(List<String> args) async {
_options = Options.parse(args);
_packageConfig = await loadPackageConfigUri(packageConfigUri);
await _resolveScripts();
await runSuite(
sdkRoot.resolve('tests/modular/'),
@ -45,17 +40,6 @@ const dillId = DataId('dill');
const jsId = DataId('js');
const txtId = DataId('txt');
String _packageConfigEntry(String name, Uri root,
{Uri packageRoot, LanguageVersion version}) {
var fields = [
'"name": "$name"',
'"rootUri": "$root"',
if (packageRoot != null) '"packageUri": "$packageRoot"',
if (version != null) '"languageVersion": "$version"'
];
return '{${fields.join(',')}}';
}
class SourceToSummaryDillStep implements IOModularStep {
@override
List<DataId> get resultData => const [dillId];
@ -96,7 +80,7 @@ class SourceToSummaryDillStep implements IOModularStep {
_sourceToImportUri(module, rootScheme, relativeUri);
var transitiveDependencies = computeTransitiveDependencies(module);
await _createPackagesFile(module, root, transitiveDependencies);
await writePackageConfig(module, transitiveDependencies, root);
List<String> sources;
List<String> extraArgs;
@ -182,7 +166,7 @@ class DDCStep implements IOModularStep {
if (_options.verbose) print('\nstep: ddc on $module');
var transitiveDependencies = computeTransitiveDependencies(module);
await _createPackagesFile(module, root, transitiveDependencies);
await writePackageConfig(module, transitiveDependencies, root);
var rootScheme = module.isSdk ? 'dev-dart-sdk' : 'dev-dart-app';
List<String> sources;
@ -337,57 +321,6 @@ String get _d8executable {
throw UnsupportedError('Unsupported platform.');
}
Future<void> _createPackagesFile(
Module module, Uri root, Set<Module> transitiveDependencies) async {
// We create both a .packages and package_config.json file which defines
// the location of this module if it is a package. The CFE requires that
// if a `package:` URI of a dependency is used in an import, then we need
// that package entry in the associated file. However, after it checks that
// the definition exists, the CFE will not actually use the resolved URI if
// a library for the import URI is already found in one of the provide
// .dill files of the dependencies. For that reason, and to ensure that
// a step only has access to the files provided in a module, we generate a
// config file with invalid folders for other packages.
// TODO(sigmund): follow up with the CFE to see if we can remove the need
// for these dummy entries..
// TODO(joshualitt): Generate just the json file.
var packagesJson = [];
var packagesContents = StringBuffer();
if (module.isPackage) {
packagesContents.write('${module.name}:${module.packageBase}\n');
packagesJson.add(_packageConfigEntry(
module.name, Uri.parse('../${module.packageBase}')));
}
var unusedNum = 0;
for (var dependency in transitiveDependencies) {
if (dependency.isPackage) {
// rootUri should be ignored for dependent modules, so we pass in a
// bogus value.
var rootUri = Uri.parse('unused$unusedNum');
unusedNum++;
var dependentPackage = _packageConfig[dependency.name];
var packageJson = dependentPackage == null
? _packageConfigEntry(dependency.name, rootUri)
: _packageConfigEntry(dependentPackage.name, rootUri,
version: dependentPackage.languageVersion);
packagesJson.add(packageJson);
packagesContents.write('${dependency.name}:$rootUri\n');
}
}
if (module.isPackage) {
await File.fromUri(root.resolve(packageConfigJsonPath))
.create(recursive: true);
await File.fromUri(root.resolve(packageConfigJsonPath)).writeAsString('{'
' "configVersion": ${_packageConfig.version},'
' "packages": [ ${packagesJson.join(',')} ]'
'}');
}
await File.fromUri(root.resolve('.packages'))
.writeAsString('$packagesContents');
}
String _sourceToImportUri(Module module, String rootScheme, Uri relativeUri) {
if (module.isPackage) {
var basePath = module.packageBase.path;

View file

@ -0,0 +1,71 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'package:package_config/package_config.dart';
import 'find_sdk_root.dart';
import 'suite.dart';
Future<void> writePackageConfig(
Module module, Set<Module> transitiveDependencies, Uri root) async {
const packageConfigJsonPath = ".dart_tool/package_config.json";
var sdkRoot = await findRoot();
Uri packageConfigUri = sdkRoot.resolve(packageConfigJsonPath);
var packageConfig = await loadPackageConfigUri(packageConfigUri);
// We create a package_config.json file to support the CFE in (a) determine
// the default nullability if the current module is a package, and (b)
// resolve `package:` imports. Technically the latter shouldn't be necessary,
// but the CFE requires that if a `package:` URI of a dependency is used in an
// import, then we need that package entry in the associated file. In fact,
// after it checks that the definition exists, the CFE will not actually use
// the resolved URI if a library for the import URI is already found in one of
// the provide .dill files of the dependencies. For that reason, and to ensure
// that a step only has access to the files provided in a module, we generate
// a config file with invalid folders for other packages.
// TODO(sigmund): follow up with the CFE to see if we can remove the need
// for these dummy entries.
var packagesJson = [];
if (module.isPackage) {
packagesJson.add(_packageConfigEntry(
module.name, Uri.parse('../${module.packageBase}')));
}
int unusedNum = 0;
for (Module dependency in transitiveDependencies) {
if (dependency.isPackage) {
// rootUri should be ignored for dependent modules, so we pass in a
// bogus value.
var rootUri = Uri.parse('unused$unusedNum');
unusedNum++;
var dependentPackage = packageConfig[dependency.name];
var packageJson = dependentPackage == null
? _packageConfigEntry(dependency.name, rootUri)
: _packageConfigEntry(dependentPackage.name, rootUri,
version: dependentPackage.languageVersion);
packagesJson.add(packageJson);
}
}
await File.fromUri(root.resolve(packageConfigJsonPath))
.create(recursive: true);
await File.fromUri(root.resolve(packageConfigJsonPath)).writeAsString('{'
' "configVersion": ${packageConfig.version},'
' "packages": [ ${packagesJson.join(',')} ]'
'}');
}
String _packageConfigEntry(String name, Uri root,
{Uri packageRoot, LanguageVersion version}) {
var fields = [
'"name": "${name}"',
'"rootUri": "$root"',
if (packageRoot != null) '"packageUri": "$packageRoot"',
if (version != null) '"languageVersion": "$version"'
];
return '{${fields.join(',')}}';
}

View file

@ -6,7 +6,7 @@ description: >
This is used within the Dart SDK to define and validate modular tests, and to
execute them using the modular pipeline of different SDK tools.
environment:
sdk: ">=2.2.1 <3.0.0"
sdk: ">=2.10.0 <3.0.0"
dependencies:
args: any