languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-11-05 18:22:09 +00:00
Code Issues Projects Releases Packages Wiki Activity

[gn] Consolidate exec_script calls to speed up generation

Browse source 
Calling out to python from GN to process gypi files is relatively
expensive with a 20-45ms fixed overhead for setup/teardown regardless
of what the script does. This makes runtime/vm/BUILD.gn take 1-1.5s
(per toolchain) to run as the template for libraries expands
out to 25 calls to gypi_to_gn.py, even though the actual time spent
processing the gypi files is negligible.

This replaces those repeated calls to gypi_to_gn.py with a call to a
custom script that process all of the gypi files needed for runtime/vm
and places the results into a single scope which can then be read from
in the template and replaces a few other scattered calls to
gypi_to_gn.py with a smaller number of calls that process multiple
gypi files and place the results into a single scope.

The end result is processing all of dart's GN files in a fuchsia build
takes ~250ms instead of >3 seconds.

R=zra@google.com

Review URL: https://codereview.chromium.org/2472813002 .
This commit is contained in:
James Robinson 2016-11-10 15:10:45 -08:00
parent f734606d0e
commit ad86d6ed26
6 changed files with 312 additions and 220 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
runtime/bin/BUILD.gn
View file

@ -2,6 +2,8 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import("gypi_contents.gni")
declare_args() {
# Whether to fall back to built-in root certificates when they cannot be
# verified at the operating system level.
@ -18,12 +20,6 @@ declare_args() {
dart_use_tcmalloc = false
}
resources_sources_gypi =
exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("vmservice/vmservice_sources.gypi") ],
"scope",
[ "vmservice/vmservice_sources.gypi" ])
# Generate a resources.cc file for the service isolate without Observatory.
action("gen_resources_cc") {
visibility = [ ":*" ] # Only targets in this file can see this.
@ -34,7 +30,7 @@ action("gen_resources_cc") {
# The path below is hard coded for the Mojo and Flutter trees. When moving
# the Dart runtime to gn, this path might need to be updated.
sources = rebase_path(resources_sources_gypi.sources, "", "../bin/vmservice/")
sources = rebase_path(resources_sources_gypi, "", "../bin/vmservice/")
outputs = [
"$target_gen_dir/resources_gen.cc",
]
@ -86,15 +82,10 @@ template("gen_library_src_path") {
}
}
builtin_sources_gypi = exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("builtin_sources.gypi") ],
"scope",
[ "builtin_sources.gypi" ])
gen_library_src_path("generate_builtin_cc_file") {
name = "_builtin"
kind = "source"
sources = builtin_sources_gypi.sources
sources = builtin_sources_gypi
output = "$target_gen_dir/builtin_gen.cc"
}
@ -113,15 +104,10 @@ gen_library_src_path("generate_io_cc_file") {
output = "$target_gen_dir/io_gen.cc"
}
io_sources_gypi = exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("io_sources.gypi") ],
"scope",
[ "io_sources.gypi" ])
gen_library_src_path("generate_io_patch_cc_file") {
name = "io"
kind = "patch"
sources = io_sources_gypi.sources
sources = io_sources_gypi
output = "$target_gen_dir/io_patch_gen.cc"
}
@ -253,12 +239,6 @@ config("libdart_builtin_config") {
}
}
builtin_impl_sources_gypi =
exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("builtin_impl_sources.gypi") ],
"scope",
[ "builtin_impl_sources.gypi" ])
static_library("libdart_builtin") {
configs += [
"..:dart_config",
@ -293,14 +273,9 @@ static_library("libdart_builtin") {
"log_linux.cc",
"log_macos.cc",
"log_win.cc",
] + builtin_impl_sources_gypi.sources
] + builtin_impl_sources_gypi
}
io_impl_sources_gypi = exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("io_impl_sources.gypi") ],
"scope",
[ "io_impl_sources.gypi" ])
executable("gen_snapshot") {
configs += [
"..:dart_config",
@ -386,7 +361,7 @@ source_set("gen_snapshot_dart_io") {
defines = [ "DART_IO_SECURE_SOCKET_DISABLED" ]
sources = io_impl_sources_gypi.sources + builtin_impl_sources_gypi.sources
sources = io_impl_sources_gypi + builtin_impl_sources_gypi
sources += [
"io_natives.cc",
"io_natives.h",
@ -452,7 +427,7 @@ template("dart_io") {
]
}
sources = io_impl_sources_gypi.sources + builtin_impl_sources_gypi.sources
sources = io_impl_sources_gypi + builtin_impl_sources_gypi
sources += [
"builtin_natives.cc",
"io_natives.cc",
@ -810,30 +785,12 @@ executable("run_vm_tests") {
# The VM sources are already included in libdart, so we just want to add in
# the tests here.
vm_tests_list = exec_script("../../tools/gypi_to_gn.py",
[
rebase_path("../vm/vm_sources.gypi"),
"--keep_only=_test.cc",
"--keep_only=_test.h",
],
"scope",
[ "../vm/vm_sources.gypi" ])
vm_tests = rebase_path(vm_tests_list.sources, ".", "../vm")
builtin_impl_tests_list =
exec_script("../../tools/gypi_to_gn.py",
[
rebase_path("builtin_impl_sources.gypi"),
"--keep_only=_test.cc",
"--keep_only=_test.h",
],
"scope",
[ "builtin_impl_sources.gypi" ])
vm_tests = rebase_path(vm_tests_list, ".", "../vm")
sources = [
"builtin_nolib.cc",
"run_vm_tests.cc",
] + builtin_impl_tests_list.sources + vm_tests
] + builtin_impl_tests_list + vm_tests
if (!is_win) {
ldflags = [ "-rdynamic" ]

43
runtime/bin/gypi_contents.gni Normal file
View file

@ -0,0 +1,43 @@
# Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
_gypi_files = [
"builtin_sources.gypi",
"builtin_impl_sources.gypi",
"vmservice/vmservice_sources.gypi",
"io_sources.gypi",
"io_impl_sources.gypi",
]
_gypi_contents = exec_script("../../tools/gypi_to_gn.py",
rebase_path(_gypi_files) + [ "--prefix" ],
"scope",
_gypi_files)
resources_sources_gypi = _gypi_contents.vmservice_sources_sources
builtin_sources_gypi = _gypi_contents.builtin_sources_sources
builtin_impl_sources_gypi = _gypi_contents.builtin_impl_sources_sources
io_sources_gypi = _gypi_contents.io_sources_sources
io_impl_sources_gypi = _gypi_contents.io_impl_sources_sources
_test_gypi_files = [
"../vm/vm_sources.gypi",
"builtin_impl_sources.gypi",
]
_test_only_gypi_contents = exec_script("../../tools/gypi_to_gn.py",
rebase_path(_test_gypi_files) + [
"--keep_only=_test.cc",
"--keep_only=_test.h",
"--prefix",
],
"scope",
_test_gypi_files)
# The VM sources are already included in libdart, so we just want to add in
# the tests here.
vm_tests_list = _test_only_gypi_contents.vm_sources_sources
builtin_impl_tests_list = _test_only_gypi_contents.builtin_impl_sources_sources

211
runtime/vm/BUILD.gn
View file

@ -4,6 +4,7 @@
import("../../build/executable_suffix.gni")
import("../../build/prebuilt_dart_sdk.gni")
import("gypi_contents.gni")
config("libdart_vm_config") {
if (defined(is_fuchsia) && is_fuchsia) {
@ -32,30 +33,12 @@ static_library("libdart_platform") {
]
public_configs = [ ":libdart_vm_config" ]
platform_headers_gypi =
exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("../platform/platform_headers.gypi") ],
"scope",
[ "../platform/platform_headers.gypi" ])
platform_headers =
rebase_path(platform_headers_gypi.sources, ".", "../platform")
sources = rebase_path(processed_gypis.platform_sources, ".", "../platform")
platform_sources_gypi =
exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("../platform/platform_sources.gypi") ],
"scope",
[ "../platform/platform_sources.gypi" ])
platform_sources =
rebase_path(platform_sources_gypi.sources, ".", "../platform")
sources = platform_headers + platform_sources
include_dirs = [ ".." ]
}
vm_sources_list = exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("vm_sources.gypi") ],
"scope",
[ "vm_sources.gypi" ])
vm_sources_list = processed_gypis.vm_sources
static_library("libdart_vm") {
configs += [
@ -68,7 +51,7 @@ static_library("libdart_vm") {
"*_test.cc",
"*_test.h",
])
sources = vm_sources_list.sources
sources = vm_sources_list
include_dirs = [ ".." ]
}
@ -84,7 +67,7 @@ static_library("libdart_vm_noopt") {
"*_test.cc",
"*_test.h",
])
sources = vm_sources_list.sources
sources = vm_sources_list
include_dirs = [ ".." ]
}
@ -99,7 +82,7 @@ static_library("libdart_vm_precompiled_runtime") {
"*_test.cc",
"*_test.h",
])
sources = vm_sources_list.sources
sources = vm_sources_list
include_dirs = [ ".." ]
}
@ -115,7 +98,7 @@ static_library("libdart_vm_nosnapshot") {
"*_test.cc",
"*_test.h",
])
sources = vm_sources_list.sources
sources = vm_sources_list
include_dirs = [ ".." ]
}
@ -131,7 +114,7 @@ static_library("libdart_vm_nosnapshot_precompiled_runtime") {
"*_test.cc",
"*_test.h",
])
sources = vm_sources_list.sources
sources = vm_sources_list
include_dirs = [ ".." ]
}
@ -147,53 +130,41 @@ static_library("libdart_vm_nosnapshot_with_precompiler") {
"*_test.cc",
"*_test.h",
])
sources = vm_sources_list.sources
sources = vm_sources_list
include_dirs = [ ".." ]
}
template("process_library_source") {
assert(defined(invoker.filename), "Need filename in $target_name")
assert(defined(invoker.libsources), "Need libsources in $target_name")
assert(defined(invoker.output), "Need output in $target_name")
assert(defined(invoker.path), "Need path in $target_name")
action(target_name) {
visibility = [ ":*" ] # Only targets in this file can see this.
filename = invoker.filename
path = invoker.path
lib_sources_gypi =
exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("${path}/${filename}_sources.gypi") ],
"scope",
[ "${path}/${filename}_sources.gypi" ])
lib_sources = rebase_path(lib_sources_gypi.sources, ".", path)
libsources = invoker.libsources
script = invoker.script
inputs = invoker.inputs + [ script ]
inputs += lib_sources
inputs = invoker.inputs + libsources
outputs = [
invoker.output,
]
args = invoker.args + rebase_path(lib_sources, root_build_dir)
args = invoker.args + rebase_path(libsources, root_build_dir)
}
}
template("generate_library_source") {
assert(defined(invoker.libname), "Need libname in $target_name")
assert(defined(invoker.filename), "Need a filename in $target_name")
assert(defined(invoker.libsources), "Need libsources in $target_name")
assert(defined(invoker.kind), "Need kind in $target_name")
assert(defined(invoker.output), "Need output in $target_name")
assert(defined(invoker.path), "Need path in $target_name")
process_library_source(target_name) {
libname = invoker.libname
filename = invoker.filename
libsources = invoker.libsources
kind = invoker.kind
output = invoker.output
path = invoker.path
script = "../tools/gen_library_src_paths.py"
inputs = [
"../lib/libgen_in.cc",
]
output = invoker.output
args = [
"--output",
rebase_path(invoker.output, root_build_dir),
@ -209,51 +180,45 @@ template("generate_library_source") {
}
}
# This templates expects invoker.sources to be a list of pairs of strings.
# The pairs of strings mean the following.
# library name, file name
# e.g. for the "internal" library named "dart:_internal",
# with sources listed at sdk/lib/internal/internal_sources.gypi and
# lib/internal_sources.gypi, we have: ["_internal", "internal"]
# This templates expects invoker.sources to be a list of lists.
# The lists contain the following information about each library:
# library name (string)
# library sources (list of strings)
# library source base path (string)
# filename (string)
# do_patch (boolean)
#
# If do_patch is true, the lists additionally contains
# patch sources (list of strings)
# patch source base path (string)
#
# The template iterates over the list, and generates generate_library_source
# actions for each. After that, it generates targets to compile the generated
# sources to make libdart_lib_nosnapshot and libdart_lib.
# sources to make libdart_lib_*.
template("generate_core_libraries") {
assert(defined(invoker.sources), "Need sources in $target_name")
assert(defined(invoker.allsources), "Need allsources in $target_name")
liboutputs = []
libsources = []
libdeps = []
foreach(lib, invoker.sources) {
libname = lib[0]
filename = lib[1]
do_patch = lib[2]
source_path = lib[3]
filename = lib[3]
do_patch = lib[4]
generate_library_source("generate_${filename}_cc_file") {
libname = libname
filename = filename
libsources = rebase_path(lib[1], ".", lib[2])
kind = "source"
path = source_path
output = "$target_gen_dir/${filename}_gen.cc"
}
if (do_patch) {
patch_path = lib[4]
generate_library_source("generate_${filename}_patch_cc_file") {
libname = libname
filename = filename
libsources = rebase_path(lib[5], ".", lib[6])
kind = "patch"
path = patch_path
output = "$target_gen_dir/${filename}_patch_gen.cc"
}
}
lib_sources_gypi = {
}
lib_sources_gypi =
exec_script("../../tools/gypi_to_gn.py",
[ rebase_path("../lib/${filename}_sources.gypi") ],
"scope",
[ "../lib/${filename}_sources.gypi" ])
libsources += rebase_path(lib_sources_gypi.sources, ".", "../lib")
liboutputs += [ "$target_gen_dir/${filename}_gen.cc" ]
libdeps += [ ":generate_${filename}_cc_file" ]
if (do_patch) {
@ -262,6 +227,8 @@ template("generate_core_libraries") {
}
}
all_libsources = rebase_path(invoker.allsources, ".", "../lib")
static_library("libdart_lib_nosnapshot") {
configs += [
"..:dart_config",
@ -269,7 +236,7 @@ template("generate_core_libraries") {
"..:dart_maybe_precompiled_runtime_config",
]
deps = libdeps
sources = libsources + [ "bootstrap.cc" ] + liboutputs
sources = all_libsources + [ "bootstrap.cc" ] + liboutputs
include_dirs = [ ".." ]
}
static_library("libdart_lib_nosnapshot_precompiled_runtime") {
@ -279,7 +246,7 @@ template("generate_core_libraries") {
"..:dart_precompiled_runtime_config",
]
deps = libdeps
sources = libsources + [ "bootstrap.cc" ] + liboutputs
sources = all_libsources + [ "bootstrap.cc" ] + liboutputs
include_dirs = [ ".." ]
}
static_library("libdart_lib_nosnapshot_with_precompiler") {
@ -289,7 +256,7 @@ template("generate_core_libraries") {
"..:dart_precompiler_config",
]
deps = libdeps
sources = libsources + [ "bootstrap.cc" ] + liboutputs
sources = all_libsources + [ "bootstrap.cc" ] + liboutputs
include_dirs = [ ".." ]
}
static_library("libdart_lib") {
@ -298,7 +265,7 @@ template("generate_core_libraries") {
"..:dart_maybe_product_config",
"..:dart_maybe_precompiled_runtime_config",
]
sources = libsources + [ "bootstrap_nocore.cc" ]
sources = all_libsources + [ "bootstrap_nocore.cc" ]
include_dirs = [ ".." ]
}
static_library("libdart_lib_precompiled_runtime") {
@ -307,7 +274,7 @@ template("generate_core_libraries") {
"..:dart_maybe_product_config",
"..:dart_precompiled_runtime_config",
]
sources = libsources + [ "bootstrap_nocore.cc" ]
sources = all_libsources + [ "bootstrap_nocore.cc" ]
include_dirs = [ ".." ]
}
}
@ -316,104 +283,125 @@ generate_core_libraries("core_libraries") {
sources = [
[
"async",
processed_gypis.async_sdk_sources,
"../../sdk/lib/async",
"async",
true,
"../../sdk/lib/async",
processed_gypis.async_runtime_sources,
"../lib",
],
[
"core",
"core",
true,
processed_gypis.core_sdk_sources,
"../../sdk/lib/core",
"core",
true,
processed_gypis.core_runtime_sources,
"../lib",
],
[
"collection",
"collection",
true,
processed_gypis.collection_sdk_sources,
"../../sdk/lib/collection",
"collection",
true,
processed_gypis.collection_runtime_sources,
"../lib",
],
[
"convert",
"convert",
true,
processed_gypis.convert_sdk_sources,
"../../sdk/lib/convert",
"convert",
true,
processed_gypis.convert_runtime_sources,
"../lib",
],
[
"developer",
processed_gypis.developer_sdk_sources,
"../../sdk/lib/developer",
"developer",
true,
"../../sdk/lib/developer",
processed_gypis.developer_runtime_sources,
"../lib",
],
[
"_internal",
processed_gypis.internal_sdk_sources,
"../../sdk/lib/internal",
"internal",
true,
"../../sdk/lib/internal",
processed_gypis.internal_runtime_sources,
"../lib",
],
[
"isolate",
"isolate",
true,
processed_gypis.isolate_sdk_sources,
"../../sdk/lib/isolate",
"isolate",
true,
processed_gypis.isolate_runtime_sources,
"../lib",
],
[
"math",
"math",
true,
processed_gypis.math_sdk_sources,
"../../sdk/lib/math",
"math",
true,
processed_gypis.math_runtime_sources,
"../lib",
],
[
"mirrors",
processed_gypis.mirrors_sdk_sources,
"../../sdk/lib/mirrors",
"mirrors",
true,
"../../sdk/lib/mirrors",
processed_gypis.mirrors_runtime_sources,
"../lib",
],
[
"profiler",
"profiler",
false,
processed_gypis.profiler_sdk_sources,
"../../sdk/lib/profiler",
"profiler",
false,
],
[
"typed_data",
processed_gypis.typed_data_runtime_sources,
"../lib",
"typed_data",
false,
"../lib",
],
[
"_vmservice",
processed_gypis.vmservice_sdk_sources,
"../../sdk/lib/vmservice",
"vmservice",
true,
"../../sdk/lib/vmservice",
processed_gypis.vmservice_runtime_sources,
"../lib",
],
]
allsources = processed_gypis.allsources
}
template("concatenate_patch") {
assert(defined(invoker.libname), "Need a name in $target_name")
assert(defined(invoker.dir), "Need a dir in $target_name")
assert(defined(invoker.libsources), "Need a sources in $target_name")
assert(defined(invoker.output), "Need an output in $target_name")
process_library_source(target_name) {
libsources = invoker.libsources
inputs = []
output = invoker.output
path = "../${invoker.dir}"
filename = invoker.libname
script = "../tools/concatenate_patches.py"
args = [
"--output",
rebase_path(output, root_build_dir),
]
inputs = []
}
}
@ -426,12 +414,11 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
# Concatenate vm library patches.
foreach(library, invoker.libraries) {
name = library[1]
name = library[0]
target_output = "$target_gen_dir/patches/${name}_patch.dart"
concatenate_patch("concatenate_${name}_patch") {
libname = name
dir = library[0]
libsources = library[1]
output = target_output
}
concatenation_target_names += [ ":concatenate_${name}_patch" ]
@ -507,52 +494,52 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
generate_patched_sdk("patched_sdk") {
libraries = [
[
"lib",
"async",
processed_gypis.async_runtime_sources,
],
[
"lib",
"collection",
processed_gypis.collection_runtime_sources,
],
[
"lib",
"convert",
processed_gypis.convert_runtime_sources,
],
[
"lib",
"core",
processed_gypis.core_runtime_sources,
],
[
"lib",
"developer",
processed_gypis.developer_runtime_sources,
],
[
"lib",
"internal",
processed_gypis.internal_runtime_sources,
],
[
"lib",
"isolate",
processed_gypis.isolate_runtime_sources,
],
[
"lib",
"math",
processed_gypis.math_runtime_sources,
],
[
"lib",
"mirrors",
processed_gypis.mirrors_runtime_sources,
],
[
"lib",
"profiler",
processed_gypis.profiler_runtime_sources,
],
[
"lib",
"vmservice",
processed_gypis.vmservice_runtime_sources,
],
[
"bin",
"io",
processed_gypis.bin_io_sources,
],
]
}

54
runtime/vm/gypi_contents.gni Normal file
View file

@ -0,0 +1,54 @@
# Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
_core_library_names = [
"async",
"core",
"collection",
"convert",
"developer",
"internal",
"isolate",
"math",
"mirrors",
"profiler",
"typed_data",
"vmservice",
]
_core_library_gypis = [
"../platform/platform_headers.gypi",
"../platform/platform_sources.gypi",
"../vm/vm_sources.gypi",
"../lib/async_sources.gypi",
"../lib/core_sources.gypi",
"../lib/collection_sources.gypi",
"../lib/convert_sources.gypi",
"../lib/developer_sources.gypi",
"../lib/internal_sources.gypi",
"../lib/isolate_sources.gypi",
"../lib/math_sources.gypi",
"../lib/mirrors_sources.gypi",
"../lib/typed_data_sources.gypi",
"../lib/vmservice_sources.gypi",
"../../sdk/lib/async/async_sources.gypi",
"../../sdk/lib/collection/collection_sources.gypi",
"../../sdk/lib/convert/convert_sources.gypi",
"../../sdk/lib/core/core_sources.gypi",
"../../sdk/lib/developer/developer_sources.gypi",
"../../sdk/lib/internal/internal_sources.gypi",
"../../sdk/lib/isolate/isolate_sources.gypi",
"../../sdk/lib/math/math_sources.gypi",
"../../sdk/lib/mirrors/mirrors_sources.gypi",
"../../sdk/lib/profiler/profiler_sources.gypi",
"../../sdk/lib/vmservice/vmservice_sources.gypi",
"../bin/io_sources.gypi",
]
processed_gypis = exec_script("../../tools/process_gypis.py",
_core_library_names,
"scope",
_core_library_gypis)

92
tools/gypi_to_gn.py
View file

@ -3,9 +3,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Converts a given gypi file to a python scope and writes the result to stdout.
"""Converts given gypi files to a python scope and writes the result to stdout.
It is assumed that the file contains a toplevel dictionary, and this script
It is assumed that the files contain a toplevel dictionary, and this script
will return that dictionary as a GN "scope" (see example below). This script
does not know anything about GYP and it will not expand variables or execute
conditions.
@ -22,10 +22,11 @@ Say your_file.gypi looked like this:
}
You would call it like this:
gypi_files = [ "your_file.gypi", "your_other_file.gypi" ]
gypi_values = exec_script("//build/gypi_to_gn.py",
[ rebase_path("your_file.gypi") ],
[ rebase_path(gypi_files) ],
"scope",
[ "your_file.gypi" ])
[ gypi_files ])
Notes:
- The rebase_path call converts the gypi file from being relative to the
@ -41,14 +42,14 @@ Notes:
Read the values into a target like this:
component("mycomponent") {
sources = gypi_values.sources
defines = gypi_values.defines
sources = gypi_values.your_file_sources
defines = gypi_values.your_file_defines
}
Sometimes your .gypi file will include paths relative to a different
directory than the current .gn file. In this case, you can rebase them to
be relative to the current directory.
sources = rebase_path(gypi_values.sources, ".",
sources = rebase_path(gypi_values.your_files_sources, ".",
"//path/gypi/input/values/are/relative/to")
This script will tolerate a 'variables' in the toplevel dictionary or not. If
@ -73,6 +74,7 @@ the input will be replaced with "bar":
import gn_helpers
from optparse import OptionParser
import sys
import os.path
def LoadPythonDictionary(path):
file_string = open(path).read()
@ -105,30 +107,6 @@ def LoadPythonDictionary(path):
return file_data
def ReplaceSubstrings(values, search_for, replace_with):
"""Recursively replaces substrings in a value.
Replaces all substrings of the "search_for" with "repace_with" for all
strings occurring in "values". This is done by recursively iterating into
lists as well as the keys and values of dictionaries."""
if isinstance(values, str):
return values.replace(search_for, replace_with)
if isinstance(values, list):
return [ReplaceSubstrings(v, search_for, replace_with) for v in values]
if isinstance(values, dict):
# For dictionaries, do the search for both the key and values.
result = {}
for key, value in values.items():
new_key = ReplaceSubstrings(key, search_for, replace_with)
new_value = ReplaceSubstrings(value, search_for, replace_with)
result[new_key] = new_value
return result
# Assume everything else is unchanged.
return values
def KeepOnly(values, filters):
"""Recursively filters out strings not ending in "f" from "values"""
@ -147,37 +125,41 @@ def KeepOnly(values, filters):
def main():
parser = OptionParser()
parser.add_option("-r", "--replace", action="append",
help="Replaces substrings. If passed a=b, replaces all substrs a with b.")
parser.add_option("-k", "--keep_only", default = [], action="append",
help="Keeps only files ending with the listed strings.")
parser.add_option("--prefix", action="store_true",
help="Prefix variables with base name")
(options, args) = parser.parse_args()
if len(args) != 1:
raise Exception("Need one argument which is the .gypi file to read.")
if len(args) < 1:
raise Exception("Need at least one .gypi file to read.")
data = LoadPythonDictionary(args[0])
if options.replace:
# Do replacements for all specified patterns.
for replace in options.replace:
split = replace.split('=')
# Allow "foo=" to replace with nothing.
if len(split) == 1:
split.append('')
assert len(split) == 2, "Replacement must be of the form 'key=value'."
data = ReplaceSubstrings(data, split[0], split[1])
data = {}
if options.keep_only != []:
data = KeepOnly(data, options.keep_only)
for gypi in args:
gypi_data = LoadPythonDictionary(gypi)
# Sometimes .gypi files use the GYP syntax with percents at the end of the
# variable name (to indicate not to overwrite a previously-defined value):
# 'foo%': 'bar',
# Convert these to regular variables.
for key in data:
if len(key) > 1 and key[len(key) - 1] == '%':
data[key[:-1]] = data[key]
del data[key]
if options.keep_only != []:
gypi_data = KeepOnly(gypi_data, options.keep_only)
# Sometimes .gypi files use the GYP syntax with percents at the end of the
# variable name (to indicate not to overwrite a previously-defined value):
# 'foo%': 'bar',
# Convert these to regular variables.
for key in gypi_data:
if len(key) > 1 and key[len(key) - 1] == '%':
gypi_data[key[:-1]] = gypi_data[key]
del gypi_data[key]
gypi_name = os.path.basename(gypi)[:-len(".gypi")]
for key in gypi_data:
if options.prefix:
# Prefix all variables from this gypi file with the name to disambiguate
data[gypi_name + "_" + key] = gypi_data[key]
elif key in data:
for entry in gypi_data[key]:
data[key].append(entry)
else:
data[key] = gypi_data[key]
print gn_helpers.ToGNString(data)

69
tools/process_gypis.py Executable file
View file

@ -0,0 +1,69 @@
#!/usr/bin/env python
# Copyright 2016, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import gn_helpers
import os.path
import sys
# Given a list of dart package names read in the set of runtime and sdk library
# sources into variables in a gn scope.
def LoadPythonDictionary(path):
file_string = open(path).read()
try:
file_data = eval(file_string, {'__builtins__': None}, None)
except SyntaxError, e:
e.filename = path
raise
except Exception, e:
raise Exception('Unexpected error while reading %s: %s' %
(path, str(e)))
assert isinstance(
file_data, dict), '%s does not eval to a dictionary' % path
return file_data
def main():
dart_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
runtime_dir = os.path.join(dart_root_dir, 'runtime')
runtime_lib_dir = os.path.join(runtime_dir, 'lib')
sdk_lib_dir = os.path.join(dart_root_dir, 'sdk', 'lib')
libs = sys.argv[1:]
data = {}
data['allsources'] = []
for lib in libs:
runtime_path = os.path.join(runtime_lib_dir, lib + '_sources.gypi')
sdk_path = os.path.join(sdk_lib_dir, lib, lib + '_sources.gypi')
runtime_dict = LoadPythonDictionary(runtime_path)
for source in runtime_dict['sources']:
data['allsources'].append(source)
data[lib + '_runtime_sources'] = runtime_dict['sources']
sdk_dict = LoadPythonDictionary(sdk_path)
data[lib + '_sdk_sources'] = sdk_dict['sources']
vm_sources_path = os.path.join(runtime_dir, 'vm', 'vm_sources.gypi')
vm_sources_dict = LoadPythonDictionary(vm_sources_path)
data['vm_sources'] = vm_sources_dict['sources']
platform_sources_base = os.path.join(runtime_dir, 'platform', 'platform_')
platform_headers_dict = LoadPythonDictionary(
platform_sources_base + 'headers.gypi')
platform_sources_dict = LoadPythonDictionary(
platform_sources_base + 'sources.gypi')
data['platform_sources'] = platform_headers_dict[
'sources'] + platform_sources_dict['sources']
bin_io_sources_path = os.path.join(runtime_dir, 'bin', 'io_sources.gypi')
bin_io_sources_dict = LoadPythonDictionary(bin_io_sources_path)
data['bin_io_sources'] = bin_io_sources_dict['sources']
print gn_helpers.ToGNString(data)
return 0
if __name__ == '__main__':
sys.exit(main())