Make list_files.py and list_dart_files.py return absolute paths for GN

This makes doing an SDK build in the Fuchsia tree easier.

The previous attempt at this change failed in gyp builds targeting xcode.
They failed because input lists for actions of the same target (e.g. the
actions for dartanalyzer) are merged and de-duped when the lists contain
relative paths, but merged and *not* de-duped when the lists contain
absolute paths. gyp rejects xcode files that it generated itself
when they contain duplicate entries.

This CL works around that bug, by using absolute paths in the GN build
and keeping relative paths in the gyp build.

R=rmacnak@google.com

Review-Url: https://codereview.chromium.org/2567213002 .
This commit is contained in:
Zachary Anderson 2016-12-13 10:01:28 -08:00
parent 28046fcf7d
commit ccfafee023
18 changed files with 94 additions and 57 deletions

View file

@ -70,12 +70,13 @@ action("create_sdk") {
] ]
sdk_lib_files = exec_script("tools/list_dart_files.py", sdk_lib_files = exec_script("tools/list_dart_files.py",
[ rebase_path("sdk/lib") ], [ "absolute", rebase_path("sdk/lib") ],
"list lines") "list lines")
preamble_files = preamble_files =
exec_script("tools/list_files.py", exec_script("tools/list_files.py",
[ [
"absolute",
"", "",
rebase_path("sdk/lib/_internal/js_runtime/lib/preambles"), rebase_path("sdk/lib/_internal/js_runtime/lib/preambles"),
], ],
@ -83,14 +84,13 @@ action("create_sdk") {
sdk_bin_files = exec_script("tools/list_files.py", sdk_bin_files = exec_script("tools/list_files.py",
[ [
"absolute",
"", "",
rebase_path("sdk/bin"), rebase_path("sdk/bin"),
], ],
"list lines") "list lines")
inputs = rebase_path(sdk_lib_files, "", "sdk/lib") + inputs = sdk_lib_files + preamble_files + sdk_bin_files +
rebase_path(preamble_files, "", "sdk/lib") +
rebase_path(sdk_bin_files, "", "sdk/bin") +
[ [
"sdk/lib/dart_client.platform", "sdk/lib/dart_client.platform",
"sdk/lib/dart_server.platform", "sdk/lib/dart_server.platform",

View file

@ -23,15 +23,15 @@
'inputs': [ 'inputs': [
# Xcode can only handle a certain amount of files in one list # Xcode can only handle a certain amount of files in one list
# (also depending on the length of the path from where you run). # (also depending on the length of the path from where you run).
'<!@(["python", "tools/list_files.py",' '<!@(["python", "tools/list_files.py", "relative", "dart$",'
'"dart$",'
'"sdk/lib"])', '"sdk/lib"])',
'sdk/lib/dart_client.platform', 'sdk/lib/dart_client.platform',
'sdk/lib/dart_server.platform', 'sdk/lib/dart_server.platform',
'sdk/lib/dart_shared.platform', 'sdk/lib/dart_shared.platform',
'<!@(["python", "tools/list_files.py", "", ' '<!@(["python", "tools/list_files.py", "relative", "", '
'"sdk/lib/_internal/js_runtime/lib/preambles"])', '"sdk/lib/_internal/js_runtime/lib/preambles"])',
'<!@(["python", "tools/list_files.py", "", "sdk/bin"])', '<!@(["python", "tools/list_files.py", "relative", "", '
'"sdk/bin"])',
'tools/create_sdk.py', 'tools/create_sdk.py',
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'<(SHARED_INTERMEDIATE_DIR)/dart2js.dart.snapshot', '<(SHARED_INTERMEDIATE_DIR)/dart2js.dart.snapshot',

View file

@ -16,7 +16,6 @@ template("make_third_party_pkg_files_stamp") {
pattern = invoker.pattern pattern = invoker.pattern
} }
path = path path = path
new_base = "//"
output = "$target_gen_dir/third_party_pkg_files_$id.stamp" output = "$target_gen_dir/third_party_pkg_files_$id.stamp"
} }
} }

View file

@ -19,7 +19,7 @@
'action_name': 'make_pkg_files_stamp', 'action_name': 'make_pkg_files_stamp',
'inputs': [ 'inputs': [
'../tools/create_timestamp_file.py', '../tools/create_timestamp_file.py',
'<!@(["python", "../tools/list_dart_files.py", "."])', '<!@(["python", "../tools/list_dart_files.py", "relative", "."])',
'<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_a_k.stamp', '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_a_k.stamp',
'<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_l_r.stamp', '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_l_r.stamp',
'<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_s_z.stamp', '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_s_z.stamp',
@ -36,7 +36,8 @@
'action_name': 'make_third_party_pkg_files_a_k_stamp', 'action_name': 'make_third_party_pkg_files_a_k_stamp',
'inputs': [ 'inputs': [
'../tools/create_timestamp_file.py', '../tools/create_timestamp_file.py',
'<!@(["python", "../tools/list_dart_files.py", "../third_party/pkg", "[a-k].*"])', '<!@(["python", "../tools/list_dart_files.py", "relative", '
'"../third_party/pkg", "[a-k].*"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_a_k.stamp', '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_a_k.stamp',
@ -50,7 +51,8 @@
'action_name': 'make_third_party_pkg_files_l_r_stamp', 'action_name': 'make_third_party_pkg_files_l_r_stamp',
'inputs': [ 'inputs': [
'../tools/create_timestamp_file.py', '../tools/create_timestamp_file.py',
'<!@(["python", "../tools/list_dart_files.py", "../third_party/pkg", "[l-r].*"])', '<!@(["python", "../tools/list_dart_files.py", "relative", '
'"../third_party/pkg", "[l-r].*"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_l_r.stamp', '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_l_r.stamp',
@ -64,7 +66,8 @@
'action_name': 'make_third_party_pkg_files_s_z_stamp', 'action_name': 'make_third_party_pkg_files_s_z_stamp',
'inputs': [ 'inputs': [
'../tools/create_timestamp_file.py', '../tools/create_timestamp_file.py',
'<!@(["python", "../tools/list_dart_files.py", "../third_party/pkg", "[s-z].*"])', '<!@(["python", "../tools/list_dart_files.py", "relative", '
'"../third_party/pkg", "[s-z].*"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_s_z.stamp', '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_s_z.stamp',

View file

@ -411,10 +411,13 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
script = "../../tools/patch_sdk.py" script = "../../tools/patch_sdk.py"
# We list all files which make up the sdk (modulo patches) and get them back # We list all files which make up the sdk (modulo patches) and get them
# as a GN list object. # back as a GN list object.
shared_sdk_sources = exec_script("../../tools/list_dart_files.py", shared_sdk_sources = exec_script("../../tools/list_dart_files.py",
[ "../../sdk/lib" ], [
"absolute",
rebase_path("../../sdk/lib"),
],
"list lines") "list lines")
# We list the `patch_sdk.dart` tool here because the [script] (which is # We list the `patch_sdk.dart` tool here because the [script] (which is
@ -423,8 +426,8 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
"../../tools/patch_sdk.dart", "../../tools/patch_sdk.dart",
] ]
# Files below are not patches, they will not be in [concatenation_files] but # Files below are not patches, they will not be in [concatenation_files]
# the `patch_sdk.dart` script will copy them into the patched sdk. # but the `patch_sdk.dart` script will copy them into the patched sdk.
inputs += [ inputs += [
"../lib/typed_data.dart", "../lib/typed_data.dart",
"../bin/builtin.dart", "../bin/builtin.dart",

View file

@ -1261,7 +1261,8 @@
{ {
'action_name': 'patch_sdk', 'action_name': 'patch_sdk',
'inputs': [ 'inputs': [
'<!@(["python", "../tools/list_files.py",' '"dart$",' '"sdk/lib"])', '<!@(["python", "../tools/list_files.py", "relative",'
'"dart$", "sdk/lib"])',
'../../tools/patch_sdk.py', '../../tools/patch_sdk.py',
'../../tools/patch_sdk.dart', '../../tools/patch_sdk.dart',
# Unlike the other libraries in the SDK, dart:typed_data is not # Unlike the other libraries in the SDK, dart:typed_data is not

View file

@ -5,8 +5,12 @@
"""Tool for listing Dart source files. """Tool for listing Dart source files.
If the first argument is 'relative', the script produces paths relative to the
current working directory. If the first argument is 'absolute', the script
produces absolute paths.
Usage: Usage:
python tools/list_dart_files.py <directory> <root directory pattern> python tools/list_dart_files.py {absolute, relative} <directory> <pattern>
""" """
import os import os
@ -15,11 +19,16 @@ import sys
def main(argv): def main(argv):
directory = argv[1] mode = argv[1]
if mode not in ['absolute', 'relative']:
raise Exception("First argument must be 'absolute' or 'relative'")
directory = argv[2]
if mode in 'absolute' and not os.path.isabs(directory):
directory = os.path.realpath(directory)
pattern = None pattern = None
if len(argv) > 2: if len(argv) > 3:
pattern = re.compile(argv[2]) pattern = re.compile(argv[3])
for root, directories, files in os.walk(directory): for root, directories, files in os.walk(directory):
# We only care about actual source files, not generated code or tests. # We only care about actual source files, not generated code or tests.
@ -34,7 +43,10 @@ def main(argv):
for filename in files: for filename in files:
if filename.endswith('.dart') and not filename.endswith('_test.dart'): if filename.endswith('.dart') and not filename.endswith('_test.dart'):
fullname = os.path.relpath(os.path.join(root, filename)) if mode in 'absolute':
fullname = os.path.join(directory, root, filename)
else:
fullname = os.path.relpath(os.path.join(root, filename))
fullname = fullname.replace(os.sep, '/') fullname = fullname.replace(os.sep, '/')
print fullname print fullname

View file

@ -5,8 +5,12 @@
"""Tool for listing files whose name match a pattern. """Tool for listing files whose name match a pattern.
If the first argument is 'relative', the script produces paths relative to the
current working directory. If the first argument is 'absolute', the script
produces absolute paths.
Usage: Usage:
python tools/list_files.py PATTERN DIRECTORY... python tools/list_files.py {absolute, relative} PATTERN DIRECTORY...
""" """
import os import os
@ -15,13 +19,21 @@ import sys
def main(argv): def main(argv):
pattern = re.compile(argv[1]) mode = argv[1]
for directory in argv[2:]: if mode not in ['absolute', 'relative']:
raise Exception("First argument must be 'absolute' or 'relative'")
pattern = re.compile(argv[2])
for directory in argv[3:]:
if mode in 'absolute' and not os.path.isabs(directory):
directory = os.path.realpath(directory)
for root, directories, files in os.walk(directory): for root, directories, files in os.walk(directory):
if '.git' in directories: if '.git' in directories:
directories.remove('.git') directories.remove('.git')
for filename in files: for filename in files:
fullname = os.path.relpath(os.path.join(root, filename)) if mode in 'absolute':
fullname = os.path.join(directory, root, filename)
else:
fullname = os.path.relpath(os.path.join(root, filename))
fullname = fullname.replace(os.sep, '/') fullname = fullname.replace(os.sep, '/')
if re.search(pattern, fullname): if re.search(pattern, fullname):
print fullname print fullname

View file

@ -20,7 +20,10 @@
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart', '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../runtime/lib", "../../sdk/lib/_internal/dartdoc"])', '<!@(["python", "../../tools/list_files.py", "relative", '
'"\\.dart$", '
'"../../runtime/lib", '
'"../../sdk/lib/_internal/dartdoc"])',
'create_snapshot.dart', 'create_snapshot.dart',
'<(SHARED_INTERMEDIATE_DIR)/dart2js_files.stamp', '<(SHARED_INTERMEDIATE_DIR)/dart2js_files.stamp',
'../../tools/VERSION', '../../tools/VERSION',
@ -51,7 +54,8 @@
'action_name': 'make_dart2js_files_stamp', 'action_name': 'make_dart2js_files_stamp',
'inputs': [ 'inputs': [
'../../tools/create_timestamp_file.py', '../../tools/create_timestamp_file.py',
'<!@(["python", "../../tools/list_files.py", "\\.dart$",' '<!@(["python", "../../tools/list_files.py", "relative", '
'"\\.dart$", '
' "../../pkg/compiler/lib"])', ' "../../pkg/compiler/lib"])',
], ],
'outputs': [ 'outputs': [

View file

@ -7,14 +7,6 @@ _dart_root = rebase_path("..")
template("create_timestamp_file") { template("create_timestamp_file") {
assert(defined(invoker.path), "Must specify 'path'") assert(defined(invoker.path), "Must specify 'path'")
assert(defined(invoker.output), "Must specify 'output'") assert(defined(invoker.output), "Must specify 'output'")
new_base = "."
if (defined(invoker.new_base)) {
new_base = invoker.new_base
}
current_base = "."
if (defined(invoker.current_base)) {
current_base = invoker.current_base
}
path = invoker.path path = invoker.path
output = invoker.output output = invoker.output
action(target_name) { action(target_name) {
@ -23,10 +15,9 @@ template("create_timestamp_file") {
list_args += [ invoker.pattern ] list_args += [ invoker.pattern ]
} }
files = exec_script("$_dart_root/tools/list_dart_files.py", files = exec_script("$_dart_root/tools/list_dart_files.py",
list_args, [ "absolute" ] + list_args,
"list lines") "list lines")
inputs = [ "$_dart_root/tools/list_dart_files.py" ] + inputs = [ "$_dart_root/tools/list_dart_files.py" ] + files
rebase_path(files, new_base, current_base)
outputs = [ outputs = [
output, output,
] ]

View file

@ -14,7 +14,7 @@ group("dartanalyzer") {
} }
analyzer_files = exec_script("../../tools/list_dart_files.py", analyzer_files = exec_script("../../tools/list_dart_files.py",
[ rebase_path("../../pkg/analyzer") ], [ "absolute", rebase_path("../../pkg/analyzer") ],
"list lines") "list lines")
application_snapshot("generate_dartanalyzer_snapshot") { application_snapshot("generate_dartanalyzer_snapshot") {
@ -25,13 +25,13 @@ application_snapshot("generate_dartanalyzer_snapshot") {
] ]
name = "dartanalyzer" name = "dartanalyzer"
cli_files = exec_script("../../tools/list_dart_files.py", cli_files = exec_script("../../tools/list_dart_files.py",
[ rebase_path("../../pkg/analyzer_cli") ], [ "absolute", rebase_path("../../pkg/analyzer_cli") ],
"list lines") "list lines")
inputs = cli_files + analyzer_files inputs = cli_files + analyzer_files
} }
sdk_lib_files = exec_script("../../tools/list_dart_files.py", sdk_lib_files = exec_script("../../tools/list_dart_files.py",
[ rebase_path("../../sdk/lib") ], [ "absolute", rebase_path("../../sdk/lib") ],
"list lines") "list lines")
template("generate_summary") { template("generate_summary") {

View file

@ -16,8 +16,10 @@
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart', '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer_cli"])', '<!@(["python", "../../tools/list_dart_files.py", "relative", '
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer"])', '"../../pkg/analyzer_cli"])',
'<!@(["python", "../../tools/list_dart_files.py", "relative", '
'"../../pkg/analyzer"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/dartanalyzer.dart.snapshot', '<(SHARED_INTERMEDIATE_DIR)/dartanalyzer.dart.snapshot',
@ -33,8 +35,10 @@
'action_name': 'generate_summary_spec', 'action_name': 'generate_summary_spec',
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../sdk/lib"])', '<!@(["python", "../../tools/list_dart_files.py", "relative", '
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer"])', '"../../sdk/lib"])',
'<!@(["python", "../../tools/list_dart_files.py", "relative", '
'"../../pkg/analyzer"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/spec.sum', '<(SHARED_INTERMEDIATE_DIR)/spec.sum',
@ -51,8 +55,10 @@
'action_name': 'generate_summary_strong', 'action_name': 'generate_summary_strong',
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../sdk/lib"])', '<!@(["python", "../../tools/list_dart_files.py", "relative", '
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer"])', '"../../sdk/lib"])',
'<!@(["python", "../../tools/list_dart_files.py", "relative", '
'"../../pkg/analyzer"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/strong.sum', '<(SHARED_INTERMEDIATE_DIR)/strong.sum',

View file

@ -14,6 +14,7 @@ application_snapshot("dartdevc") {
"--help", "--help",
] ]
inputs = exec_script("../../tools/list_dart_files.py", inputs = exec_script("../../tools/list_dart_files.py",
[ rebase_path("../../pkg/dev_compiler/bin") ], [ "absolute",
rebase_path("../../pkg/dev_compiler/bin") ],
"list lines") "list lines")
} }

View file

@ -16,7 +16,8 @@
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart', '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/dev_compiler/bin"])', '<!@(["python", "../../tools/list_dart_files.py", "relative", '
'"../../pkg/dev_compiler/bin"])',
'<(SHARED_INTERMEDIATE_DIR)/pkg_files.stamp', '<(SHARED_INTERMEDIATE_DIR)/pkg_files.stamp',
], ],
'outputs': [ 'outputs': [

View file

@ -8,6 +8,7 @@ application_snapshot("dartdoc") {
main_dart = "../../third_party/pkg/dartdoc/bin/dartdoc.dart" main_dart = "../../third_party/pkg/dartdoc/bin/dartdoc.dart"
training_args = [ "--help" ] training_args = [ "--help" ]
inputs = exec_script("../../tools/list_dart_files.py", inputs = exec_script("../../tools/list_dart_files.py",
[ rebase_path("../../third_party/pkg/dartdoc") ], [ "absolute",
rebase_path("../../third_party/pkg/dartdoc") ],
"list lines") "list lines")
} }

View file

@ -16,7 +16,8 @@
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart', '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../third_party/pkg/dartdoc"])', '<!@(["python", "../../tools/list_dart_files.py", "relative", '
'"../../third_party/pkg/dartdoc"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/dartdoc.dart.snapshot', '<(SHARED_INTERMEDIATE_DIR)/dartdoc.dart.snapshot',
@ -31,4 +32,4 @@
], ],
}, },
], ],
} }

View file

@ -9,6 +9,7 @@ application_snapshot("dartfmt") {
training_args = [ "--help" ] training_args = [ "--help" ]
inputs = inputs =
exec_script("../../tools/list_dart_files.py", exec_script("../../tools/list_dart_files.py",
[ rebase_path("../../third_party/pkg_tested/dart_style") ], [ "absolute",
rebase_path("../../third_party/pkg_tested/dart_style") ],
"list lines") "list lines")
} }

View file

@ -16,7 +16,8 @@
'inputs': [ 'inputs': [
'<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
'../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart', '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
'<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../third_party/pkg_tested/dart_style"])', '<!@(["python", "../../tools/list_dart_files.py", "relative", '
'"../../third_party/pkg_tested/dart_style"])',
], ],
'outputs': [ 'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/dartfmt.dart.snapshot', '<(SHARED_INTERMEDIATE_DIR)/dartfmt.dart.snapshot',