languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-07-05 09:20:04 +00:00
Code Issues Projects Releases Wiki Activity

[gn] Deprecate copy_trees()

Browse Source 
Part of https://github.com/flutter/flutter/issues/144430

Previously, this code did backflips to accomplish two goals
1. Collect the paths of all files that would be copied from one place to another after applying regex filters so that they could be listed as the "inputs" to a GN action.
2. Arrange so that the python script doing the above would only be invoked once during GN to reduce the cost of calling out to python.

However, this is exactly the use-case for the "depfile" parameter to a GN action. Instead of running the script during GN to populate the "inputs" list, when we run the copy_tree.py script, we can instead ask it to generate a depfile to collect all the input files that were actually copied after applying the regex filter. Using that, we don't have to run the python script at all during GN.

TEST=it builds

Change-Id: I41a251ce4659119cdc3997bb2d6fc7ee0831bb6d
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/356481
Reviewed-by: Siva Annamalai <asiva@google.com>
Commit-Queue: Zach Anderson <zra@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>
This commit is contained in:
Zach Anderson 2024-03-08 20:47:47 +00:00 committed by Commit Queue
parent 9efa956749
commit f60d32167f
5 changed files with 88 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
build/dart/copy_tree.gni
View File

@ -10,13 +10,11 @@ _dart_root = rebase_path("../..")
# Optional parameters: # Optional parameters:
# exclude - A comma separated list that is passed to shutil.ignore_patterns() # exclude - A comma separated list that is passed to shutil.ignore_patterns()
# in tools/copy_tree.py. # in tools/copy_tree.py.
template("_copy_tree") { template("copy_tree") {
assert(defined(invoker.source), "copy_tree must define 'source'") assert(defined(invoker.source), "copy_tree must define 'source'")
assert(defined(invoker.dest), "copy_tree must define 'dest'") assert(defined(invoker.dest), "copy_tree must define 'dest'")
assert(defined(invoker.inputs), "copy_tree must define 'inputs'")
source = invoker.source source = invoker.source
dest = invoker.dest dest = invoker.dest
inputs = invoker.inputs
action(target_name) { action(target_name) {
if (defined(invoker.visibility)) { if (defined(invoker.visibility)) {
visibility = invoker.visibility visibility = invoker.visibility
@ -27,11 +25,18 @@ template("_copy_tree") {
deps += invoker.deps deps += invoker.deps
} }
depfile = "$target_gen_dir/$target_name.d"
stampfile = "$target_gen_dir/$target_name.stamp"
common_args = [ common_args = [
"--from", "--from",
rebase_path(source), rebase_path(source),
"--to", "--to",
rebase_path(dest), rebase_path(dest),
"--depfile",
rebase_path(depfile),
"--stamp",
rebase_path(stampfile),
] ]
if (defined(invoker.exclude)) { if (defined(invoker.exclude)) {
common_args += [ common_args += [
@ -40,33 +45,14 @@ template("_copy_tree") {
] ]
} }
relative_files = rebase_path(inputs, rebase_path(source)) outputs = [ stampfile ]
output_files = []
foreach(input, relative_files) {
output_files += [ "$dest/$input" ]
}
outputs = output_files
script = "$_dart_root/tools/copy_tree.py" script = "$_dart_root/tools/copy_tree.py"
args = common_args args = common_args
} }
} }
# copy_trees() arranges to invoke copy_tree.py only once to gather the list of # DEPRECATED: This can be removed after the uses in the flutter/engine tree
# input source files for every _copy_tree() target. It takes a list of scopes as # are migrated to use copy_tree().
# a parameter. The scopes should contain the following mappings.
#
# target: The target name for the _copy_tree() target.
# visibility: The visibility for the _copy_tree() target.
# source: The source directory relative to this directory.
# dest: The destination directory for the _copy_tree() target.
# deps: Any deps needed for the _copy_tree() target.
# ignore_patterns: Patterns to ignore when walking the directory tree.
# This should be '{}' if nothing should be ignored.
#
# copy_trees() will then make sure each invocation of _copy_tree() has the
# correct 'inputs' parameter
template("copy_trees") { template("copy_trees") {
assert(defined(invoker.sources), "$target_name must define 'source'") assert(defined(invoker.sources), "$target_name must define 'source'")
sources = invoker.sources sources = invoker.sources
@ -78,21 +64,12 @@ template("copy_trees") {
] ]
} }
# Evaluate script output as GN, producing a scope containing a single value
# "sources"
copy_tree_inputs_scope = exec_script("$_dart_root/tools/copy_tree.py",
[ "--gn" ] + copy_tree_source_paths,
"scope")
# A list of lists of input source files for copy_tree. # A list of lists of input source files for copy_tree.
copy_tree_inputs = copy_tree_inputs_scope.sources
copy_tree_inputs_index = 0
foreach(copy_tree_spec, sources) { foreach(copy_tree_spec, sources) {
_copy_tree(copy_tree_spec.target) { copy_tree(copy_tree_spec.target) {
visibility = copy_tree_spec.visibility visibility = copy_tree_spec.visibility
source = copy_tree_spec.source source = copy_tree_spec.source
dest = copy_tree_spec.dest dest = copy_tree_spec.dest
inputs = copy_tree_inputs[copy_tree_inputs_index]
if (defined(copy_tree_spec.deps)) { if (defined(copy_tree_spec.deps)) {
deps = copy_tree_spec.deps deps = copy_tree_spec.deps
} }
@ -100,6 +77,5 @@ template("copy_trees") {
exclude = copy_tree_spec.ignore_patterns exclude = copy_tree_spec.ignore_patterns
} }
} }
copy_tree_inputs_index = copy_tree_inputs_index + 1
} }
} }

38
runtime/observatory/BUILD.gn
View File

@ -74,39 +74,25 @@ if (!is_debug) {
observatory_ignore_patterns += [ "*.map" ] observatory_ignore_patterns += [ "*.map" ]
} }
# The ignore_patterns entry in the scopes accepted by copy_trees() is a # The ignore_patterns entry in the scopes accepted by copy_tree() is a
# string of comma delimited patterns. # string of comma delimited patterns.
observatory_ignore_string = "\$sdk" observatory_ignore_string = "\$sdk"
foreach(pattern, observatory_ignore_patterns) { foreach(pattern, observatory_ignore_patterns) {
observatory_ignore_string = "$observatory_ignore_string,$pattern" observatory_ignore_string = "$observatory_ignore_string,$pattern"
} }
copy_tree_specs = [] copy_tree("copy_web_package") {
visibility = [ ":deploy_observatory" ]
source = "web"
dest = "$target_out_dir/observatory/deployed/web"
exclude = observatory_ignore_string
}
copy_tree_specs += [ copy_tree("copy_observatory_package") {
{ visibility = [ ":deploy_observatory" ]
target = "copy_web_package" source = "lib"
visibility = [ ":deploy_observatory" ] dest = "$target_out_dir/observatory/deployed/web/packages/observatory"
source = "web" exclude = observatory_ignore_string
dest = "$target_out_dir/observatory/deployed/web"
ignore_patterns = observatory_ignore_string
},
]
copy_tree_specs += [
{
target = "copy_observatory_package"
visibility = [ ":deploy_observatory" ]
source = "lib"
dest = "$target_out_dir/observatory/deployed/web/packages/observatory"
ignore_patterns = observatory_ignore_string
},
]
# This is not a rule, rather, it generates rules with names of the form:
# "copy_$package_package" for the packages in observatory_pub_packages.
copy_trees("copy_observatory_packages") {
sources = copy_tree_specs
} }
copy("copy_main_dart_js") { copy("copy_main_dart_js") {

76
sdk/BUILD.gn
View File

@ -212,66 +212,44 @@ _full_sdk_libraries = [
# ] # ]
_platform_sdk_libraries = _full_sdk_libraries _platform_sdk_libraries = _full_sdk_libraries
# From here down to the copy_trees() invocation, we collect all the information
# about trees that need to be copied in the list of scopes, copy_tree_specs.
copy_tree_specs = []
# This rule copies dartdoc templates to # This rule copies dartdoc templates to
# bin/resources/dartdoc/templates # bin/resources/dartdoc/templates
copy_tree_specs += [ copy_tree("copy_dartdoc_templates") {
{ visibility = [ ":copy_dartdoc_files" ]
target = "copy_dartdoc_templates" source = "../third_party/pkg/dartdoc/lib/templates"
visibility = [ ":copy_dartdoc_files" ] dest = "$root_out_dir/$dart_sdk_output/bin/resources/dartdoc/templates"
source = "../third_party/pkg/dartdoc/lib/templates" exclude = "{}"
dest = "$root_out_dir/$dart_sdk_output/bin/resources/dartdoc/templates" }
ignore_patterns = "{}"
},
]
# This rule copies dartdoc resources to # This rule copies dartdoc resources to
# bin/resources/dartdoc/resources # bin/resources/dartdoc/resources
copy_tree_specs += [ copy_tree("copy_dartdoc_resources") {
{ visibility = [ ":copy_dartdoc_files" ]
target = "copy_dartdoc_resources" source = "../third_party/pkg/dartdoc/lib/resources"
visibility = [ ":copy_dartdoc_files" ] dest = "$root_out_dir/$dart_sdk_output/bin/resources/dartdoc/resources"
source = "../third_party/pkg/dartdoc/lib/resources" exclude = "{}"
dest = "$root_out_dir/$dart_sdk_output/bin/resources/dartdoc/resources" }
ignore_patterns = "{}"
},
]
# This rule copies the pre-built DevTools application to # This rule copies the pre-built DevTools application to
# bin/resources/devtools/ # bin/resources/devtools/
copy_tree_specs += [ copy_tree("copy_prebuilt_devtools") {
{ visibility = [ ":create_common_sdk" ]
target = "copy_prebuilt_devtools" source = "../third_party/devtools/web"
visibility = [ ":create_common_sdk" ] dest = "$root_out_dir/$dart_sdk_output/bin/resources/devtools"
source = "../third_party/devtools/web" exclude = "{}"
dest = "$root_out_dir/$dart_sdk_output/bin/resources/devtools" }
ignore_patterns = "{}"
},
]
# This loop generates rules to copy libraries to lib/ # This loop generates rules to copy libraries to lib/
foreach(library, _full_sdk_libraries) { foreach(library, _full_sdk_libraries) {
copy_tree_specs += [ copy_tree("copy_${library}_library") {
{ visibility = [
target = "copy_${library}_library" ":copy_full_sdk_libraries",
visibility = [ ":copy_platform_sdk_libraries",
":copy_full_sdk_libraries", ]
":copy_platform_sdk_libraries", source = "lib/$library"
] dest = "$root_out_dir/$dart_sdk_output/lib/$library"
source = "lib/$library" exclude = "*.svn,doc,*.py,*.gypi,*.sh,.gitignore"
dest = "$root_out_dir/$dart_sdk_output/lib/$library" }
ignore_patterns = "*.svn,doc,*.py,*.gypi,*.sh,.gitignore"
},
]
}
# This generates targets for everything in copy_tree_specs. The targets have the
# same name as the "target" fields in the scopes of copy_tree_specs.
copy_trees("copy_trees") {
sources = copy_tree_specs
} }
_has_dot_sym = !is_win && rebase_path(".") == rebase_path("//sdk") _has_dot_sym = !is_win && rebase_path(".") == rebase_path("//sdk")

96
tools/copy_tree.py
View File

@ -16,6 +16,10 @@ def ParseArgs(args):
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='A script to copy a file tree somewhere') description='A script to copy a file tree somewhere')
parser.add_argument('--depfile',
'-d',
type=str,
help='Path to a depfile to write when copying.')
parser.add_argument( parser.add_argument(
'--exclude_patterns', '--exclude_patterns',
'-e', '-e',
@ -24,33 +28,16 @@ def ParseArgs(args):
parser.add_argument( parser.add_argument(
'--from', '-f', dest="copy_from", type=str, help='Source directory') '--from', '-f', dest="copy_from", type=str, help='Source directory')
parser.add_argument( parser.add_argument(
'--gn', '--stamp',
'-g', '-s',
dest='gn',
default=False,
action='store_true',
help='Output for GN for multiple sources, but do not copy anything.')
parser.add_argument(
'gn_paths',
metavar='name path ignore_pattern',
type=str, type=str,
nargs='*', help='The path to a stamp file to output when finished.')
default=None,
help='When --gn is given, the specification of source paths to list.')
parser.add_argument('--to', '-t', type=str, help='Destination directory') parser.add_argument('--to', '-t', type=str, help='Destination directory')
return parser.parse_args(args) return parser.parse_args(args)
def ValidateArgs(args): def ValidateArgs(args):
if args.gn:
if args.exclude_patterns or args.copy_from or args.to:
print("--gn mode does not accept other switches")
return False
if not args.gn_paths:
print("--gn mode requires a list of source specifications")
return False
return True
if not args.copy_from or not os.path.isdir(args.copy_from): if not args.copy_from or not os.path.isdir(args.copy_from):
print("--from argument must refer to a directory") print("--from argument must refer to a directory")
return False return False
@ -60,7 +47,10 @@ def ValidateArgs(args):
return True return True
# Returns a list of the files under 'src' that were copied.
def CopyTree(src, dst, ignore=None): def CopyTree(src, dst, ignore=None):
copied_files = []
# Recursive helper method to collect errors but keep processing. # Recursive helper method to collect errors but keep processing.
def copy_tree(src, dst, ignore, errors): def copy_tree(src, dst, ignore, errors):
names = os.listdir(src) names = os.listdir(src)
@ -80,6 +70,7 @@ def CopyTree(src, dst, ignore=None):
if os.path.isdir(srcname): if os.path.isdir(srcname):
copy_tree(srcname, dstname, ignore, errors) copy_tree(srcname, dstname, ignore, errors)
else: else:
copied_files.append(srcname)
shutil.copy(srcname, dstname) shutil.copy(srcname, dstname)
except (IOError, os.error) as why: except (IOError, os.error) as why:
errors.append((srcname, dstname, str(why))) errors.append((srcname, dstname, str(why)))
@ -106,45 +97,18 @@ def CopyTree(src, dst, ignore=None):
msg = '\n'.join(parts) msg = '\n'.join(parts)
raise RuntimeError(msg) raise RuntimeError(msg)
return copied_files
def ListTree(src, ignore=None):
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
srcnames = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
if os.path.isdir(srcname):
srcnames.extend(ListTree(srcname, ignore))
else:
srcnames.append(srcname)
return srcnames
# source_dirs is organized such that sources_dirs[n] is the path for the source def WriteDepfile(depfile, stamp, dep_list):
# directory, and source_dirs[n+1] is a list of ignore patterns. os.makedirs(os.path.dirname(depfile), exist_ok=True)
def SourcesToGN(source_dirs): # Paths in the depfile must be relative to the root build output directory,
if len(source_dirs) % 2 != 0: # which is the cwd that ninja invokes the script from.
print("--gn list length should be a multiple of 2.") cwd = os.getcwd()
return False relstamp = os.path.relpath(stamp, cwd)
data = [] reldep_list = [os.path.relpath(d, cwd) for d in dep_list]
for i in range(0, len(source_dirs), 2): with open(depfile, 'w') as f:
path = source_dirs[i] print("{0}: {1}".format(relstamp, " ".join(reldep_list)), file=f)
ignores = source_dirs[i + 1]
if ignores in ["{}"]:
sources = ListTree(path)
else:
patterns = ignores.split(',')
sources = ListTree(path, ignore=shutil.ignore_patterns(*patterns))
data.append(sources)
scope_data = {"sources": data}
print(gn_helpers.ToGNString(scope_data))
return True
def Main(argv): def Main(argv):
@ -152,16 +116,20 @@ def Main(argv):
if not ValidateArgs(args): if not ValidateArgs(args):
return -1 return -1
if args.gn:
SourcesToGN(args.gn_paths)
return 0
if args.exclude_patterns == None: if args.exclude_patterns == None:
CopyTree(args.copy_from, args.to) copied_files = CopyTree(args.copy_from, args.to)
else: else:
patterns = args.exclude_patterns.split(',') patterns = args.exclude_patterns.split(',')
CopyTree( copied_files = CopyTree(args.copy_from,
args.copy_from, args.to, ignore=shutil.ignore_patterns(*patterns)) args.to,
ignore=shutil.ignore_patterns(*patterns))
if args.depfile and args.stamp:
WriteDepfile(args.depfile, args.stamp, copied_files)
if args.stamp:
open(args.stamp, 'w').close()
return 0 return 0

15
utils/dartdev/BUILD.gn
View File

@ -37,14 +37,9 @@ application_snapshot("generate_dartdev_snapshot") {
output = "$root_gen_dir/dartdev.dart.snapshot" output = "$root_gen_dir/dartdev.dart.snapshot"
} }
copy_trees("copy_prebuilt_devtools") { copy_tree("copy_prebuilt_devtools") {
sources = [ visibility = [ ":dartdev" ]
{ source = "../../third_party/devtools/web"
target = "copy_prebuilt_devtools" dest = "$root_out_dir/devtools"
visibility = [ ":dartdev" ] exclude = "{}"
source = "../../third_party/devtools/web"
dest = "$root_out_dir/devtools"
ignore_patterns = "{}"
},
]
} }