From ccfafee0239068d5c98017ae5e14ccef29c835c4 Mon Sep 17 00:00:00 2001
From: Zachary Anderson <zra@google.com>
Date: Tue, 13 Dec 2016 10:01:28 -0800
Subject: [PATCH] Make list_files.py and list_dart_files.py return absolute
 paths for GN

This makes doing an SDK build in the Fuchsia tree easier.

The previous attempt at this change failed in gyp builds targeting xcode.
They failed because input lists for actions of the same target (e.g. the
actions for dartanalyzer) are merged and de-duped when the lists contain
relative paths, but merged and *not* de-duped when the lists contain
absolute paths. gyp rejects xcode files that it generated itself
when they contain duplicate entries.

This CL works around that bug, by using absolute paths in the GN build
and keeping relative paths in the gyp build.

R=rmacnak@google.com

Review-Url: https://codereview.chromium.org/2567213002 .
---
 BUILD.gn                            |  8 ++++----
 create_sdk.gyp                      |  8 ++++----
 pkg/BUILD.gn                        |  1 -
 pkg/pkg_files.gyp                   | 11 +++++++----
 runtime/vm/BUILD.gn                 | 13 ++++++++-----
 runtime/vm/vm.gypi                  |  3 ++-
 tools/list_dart_files.py            | 22 +++++++++++++++++-----
 tools/list_files.py                 | 20 ++++++++++++++++----
 utils/compiler/compiler.gyp         |  8 ++++++--
 utils/create_timestamp.gni          | 13 ++-----------
 utils/dartanalyzer/BUILD.gn         |  6 +++---
 utils/dartanalyzer/dartanalyzer.gyp | 18 ++++++++++++------
 utils/dartdevc/BUILD.gn             |  3 ++-
 utils/dartdevc/dartdevc.gyp         |  3 ++-
 utils/dartdoc/BUILD.gn              |  3 ++-
 utils/dartdoc/dartdoc.gyp           |  5 +++--
 utils/dartfmt/BUILD.gn              |  3 ++-
 utils/dartfmt/dartfmt.gyp           |  3 ++-
 18 files changed, 94 insertions(+), 57 deletions(-)

diff --git a/BUILD.gn b/BUILD.gn
index a027a2e6150..7d1f7fd8764 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -70,12 +70,13 @@ action("create_sdk") {
   ]
 
   sdk_lib_files = exec_script("tools/list_dart_files.py",
-                              [ rebase_path("sdk/lib") ],
+                              [ "absolute", rebase_path("sdk/lib") ],
                               "list lines")
 
   preamble_files =
       exec_script("tools/list_files.py",
                   [
+                    "absolute",
                     "",
                     rebase_path("sdk/lib/_internal/js_runtime/lib/preambles"),
                   ],
@@ -83,14 +84,13 @@ action("create_sdk") {
 
   sdk_bin_files = exec_script("tools/list_files.py",
                               [
+                                "absolute",
                                 "",
                                 rebase_path("sdk/bin"),
                               ],
                               "list lines")
 
-  inputs = rebase_path(sdk_lib_files, "", "sdk/lib") +
-           rebase_path(preamble_files, "", "sdk/lib") +
-           rebase_path(sdk_bin_files, "", "sdk/bin") +
+  inputs = sdk_lib_files + preamble_files + sdk_bin_files +
            [
              "sdk/lib/dart_client.platform",
              "sdk/lib/dart_server.platform",
diff --git a/create_sdk.gyp b/create_sdk.gyp
index b1b9361ddda..9e41ef91ee6 100644
--- a/create_sdk.gyp
+++ b/create_sdk.gyp
@@ -23,15 +23,15 @@
           'inputs': [
             # Xcode can only handle a certain amount of files in one list
             # (also depending on the length of the path from where you run).
-            '<!@(["python", "tools/list_files.py",'
-                '"dart$",'
+            '<!@(["python", "tools/list_files.py", "relative", "dart$",'
                 '"sdk/lib"])',
             'sdk/lib/dart_client.platform',
             'sdk/lib/dart_server.platform',
             'sdk/lib/dart_shared.platform',
-            '<!@(["python", "tools/list_files.py", "", '
+            '<!@(["python", "tools/list_files.py", "relative", "", '
                 '"sdk/lib/_internal/js_runtime/lib/preambles"])',
-            '<!@(["python", "tools/list_files.py", "", "sdk/bin"])',
+            '<!@(["python", "tools/list_files.py", "relative",  "", '
+                '"sdk/bin"])',
             'tools/create_sdk.py',
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
             '<(SHARED_INTERMEDIATE_DIR)/dart2js.dart.snapshot',
diff --git a/pkg/BUILD.gn b/pkg/BUILD.gn
index e1ce11d2e4e..bd7eb4ca87a 100644
--- a/pkg/BUILD.gn
+++ b/pkg/BUILD.gn
@@ -16,7 +16,6 @@ template("make_third_party_pkg_files_stamp") {
       pattern = invoker.pattern
     }
     path = path
-    new_base = "//"
     output = "$target_gen_dir/third_party_pkg_files_$id.stamp"
   }
 }
diff --git a/pkg/pkg_files.gyp b/pkg/pkg_files.gyp
index 55dd28467b8..9e41a67fe64 100644
--- a/pkg/pkg_files.gyp
+++ b/pkg/pkg_files.gyp
@@ -19,7 +19,7 @@
           'action_name': 'make_pkg_files_stamp',
           'inputs': [
             '../tools/create_timestamp_file.py',
-            '<!@(["python", "../tools/list_dart_files.py", "."])',
+            '<!@(["python", "../tools/list_dart_files.py", "relative", "."])',
             '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_a_k.stamp',
             '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_l_r.stamp',
             '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_s_z.stamp',
@@ -36,7 +36,8 @@
           'action_name': 'make_third_party_pkg_files_a_k_stamp',
           'inputs': [
             '../tools/create_timestamp_file.py',
-            '<!@(["python", "../tools/list_dart_files.py", "../third_party/pkg", "[a-k].*"])',
+            '<!@(["python", "../tools/list_dart_files.py", "relative", '
+                '"../third_party/pkg", "[a-k].*"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_a_k.stamp',
@@ -50,7 +51,8 @@
           'action_name': 'make_third_party_pkg_files_l_r_stamp',
           'inputs': [
             '../tools/create_timestamp_file.py',
-            '<!@(["python", "../tools/list_dart_files.py", "../third_party/pkg", "[l-r].*"])',
+            '<!@(["python", "../tools/list_dart_files.py", "relative", '
+                '"../third_party/pkg", "[l-r].*"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_l_r.stamp',
@@ -64,7 +66,8 @@
           'action_name': 'make_third_party_pkg_files_s_z_stamp',
           'inputs': [
             '../tools/create_timestamp_file.py',
-            '<!@(["python", "../tools/list_dart_files.py", "../third_party/pkg", "[s-z].*"])',
+            '<!@(["python", "../tools/list_dart_files.py", "relative", '
+                '"../third_party/pkg", "[s-z].*"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/third_party_pkg_files_s_z.stamp',
diff --git a/runtime/vm/BUILD.gn b/runtime/vm/BUILD.gn
index 9bbc3a49538..bf3a4537b27 100644
--- a/runtime/vm/BUILD.gn
+++ b/runtime/vm/BUILD.gn
@@ -411,10 +411,13 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
 
       script = "../../tools/patch_sdk.py"
 
-      # We list all files which make up the sdk (modulo patches) and get them back
-      # as a GN list object.
+      # We list all files which make up the sdk (modulo patches) and get them
+      # back as a GN list object.
       shared_sdk_sources = exec_script("../../tools/list_dart_files.py",
-                                       [ "../../sdk/lib" ],
+                                       [
+                                         "absolute",
+                                         rebase_path("../../sdk/lib"),
+                                       ],
                                        "list lines")
 
       # We list the `patch_sdk.dart` tool here because the [script] (which is
@@ -423,8 +426,8 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
         "../../tools/patch_sdk.dart",
       ]
 
-      # Files below are not patches, they will not be in [concatenation_files] but
-      # the `patch_sdk.dart` script will copy them into the patched sdk.
+      # Files below are not patches, they will not be in [concatenation_files]
+      # but the `patch_sdk.dart` script will copy them into the patched sdk.
       inputs += [
         "../lib/typed_data.dart",
         "../bin/builtin.dart",
diff --git a/runtime/vm/vm.gypi b/runtime/vm/vm.gypi
index a072f6530ad..3da329123e9 100644
--- a/runtime/vm/vm.gypi
+++ b/runtime/vm/vm.gypi
@@ -1261,7 +1261,8 @@
         {
           'action_name': 'patch_sdk',
           'inputs': [
-            '<!@(["python", "../tools/list_files.py",' '"dart$",' '"sdk/lib"])',
+            '<!@(["python", "../tools/list_files.py", "relative",'
+                  '"dart$", "sdk/lib"])',
             '../../tools/patch_sdk.py',
             '../../tools/patch_sdk.dart',
             # Unlike the other libraries in the SDK, dart:typed_data is not
diff --git a/tools/list_dart_files.py b/tools/list_dart_files.py
index 60d2295652e..7a9ee8ab6c0 100755
--- a/tools/list_dart_files.py
+++ b/tools/list_dart_files.py
@@ -5,8 +5,12 @@
 
 """Tool for listing Dart source files.
 
+If the first argument is 'relative', the script produces paths relative to the
+current working directory. If the first argument is 'absolute', the script
+produces absolute paths.
+
 Usage:
-  python tools/list_dart_files.py <directory> <root directory pattern>
+  python tools/list_dart_files.py {absolute, relative} <directory> <pattern>
 """
 
 import os
@@ -15,11 +19,16 @@ import sys
 
 
 def main(argv):
-  directory = argv[1]
+  mode = argv[1]
+  if mode not in ['absolute', 'relative']:
+    raise Exception("First argument must be 'absolute' or 'relative'")
+  directory = argv[2]
+  if mode in 'absolute' and not os.path.isabs(directory):
+    directory = os.path.realpath(directory)
 
   pattern = None
-  if len(argv) > 2:
-    pattern = re.compile(argv[2])
+  if len(argv) > 3:
+    pattern = re.compile(argv[3])
 
   for root, directories, files in os.walk(directory):
     # We only care about actual source files, not generated code or tests.
@@ -34,7 +43,10 @@ def main(argv):
 
     for filename in files:
       if filename.endswith('.dart') and not filename.endswith('_test.dart'):
-        fullname = os.path.relpath(os.path.join(root, filename))
+        if mode in 'absolute':
+          fullname = os.path.join(directory, root, filename)
+        else:
+          fullname = os.path.relpath(os.path.join(root, filename))
         fullname = fullname.replace(os.sep, '/')
         print fullname
 
diff --git a/tools/list_files.py b/tools/list_files.py
index ad277e853f0..b63d6ab0e32 100755
--- a/tools/list_files.py
+++ b/tools/list_files.py
@@ -5,8 +5,12 @@
 
 """Tool for listing files whose name match a pattern.
 
+If the first argument is 'relative', the script produces paths relative to the
+current working directory. If the first argument is 'absolute', the script
+produces absolute paths.
+
 Usage:
-  python tools/list_files.py PATTERN DIRECTORY...
+  python tools/list_files.py {absolute, relative} PATTERN DIRECTORY...
 """
 
 import os
@@ -15,13 +19,21 @@ import sys
 
 
 def main(argv):
-  pattern = re.compile(argv[1])
-  for directory in argv[2:]:
+  mode = argv[1]
+  if mode not in ['absolute', 'relative']:
+    raise Exception("First argument must be 'absolute' or 'relative'")
+  pattern = re.compile(argv[2])
+  for directory in argv[3:]:
+    if mode in 'absolute' and not os.path.isabs(directory):
+      directory = os.path.realpath(directory)
     for root, directories, files in os.walk(directory):
       if '.git' in directories:
         directories.remove('.git')
       for filename in files:
-        fullname = os.path.relpath(os.path.join(root, filename))
+        if mode in 'absolute':
+          fullname = os.path.join(directory, root, filename)
+        else:
+          fullname = os.path.relpath(os.path.join(root, filename))
         fullname = fullname.replace(os.sep, '/')
         if re.search(pattern, fullname):
           print fullname
diff --git a/utils/compiler/compiler.gyp b/utils/compiler/compiler.gyp
index 63c4daeb36d..272dd0aa137 100644
--- a/utils/compiler/compiler.gyp
+++ b/utils/compiler/compiler.gyp
@@ -20,7 +20,10 @@
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
             '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../runtime/lib", "../../sdk/lib/_internal/dartdoc"])',
+            '<!@(["python", "../../tools/list_files.py",  "relative", '
+                '"\\.dart$", '
+                '"../../runtime/lib", '
+                '"../../sdk/lib/_internal/dartdoc"])',
             'create_snapshot.dart',
             '<(SHARED_INTERMEDIATE_DIR)/dart2js_files.stamp',
             '../../tools/VERSION',
@@ -51,7 +54,8 @@
           'action_name': 'make_dart2js_files_stamp',
           'inputs': [
             '../../tools/create_timestamp_file.py',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$",'
+            '<!@(["python", "../../tools/list_files.py", "relative", '
+                '"\\.dart$", '
                 ' "../../pkg/compiler/lib"])',
           ],
           'outputs': [
diff --git a/utils/create_timestamp.gni b/utils/create_timestamp.gni
index c579da1e5e3..75ae4cda7a5 100644
--- a/utils/create_timestamp.gni
+++ b/utils/create_timestamp.gni
@@ -7,14 +7,6 @@ _dart_root = rebase_path("..")
 template("create_timestamp_file") {
   assert(defined(invoker.path), "Must specify 'path'")
   assert(defined(invoker.output), "Must specify 'output'")
-  new_base = "."
-  if (defined(invoker.new_base)) {
-    new_base = invoker.new_base
-  }
-  current_base = "."
-  if (defined(invoker.current_base)) {
-    current_base = invoker.current_base
-  }
   path = invoker.path
   output = invoker.output
   action(target_name) {
@@ -23,10 +15,9 @@ template("create_timestamp_file") {
       list_args += [ invoker.pattern ]
     }
     files = exec_script("$_dart_root/tools/list_dart_files.py",
-                        list_args,
+                        [ "absolute" ] + list_args,
                         "list lines")
-    inputs = [ "$_dart_root/tools/list_dart_files.py" ] +
-             rebase_path(files, new_base, current_base)
+    inputs = [ "$_dart_root/tools/list_dart_files.py" ] + files
     outputs = [
       output,
     ]
diff --git a/utils/dartanalyzer/BUILD.gn b/utils/dartanalyzer/BUILD.gn
index dbefeecc342..7ea889e05c3 100644
--- a/utils/dartanalyzer/BUILD.gn
+++ b/utils/dartanalyzer/BUILD.gn
@@ -14,7 +14,7 @@ group("dartanalyzer") {
 }
 
 analyzer_files = exec_script("../../tools/list_dart_files.py",
-                             [ rebase_path("../../pkg/analyzer") ],
+                             [ "absolute", rebase_path("../../pkg/analyzer") ],
                              "list lines")
 
 application_snapshot("generate_dartanalyzer_snapshot") {
@@ -25,13 +25,13 @@ application_snapshot("generate_dartanalyzer_snapshot") {
   ]
   name = "dartanalyzer"
   cli_files = exec_script("../../tools/list_dart_files.py",
-                          [ rebase_path("../../pkg/analyzer_cli") ],
+                          [ "absolute", rebase_path("../../pkg/analyzer_cli") ],
                           "list lines")
   inputs = cli_files + analyzer_files
 }
 
 sdk_lib_files = exec_script("../../tools/list_dart_files.py",
-                            [ rebase_path("../../sdk/lib") ],
+                            [ "absolute", rebase_path("../../sdk/lib") ],
                             "list lines")
 
 template("generate_summary") {
diff --git a/utils/dartanalyzer/dartanalyzer.gyp b/utils/dartanalyzer/dartanalyzer.gyp
index daf226cfb84..282d3042c78 100644
--- a/utils/dartanalyzer/dartanalyzer.gyp
+++ b/utils/dartanalyzer/dartanalyzer.gyp
@@ -16,8 +16,10 @@
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
             '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer_cli"])',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../pkg/analyzer_cli"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../pkg/analyzer"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/dartanalyzer.dart.snapshot',
@@ -33,8 +35,10 @@
           'action_name': 'generate_summary_spec',
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../sdk/lib"])',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../sdk/lib"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../pkg/analyzer"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/spec.sum',
@@ -51,8 +55,10 @@
           'action_name': 'generate_summary_strong',
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../sdk/lib"])',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/analyzer"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../sdk/lib"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../pkg/analyzer"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/strong.sum',
diff --git a/utils/dartdevc/BUILD.gn b/utils/dartdevc/BUILD.gn
index 0c934233c8a..ba51ab4eed2 100644
--- a/utils/dartdevc/BUILD.gn
+++ b/utils/dartdevc/BUILD.gn
@@ -14,6 +14,7 @@ application_snapshot("dartdevc") {
     "--help",
   ]
   inputs = exec_script("../../tools/list_dart_files.py",
-                       [ rebase_path("../../pkg/dev_compiler/bin") ],
+                       [ "absolute",
+                         rebase_path("../../pkg/dev_compiler/bin") ],
                        "list lines")
 }
diff --git a/utils/dartdevc/dartdevc.gyp b/utils/dartdevc/dartdevc.gyp
index 167327ec99d..b141bc5e590 100644
--- a/utils/dartdevc/dartdevc.gyp
+++ b/utils/dartdevc/dartdevc.gyp
@@ -16,7 +16,8 @@
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
             '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../pkg/dev_compiler/bin"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../pkg/dev_compiler/bin"])',
             '<(SHARED_INTERMEDIATE_DIR)/pkg_files.stamp',
           ],
           'outputs': [
diff --git a/utils/dartdoc/BUILD.gn b/utils/dartdoc/BUILD.gn
index 6abdc9a1a71..ac302a821bc 100644
--- a/utils/dartdoc/BUILD.gn
+++ b/utils/dartdoc/BUILD.gn
@@ -8,6 +8,7 @@ application_snapshot("dartdoc") {
   main_dart = "../../third_party/pkg/dartdoc/bin/dartdoc.dart"
   training_args = [ "--help" ]
   inputs = exec_script("../../tools/list_dart_files.py",
-                       [ rebase_path("../../third_party/pkg/dartdoc") ],
+                       [ "absolute",
+                         rebase_path("../../third_party/pkg/dartdoc") ],
                        "list lines")
 }
diff --git a/utils/dartdoc/dartdoc.gyp b/utils/dartdoc/dartdoc.gyp
index dc5e414834c..e5327c0edc0 100644
--- a/utils/dartdoc/dartdoc.gyp
+++ b/utils/dartdoc/dartdoc.gyp
@@ -16,7 +16,8 @@
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
             '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../third_party/pkg/dartdoc"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../third_party/pkg/dartdoc"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/dartdoc.dart.snapshot',
@@ -31,4 +32,4 @@
       ],
     },
   ],
-}
\ No newline at end of file
+}
diff --git a/utils/dartfmt/BUILD.gn b/utils/dartfmt/BUILD.gn
index 41c5bc43066..7c688a318f6 100644
--- a/utils/dartfmt/BUILD.gn
+++ b/utils/dartfmt/BUILD.gn
@@ -9,6 +9,7 @@ application_snapshot("dartfmt") {
   training_args = [ "--help" ]
   inputs =
       exec_script("../../tools/list_dart_files.py",
-                  [ rebase_path("../../third_party/pkg_tested/dart_style") ],
+                  [ "absolute",
+                    rebase_path("../../third_party/pkg_tested/dart_style") ],
                   "list lines")
 }
diff --git a/utils/dartfmt/dartfmt.gyp b/utils/dartfmt/dartfmt.gyp
index a59ef9a5b02..5590883474b 100644
--- a/utils/dartfmt/dartfmt.gyp
+++ b/utils/dartfmt/dartfmt.gyp
@@ -16,7 +16,8 @@
           'inputs': [
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
             '../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart',
-            '<!@(["python", "../../tools/list_files.py", "\\.dart$", "../../third_party/pkg_tested/dart_style"])',
+            '<!@(["python", "../../tools/list_dart_files.py", "relative", '
+                '"../../third_party/pkg_tested/dart_style"])',
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/dartfmt.dart.snapshot',