From f917c5e722d7ee5abd58704eb0e5d49072249e94 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Sun, 8 Jul 2018 02:24:29 -0400 Subject: [PATCH] Clean up tools/ - Factor out tools/util.py - Move js/*.py to tools. - Rewrite tools/format.sh in python. - Run lint first in travis. --- .gitignore | 1 + .travis.yml | 2 +- BUILD.gn | 2 +- build_extra/deno.gni | 2 +- build_extra/flatbuffers/BUILD.gn | 1 - build_extra/flatbuffers/flatbuffer.gni | 12 +++-- js/run_node.py | 40 ---------------- tools/build_third_party.py | 66 ++++++-------------------- {js => tools}/flatbufferjs_hack.py | 11 +---- tools/format.py | 25 ++++++++++ tools/format.sh | 24 ---------- tools/lint.py | 26 +++------- tools/run_node.py | 19 ++++++++ tools/util.py | 42 ++++++++++++++++ 14 files changed, 123 insertions(+), 150 deletions(-) delete mode 100755 js/run_node.py rename {js => tools}/flatbufferjs_hack.py (82%) create mode 100755 tools/format.py delete mode 100755 tools/format.sh create mode 100755 tools/run_node.py create mode 100644 tools/util.py diff --git a/.gitignore b/.gitignore index f349cf0c27..b8e0ab33c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # build /out/ +*.pyc # npm deps node_modules diff --git a/.travis.yml b/.travis.yml index afcd2fc22b..1718e2eb3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,8 +44,8 @@ install: # Travis hangs without -j2 argument to ninja. - ninja -j2 -C $BUILD_PATH mock_runtime_test handlers_test deno_cc deno script: + - ./tools/lint.py - $BUILD_PATH/mock_runtime_test - $BUILD_PATH/handlers_test - $BUILD_PATH/deno_cc foo bar - $BUILD_PATH/deno meow - - ./tools/lint.py diff --git a/BUILD.gn b/BUILD.gn index 32cebba5b3..c658943737 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -177,7 +177,7 @@ run_node("run_tsc") { # be removed at some point. If msg.fps is changed, commit changes to the # generated JS files. The stamp file is just to make gn work. action("flatbufferjs") { - script = "js/flatbufferjs_hack.py" + script = "//tools/flatbufferjs_hack.py" sources = [ "src/msg.fbs", ] diff --git a/build_extra/deno.gni b/build_extra/deno.gni index a8804e9f93..6d1293bc16 100644 --- a/build_extra/deno.gni +++ b/build_extra/deno.gni @@ -1,7 +1,7 @@ template("run_node") { action(target_name) { forward_variables_from(invoker, "*") - script = "//js/run_node.py" + script = "//tools/run_node.py" } } diff --git a/build_extra/flatbuffers/BUILD.gn b/build_extra/flatbuffers/BUILD.gn index 0433c5d1d5..b46cb3d2c0 100644 --- a/build_extra/flatbuffers/BUILD.gn +++ b/build_extra/flatbuffers/BUILD.gn @@ -100,7 +100,6 @@ flatbuffer("flatbuffers_samplebuffer") { ] flatc_include_dirs = [ "$fb_src/tests/include_test" ] } - # test("flatbuffers_unittest") { # sources = [ # "src/tests/test.cpp", diff --git a/build_extra/flatbuffers/flatbuffer.gni b/build_extra/flatbuffers/flatbuffer.gni index 0bac1802c6..ff4a48dfa0 100644 --- a/build_extra/flatbuffers/flatbuffer.gni +++ b/build_extra/flatbuffers/flatbuffer.gni @@ -188,15 +188,21 @@ template("ts_flatbuffer") { copy_name = target_name + "_copy" copy(copy_name) { - sources = [ "$flatbuffers_source_location/js/flatbuffers.js" ] - outputs = [ "$target_gen_dir/flatbuffers.js" ] + sources = [ + "$flatbuffers_source_location/js/flatbuffers.js", + ] + outputs = [ + "$target_gen_dir/flatbuffers.js", + ] } compiled_action_foreach(target_name) { tool = "$flatbuffers_build_location:flatc" sources = invoker.sources - deps = [ ":" + copy_name ] + deps = [ + ":" + copy_name, + ] out_dir = target_gen_dir diff --git a/js/run_node.py b/js/run_node.py deleted file mode 100755 index 154c2167ba..0000000000 --- a/js/run_node.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -gn can only run python scripts. This launches a subprocess Node process. -The working dir of this program is out/Debug/ (AKA root_build_dir) -Before running node, we symlink js/node_modules to out/Debug/node_modules. -""" -import subprocess -import sys -import os - - -def symlink(target, name, target_is_dir=False): - if os.name == "nt": - import ctypes - CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW - CreateSymbolicLinkW.restype = ctypes.c_ubyte - CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, - ctypes.c_uint32) - - flags = 0x02 # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - if (target_is_dir): - flags |= 0x01 # SYMBOLIC_LINK_FLAG_DIRECTORY - if not CreateSymbolicLinkW(name, target, flags): - raise ctypes.WinError() - else: - os.symlink(target, name) - - -root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -third_party_path = os.path.join(root_path, "third_party") -target_abs = os.path.join(third_party_path, "node_modules") -target_rel = os.path.relpath(target_abs) - -if not os.path.exists("node_modules"): - if os.path.lexists("node_modules"): - os.unlink("node_modules") - symlink(target_rel, "node_modules", True) - -args = ["node"] + sys.argv[1:] -sys.exit(subprocess.call(args)) diff --git a/tools/build_third_party.py b/tools/build_third_party.py index e5c2885dc6..87ed047838 100755 --- a/tools/build_third_party.py +++ b/tools/build_third_party.py @@ -10,58 +10,22 @@ import os from os.path import join import subprocess +from util import run, remove_and_symlink root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) third_party_path = join(root_path, "third_party") - -def main(): - try: - os.makedirs(third_party_path) - except: - pass - os.chdir(third_party_path) - remove_and_symlink(join("..", "gclient_config.py"), ".gclient") - remove_and_symlink(join("..", "package.json"), "package.json") - remove_and_symlink(join("..", "yarn.lock"), "yarn.lock") - remove_and_symlink(join("v8", "third_party", "googletest"), "googletest") - remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2") - remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build") - remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe") - run(["gclient", "sync", "--no-history"]) - run(["yarn"]) - - -def run(args): - print " ".join(args) - env = os.environ.copy() - subprocess.check_call(args, env=env) - - -def remove_and_symlink(target, name): - try: - os.unlink(name) - except: - pass - os.symlink(target, name) - - -def symlink(target, name, target_is_dir=False): - if os.name == "nt": - import ctypes - CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW - CreateSymbolicLinkW.restype = ctypes.c_ubyte - CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, - ctypes.c_uint32) - - flags = 0x02 # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - if (target_is_dir): - flags |= 0x01 # SYMBOLIC_LINK_FLAG_DIRECTORY - if not CreateSymbolicLinkW(name, target, flags): - raise ctypes.WinError() - else: - os.symlink(target, name) - - -if '__main__' == __name__: - main() +try: + os.makedirs(third_party_path) +except: + pass +os.chdir(third_party_path) +remove_and_symlink(join("..", "gclient_config.py"), ".gclient") +remove_and_symlink(join("..", "package.json"), "package.json") +remove_and_symlink(join("..", "yarn.lock"), "yarn.lock") +remove_and_symlink(join("v8", "third_party", "googletest"), "googletest") +remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2") +remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build") +remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe") +run(["gclient", "sync", "--no-history"]) +run(["yarn"]) diff --git a/js/flatbufferjs_hack.py b/tools/flatbufferjs_hack.py similarity index 82% rename from js/flatbufferjs_hack.py rename to tools/flatbufferjs_hack.py index 2f411c1db3..163a1893cb 100755 --- a/js/flatbufferjs_hack.py +++ b/tools/flatbufferjs_hack.py @@ -8,6 +8,7 @@ import subprocess import sys import os import shutil +import util # TODO(ry) Ideally flatc output files should be written into target_gen_dir, but # its difficult to get this working in a way that parcel can resolve their @@ -22,12 +23,4 @@ stamp_file = sys.argv[3] shutil.copyfile(src, dst) - -def touch(fname): - if os.path.exists(fname): - os.utime(fname, None) - else: - open(fname, 'a').close() - - -touch(stamp_file) +util.touch(stamp_file) diff --git a/tools/format.py b/tools/format.py new file mode 100755 index 0000000000..d1f555e5df --- /dev/null +++ b/tools/format.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import os +from glob import glob +from util import run + +root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +os.chdir(root_path) +# TODO(ry) Install clang-format in third_party. +run(["clang-format", "-i", "-style", "Google"] + glob("src/*.cc") + + glob("src/*.h")) +for fn in ["BUILD.gn", ".gn"] + glob("build_extra/**/*.gn*"): + run(["gn", "format", fn]) +# TODO(ry) Install yapf in third_party. +run(["yapf", "-i"] + glob("tools/*.py")) +# TODO(ry) Install prettier in third_party. +run([ + "prettier", "--write", "js/deno.d.ts", "js/main.ts", "js/mock_runtime.js", + "tsconfig.json" +]) +# Do not format these. +# js/msg_generated.ts +# js/flatbuffers.js +run(["rustfmt", "--write-mode", "overwrite"] + glob("src/*.rs")) diff --git a/tools/format.sh b/tools/format.sh deleted file mode 100755 index 44ca815ff4..0000000000 --- a/tools/format.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -set -e -cd `dirname "$0"`/.. -clang-format -i -style Google src/*.cc src/*.h - -gn format BUILD.gn -gn format build_extra/deno.gni -gn format build_extra/rust/rust.gni -gn format build_extra/rust/BUILD.gn -gn format .gn - -yapf -i js/*.py -yapf -i tools/*.py - -prettier --write \ - js/deno.d.ts \ - js/main.ts \ - js/mock_runtime.js \ - tsconfig.json -# Do not format these. -# js/msg_generated.ts -# js/flatbuffers.js - -rustfmt --write-mode overwrite src/*.rs diff --git a/tools/lint.py b/tools/lint.py index 1fa0d461d3..452d441040 100755 --- a/tools/lint.py +++ b/tools/lint.py @@ -2,7 +2,7 @@ # Does google-lint on c++ files and ts-lint on typescript files import os -import subprocess +from util import run root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) third_party_path = os.path.join(root_path, "third_party") @@ -10,21 +10,9 @@ cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py") tslint = os.path.join(third_party_path, "node_modules", "tslint", "bin", "tslint") - -def run(args): - print(" ".join(args)) - env = os.environ.copy() - subprocess.check_call(args, env=env) - - -def main(): - os.chdir(root_path) - run([ - "python", cpplint, "--filter=-build/include_subdir", - "--repository=src", "--extensions=cc,h", "--recursive", "src/." - ]) - run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"]) - - -if __name__ == "__main__": - main() +os.chdir(root_path) +run([ + "python", cpplint, "--filter=-build/include_subdir", "--repository=src", + "--extensions=cc,h", "--recursive", "src/." +]) +run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"]) diff --git a/tools/run_node.py b/tools/run_node.py new file mode 100755 index 0000000000..afa861020f --- /dev/null +++ b/tools/run_node.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +gn can only run python scripts. This launches a subprocess Node process. +The working dir of this program is out/Debug/ (AKA root_build_dir) +Before running node, we symlink js/node_modules to out/Debug/node_modules. +""" +import subprocess +import sys +import os +import util + +root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +tools_path = os.path.join(root_path, "tools") +third_party_path = os.path.join(root_path, "third_party") +target_abs = os.path.join(third_party_path, "node_modules") +target_rel = os.path.relpath(target_abs) + +util.remove_and_symlink(target_rel, "node_modules", True) +util.run(["node"] + sys.argv[1:]) diff --git a/tools/util.py b/tools/util.py new file mode 100644 index 0000000000..1d44acd48d --- /dev/null +++ b/tools/util.py @@ -0,0 +1,42 @@ +# Copyright 2018 Ryan Dahl +# All rights reserved. MIT License. +import os +import subprocess + + +def run(args): + print " ".join(args) + env = os.environ.copy() + subprocess.check_call(args, env=env) + + +def remove_and_symlink(target, name, target_is_dir=False): + try: + os.unlink(name) + except: + pass + symlink(target, name, target_is_dir) + + +def symlink(target, name, target_is_dir=False): + if os.name == "nt": + import ctypes + CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW + CreateSymbolicLinkW.restype = ctypes.c_ubyte + CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, + ctypes.c_uint32) + + flags = 0x02 # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + if (target_is_dir): + flags |= 0x01 # SYMBOLIC_LINK_FLAG_DIRECTORY + if not CreateSymbolicLinkW(name, target, flags): + raise ctypes.WinError() + else: + os.symlink(target, name) + + +def touch(fname): + if os.path.exists(fname): + os.utime(fname, None) + else: + open(fname, 'a').close()