diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ad1556aa96..d21fdb35c0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,6 +17,6 @@ https://github.com/denoland/deno/blob/master/docs/contributing.md 2. Ensure there is a related issue and it is referenced in the PR text. 3. Ensure there are tests that cover the changes. 4. Ensure `cargo test` passes. -5. Ensure `./tools/format.py` passes without changing files. -6. Ensure `./tools/lint.py` passes. +5. Ensure `./tools/format.js` passes without changing files. +6. Ensure `./tools/lint.js` passes. --> diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9c528e0c0..5e46c273b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,19 @@ jobs: rustup component add clippy rustup component add rustfmt + - name: Install Deno + if: | + !startsWith(matrix.os, 'windows') + run: |- + curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.5.1 + echo "$HOME/.deno/bin" >> $GITHUB_PATH + + - name: Install Deno (Windows) + if: startsWith(matrix.os, 'windows') + run: |- + curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.5.1 + echo "$HOME/.deno/bin" >> $env:GITHUB_PATH + - name: Install Python uses: actions/setup-python@v1 with: @@ -93,14 +106,15 @@ jobs: python --version rustc --version cargo --version + deno --version - - name: lint.py + - name: lint.js if: matrix.kind == 'lint' - run: python ./tools/lint.py + run: deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js - - name: test_format.py + - name: test_format.js if: matrix.kind == 'lint' - run: python ./tools/test_format.py + run: deno run --unstable --allow-write --allow-read --allow-run ./tools/format.js --check - name: Build release if: | @@ -133,7 +147,7 @@ jobs: DENOBOT_PAT: ${{ secrets.DENOBOT_PAT }} run: | git clone --depth 1 -b gh-pages https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git gh-pages - python ./tools/build_benchmark_jsons.py --release + deno run --unstable -A ./tools/build_benchmark_jsons.js --release cd gh-pages git config user.email "propelml@gmail.com" git config user.name "denobot" diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 2fd28a109b..9c6c4eef26 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -1728,11 +1728,6 @@ fn deno_test_no_color() { assert!(out.contains("test result: FAILED. 1 passed; 1 failed; 1 ignored; 0 measured; 0 filtered out")); } -#[test] -fn util_test() { - util::run_python_script("tools/util_test.py") -} - macro_rules! itest( ($name:ident {$( $key:ident: $value:expr,)*}) => { #[test] diff --git a/docs/contributing.md b/docs/contributing.md index daabcf055e..14af3f6aae 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -42,8 +42,8 @@ Examples of bad PR title: 2. Ensure there is a related issue and it is referenced in the PR text. 3. Ensure there are tests that cover the changes. 4. Ensure `cargo test` passes. -5. Ensure `./tools/format.py` passes without changing files. -6. Ensure `./tools/lint.py` passes. +5. Ensure `./tools/format.js` passes without changing files. +6. Ensure `./tools/lint.js` passes. ## Changes to `third_party` diff --git a/docs/contributing/development_tools.md b/docs/contributing/development_tools.md index db6f9a25a7..a11e7beff2 100644 --- a/docs/contributing/development_tools.md +++ b/docs/contributing/development_tools.md @@ -23,13 +23,13 @@ cargo test std_tests Lint the code: ```shell -./tools/lint.py +deno run -A --unstable ./tools/lint.js ``` Format the code: ```shell -./tools/format.py +deno run -A --unstable ./tools/format.js ``` ### Profiling diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index 366f8b6bae..97a2ed3343 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -776,26 +776,6 @@ pub fn deno_cmd() -> Command { c } -pub fn run_python_script(script: &str) { - let deno_dir = new_deno_dir(); - let output = Command::new("python") - .env("DENO_DIR", deno_dir.path()) - .current_dir(root_path()) - .arg(script) - .arg(format!("--build-dir={}", target_dir().display())) - .arg(format!("--executable={}", deno_exe_path().display())) - .output() - .expect("failed to spawn script"); - if !output.status.success() { - let stdout = String::from_utf8(output.stdout).unwrap(); - let stderr = String::from_utf8(output.stderr).unwrap(); - panic!( - "{} executed with failing error code\n{}{}", - script, stdout, stderr - ); - } -} - pub fn run_powershell_script_file( script_file_path: &str, args: Vec<&str>, diff --git a/third_party b/third_party index 64944a3a65..76b283c420 160000 --- a/third_party +++ b/third_party @@ -1 +1 @@ -Subproject commit 64944a3a65e15a02d117a641f40de50179358259 +Subproject commit 76b283c420be9302309e94394d1b3f1ebb425f29 diff --git a/tools/README.md b/tools/README.md index be5b5394d3..dff70e9e88 100644 --- a/tools/README.md +++ b/tools/README.md @@ -2,24 +2,24 @@ Documentation for various tooling in support of Deno development. -## format.py +## format.js -This script will format the code (currently using dprint, yapf and rustfmt). It -is a prerequisite to run this before code check in. +This script will format the code (currently using dprint, rustfmt). It is a +prerequisite to run this before code check in. To run formatting: ```sh -./tools/format.py +deno run -A --unstable ./tools/format.js ``` ## lint.py -This script will lint the code base (currently using eslint, pylint and clippy). -It is a prerequisite to run this before code check in. +This script will lint the code base (currently using dlint, clippy). It is a +prerequisite to run this before code check in. To run linting: ```sh -./tools/lint.py +deno run -A --unstable ./tools/lint.js ``` diff --git a/tools/build_benchmark_jsons.js b/tools/build_benchmark_jsons.js new file mode 100644 index 0000000000..af43226d70 --- /dev/null +++ b/tools/build_benchmark_jsons.js @@ -0,0 +1,31 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { buildPath, existsSync, join } from "./util.js"; + +const currentDataFile = join(buildPath(), "bench.json"); +const allDataFile = "gh-pages/data.json"; // Includes all benchmark data. +const recentDataFile = "gh-pages/recent.json"; // Includes recent 20 benchmark data. + +function readJson(filename) { + return JSON.parse(Deno.readTextFileSync(filename)); +} + +function writeJson(filename, data) { + return Deno.writeTextFileSync(filename, JSON.stringify(data)); +} + +if (!existsSync(currentDataFile)) { + throw new Error(`${currentDataFile} doesn't exist`); +} + +if (!existsSync(allDataFile)) { + throw new Error(`${allDataFile} doesn't exist`); +} + +const newData = readJson(currentDataFile); +const allData = readJson(allDataFile); +allData.push(newData); +const allDataLen = allData.length; +const recentData = allData.slice(allDataLen - 20); + +writeJson(allDataFile, allData); +writeJson(recentDataFile, recentData); diff --git a/tools/build_benchmark_jsons.py b/tools/build_benchmark_jsons.py deleted file mode 100755 index f9f86c7548..0000000000 --- a/tools/build_benchmark_jsons.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import os -import json -from util import build_path - - -def read_json(filename): - with open(filename) as json_file: - return json.load(json_file) - - -def write_json(filename, data): - with open(filename, 'w') as outfile: - json.dump(data, outfile) - - -current_data_file = os.path.join(build_path(), "bench.json") -all_data_file = "gh-pages/data.json" # Includes all benchmark data. -recent_data_file = "gh-pages/recent.json" # Includes recent 20 benchmark data. - -assert os.path.exists(current_data_file) -assert os.path.exists(all_data_file) - -new_data = read_json(current_data_file) -all_data = read_json(all_data_file) -all_data.append(new_data) - -write_json(all_data_file, all_data) -write_json(recent_data_file, all_data[-20:]) diff --git a/tools/format.js b/tools/format.js new file mode 100755 index 0000000000..9786e6fe37 --- /dev/null +++ b/tools/format.js @@ -0,0 +1,63 @@ +#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-run +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { getPrebuiltToolPath, getSources, join, ROOT_PATH } from "./util.js"; + +async function dprint() { + const execPath = getPrebuiltToolPath("dprint"); + console.log("dprint"); + const p = Deno.run({ + cmd: [execPath, "fmt"], + }); + const { success } = await p.status(); + if (!success) { + throw new Error("dprint failed"); + } + p.close(); +} + +async function rustfmt() { + const configFile = join(ROOT_PATH, ".rustfmt.toml"); + const sourceFiles = await getSources(ROOT_PATH, ["*.rs"]); + + if (!sourceFiles.length) { + return; + } + + console.log(`rustfmt ${sourceFiles.length} file(s)`); + const p = Deno.run({ + cmd: ["rustfmt", "--config-path=" + configFile, "--", ...sourceFiles], + }); + const { success } = await p.status(); + if (!success) { + throw new Error("rustfmt failed"); + } + p.close(); +} + +async function main() { + await Deno.chdir(ROOT_PATH); + await dprint(); + await rustfmt(); + + if (Deno.args.includes("--check")) { + const git = Deno.run({ + cmd: ["git", "status", "-uno", "--porcelain", "--ignore-submodules"], + stdout: "piped", + }); + + const { success } = await git.status(); + if (!success) { + throw new Error("git status failed"); + } + const out = new TextDecoder().decode(await git.output()); + git.close(); + + if (out) { + console.log("run tools/format.js"); + console.log(out); + Deno.exit(1); + } + } +} + +await main(); diff --git a/tools/format.py b/tools/format.py deleted file mode 100755 index 4ff17af432..0000000000 --- a/tools/format.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import os -import sys -import argparse -from third_party import python_env, get_prebuilt_tool_path -from util import git_ls_files, git_staged, third_party_path, root_path -from util import print_command, run - -cmd_args = None - - -def get_cmd_args(): - global cmd_args - - if cmd_args: - return cmd_args - - parser = argparse.ArgumentParser() - parser.add_argument("--js", help="run dprint", action="store_true") - parser.add_argument("--py", help="run yapf", action="store_true") - parser.add_argument("--rs", help="run rustfmt", action="store_true") - parser.add_argument( - "--staged", help="run only on staged files", action="store_true") - cmd_args = parser.parse_args() - return cmd_args - - -def get_sources(*args): - getter = git_staged if get_cmd_args().staged else git_ls_files - return getter(*args) - - -def main(): - os.chdir(root_path) - - args = get_cmd_args() - - did_fmt = False - if args.js: - dprint() - did_fmt = True - if args.py: - yapf() - did_fmt = True - if args.rs: - rustfmt() - did_fmt = True - - if not did_fmt: - dprint() - yapf() - rustfmt() - - -def dprint(): - executable_path = get_prebuilt_tool_path("dprint") - command = [executable_path, "fmt"] - run(command, shell=False, quiet=True) - - -def yapf(): - script = os.path.join(third_party_path, "python_packages", "bin", "yapf") - source_files = get_sources(root_path, ["*.py"]) - if source_files: - print_command("yapf", source_files) - run([sys.executable, script, "-i", "--style=pep8", "--"] + - source_files, - env=python_env(), - shell=False, - quiet=True) - - -def rustfmt(): - config_file = os.path.join(root_path, ".rustfmt.toml") - source_files = get_sources(root_path, ["*.rs"]) - if source_files: - print_command("rustfmt", source_files) - run([ - "rustfmt", - "--config-path=" + config_file, - "--", - ] + source_files, - shell=False, - quiet=True) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/lint.js b/tools/lint.js new file mode 100755 index 0000000000..18de2aef30 --- /dev/null +++ b/tools/lint.js @@ -0,0 +1,102 @@ +#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-run +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { + buildMode, + getPrebuiltToolPath, + getSources, + ROOT_PATH, +} from "./util.js"; + +async function dlint() { + const execPath = getPrebuiltToolPath("dlint"); + console.log("dlint"); + + const sourceFiles = await getSources(ROOT_PATH, [ + "*.js", + "*.ts", + ":!:cli/tests/swc_syntax_error.ts", + ":!:cli/tests/038_checkjs.js", + ":!:cli/tests/error_008_checkjs.js", + ":!:std/**/testdata/*", + ":!:std/**/node_modules/*", + ":!:cli/bench/node*.js", + ":!:cli/compilers/wasm_wrap.js", + ":!:cli/dts/**", + ":!:cli/tests/encoding/**", + ":!:cli/tests/error_syntax.js", + ":!:cli/tests/lint/**", + ":!:cli/tests/tsc/**", + ":!:cli/tsc/*typescript.js", + ]); + + if (!sourceFiles.length) { + return; + } + + const MAX_COMMAND_LEN = 30000; + const preCommand = [execPath, "run"]; + const chunks = [[]]; + let cmdLen = preCommand.join(" ").length; + for (const f of sourceFiles) { + if (cmdLen + f.length > MAX_COMMAND_LEN) { + chunks.push([f]); + cmdLen = preCommand.join(" ").length; + } else { + chunks[chunks.length - 1].push(f); + cmdLen = preCommand.join(" ").length; + } + } + for (const chunk of chunks) { + const p = Deno.run({ + cmd: [execPath, "run", ...chunk], + }); + const { success } = await p.status(); + if (!success) { + throw new Error("dlint failed"); + } + p.close(); + } +} + +async function clippy() { + console.log("clippy"); + + const currentBuildMode = buildMode(); + const cmd = ["cargo", "clippy", "--all-targets", "--locked"]; + + if (currentBuildMode != "debug") { + cmd.push("--release"); + } + + const p = Deno.run({ + cmd: [...cmd, "--", "-D", "clippy::all"], + }); + const { success } = await p.status(); + if (!success) { + throw new Error("clippy failed"); + } + p.close(); +} + +async function main() { + await Deno.chdir(ROOT_PATH); + + let didLint = false; + + if (Deno.args.includes("--js")) { + await dlint(); + didLint = true; + } + + if (Deno.args.includes("--rs")) { + await clippy(); + didLint = true; + } + + if (!didLint) { + await dlint(); + await clippy(); + } +} + +await main(); diff --git a/tools/lint.py b/tools/lint.py deleted file mode 100755 index 23bf0e81c3..0000000000 --- a/tools/lint.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# Does google-lint on c++ files and ts-lint on typescript files - -import os -import sys -import argparse -from util import enable_ansi_colors, git_ls_files, git_staged, root_path, run -from util import third_party_path, build_mode, print_command -from third_party import python_env, get_prebuilt_tool_path - -cmd_args = None - - -def get_cmd_args(): - global cmd_args - - if cmd_args: - return cmd_args - - parser = argparse.ArgumentParser() - parser.add_argument("--js", help="run dlint", action="store_true") - parser.add_argument("--py", help="run pylint", action="store_true") - parser.add_argument("--rs", help="run clippy", action="store_true") - parser.add_argument( - "--staged", help="run only on staged files", action="store_true") - cmd_args = parser.parse_args() - return cmd_args - - -def get_sources(*args): - getter = git_staged if get_cmd_args().staged else git_ls_files - return getter(*args) - - -def main(): - enable_ansi_colors() - os.chdir(root_path) - - args = get_cmd_args() - - did_fmt = False - if args.js: - dlint() - did_fmt = True - if args.py: - pylint() - did_fmt = True - if args.rs: - clippy() - did_fmt = True - - if not did_fmt: - dlint() - pylint() - clippy() - - -def dlint(): - executable_path = get_prebuilt_tool_path("dlint") - - # Find all *directories* in the main repo that contain .ts/.js files. - source_files = get_sources(root_path, [ - "*.js", - "*.ts", - ":!:cli/tests/swc_syntax_error.ts", - ":!:cli/tests/038_checkjs.js", - ":!:cli/tests/error_008_checkjs.js", - ":!:std/**/testdata/*", - ":!:std/**/node_modules/*", - ":!:cli/bench/node*.js", - ":!:cli/compilers/wasm_wrap.js", - ":!:cli/dts/**", - ":!:cli/tests/encoding/**", - ":!:cli/tests/error_syntax.js", - ":!:cli/tests/lint/**", - ":!:cli/tests/tsc/**", - ":!:cli/tsc/*typescript.js", - ]) - if source_files: - max_command_len = 30000 - pre_command = [executable_path, "run"] - chunks = [[]] - cmd_len = len(" ".join(pre_command)) - for f in source_files: - if cmd_len + len(f) > max_command_len: - chunks.append([f]) - cmd_len = len(" ".join(pre_command)) - else: - chunks[-1].append(f) - cmd_len = cmd_len + len(f) + 1 - for c in chunks: - print_command("dlint", c) - run(pre_command + c, shell=False, quiet=True) - - -def pylint(): - script = os.path.join(third_party_path, "python_packages", "pylint") - rcfile = os.path.join(root_path, "tools", "pylintrc") - msg_template = "{path}({line}:{column}) {category}: {msg} ({symbol})" - source_files = get_sources(root_path, ["*.py"]) - if source_files: - print_command("pylint", source_files) - run([ - sys.executable, script, "--rcfile=" + rcfile, - "--msg-template=" + msg_template, "--" - ] + source_files, - env=python_env(), - shell=False, - quiet=True) - - -def clippy(): - print("clippy") - current_build_mode = build_mode() - args = ["cargo", "clippy", "--all-targets", "--locked"] - if current_build_mode != "debug": - args += ["--release"] - run(args + ["--", "-D", "clippy::all"], shell=False, quiet=True) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/pylintrc b/tools/pylintrc deleted file mode 100644 index 5822be77a2..0000000000 --- a/tools/pylintrc +++ /dev/null @@ -1,337 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). -# -# These should get enabled, but the codebase has too many violations currently: -# bad-continuation -# anomalous-backslash-in-string -# bad-context-manager -# bad-indentation -# bad-str-strip-call -# bad-whitespace -# cell-var-from-loop -# deprecated-lambda -# eval-used -# function-redefined -# import-error -# locally-enabled -# missing-final-newline -# no-init -# no-name-in-module -# no-self-use -# not-callable -# old-style-class -# protected-access -# superfluous-parens -# super-on-old-class -# too-many-function-args -# trailing-whitespace -# unnecessary-semicolon -# unpacking-non-sequence -# unused-import -# useless-else-on-loop -# -# CHANGED: -disable= - invalid-name, - missing-docstring, - too-many-lines, - bad-inline-option, - locally-disabled, - duplicate-code, - too-many-ancestors, - too-many-instance-attributes, - too-few-public-methods, - too-many-public-methods, - too-many-return-statements, - too-many-branches, - too-many-arguments, - too-many-locals, - too-many-statements, - abstract-class-not-used, - abstract-class-little-used, - exec-used, - bad-builtin, - star-args, - deprecated-module, - reimported, - fixme, - global-statement, - broad-except, - logging-not-lazy, - bad-continuation, - anomalous-backslash-in-string, - assigning-non-slot, - bad-context-manager, - bad-indentation, - bad-str-strip-call, - bad-super-call, - bad-whitespace, - cell-var-from-loop, - consider-using-enumerate, - deprecated-lambda, - deprecated-method, - eval-used, - function-redefined, - import-error, - invalid-docstring-quote, - invalid-string-quote, - invalid-triple-quote, - locally-enabled, - misplaced-comparison-constant, - misplaced-bare-raise, - missing-final-newline, - multiple-imports, - no-init, - no-name-in-module, - no-self-argument, - no-self-use, - not-an-iterable, - not-callable, - old-style-class, - protected-access, - redefined-variable-type, - simplifiable-if-statement, - singleton-comparison, - superfluous-parens, - super-on-old-class, - too-many-boolean-expressions, - too-many-function-args, - too-many-nested-blocks, - trailing-whitespace, - undefined-variable, - ungrouped-imports, - unnecessary-semicolon, - unneeded-not, - unpacking-non-sequence, - unsubscriptable-object, - unsupported-membership-test, - unused-import, - useless-else-on-loop, - using-constant-test, - wrong-import-order, - wrong-import-position, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -# CHANGED: -reports=no - -# Activate the evaluation score. -score=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the beginning of the name of dummy variables -# (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent,multiprocessing.managers.SyncManager - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -# CHANGED: -indent-string=' ' - - -[BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branchs=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/tools/test_format.py b/tools/test_format.py deleted file mode 100755 index ef2a0ffb3f..0000000000 --- a/tools/test_format.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# This program fails if ./tools/format.py changes any files. - -import sys -import subprocess -import util - - -def main(): - util.run([sys.executable, "tools/format.py"]) - result = util.run_output( - ["git", "status", "-uno", "--porcelain", "--ignore-submodules"], - exit_on_fail=True) - if result.out: - print("Run tools/format.py ") - print(result.out) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/tools/test_util.py b/tools/test_util.py deleted file mode 100644 index 7dad0f4ff2..0000000000 --- a/tools/test_util.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# Runs the full test suite. -# Usage: ./tools/test.py out/Debug -import argparse -import contextlib -import os -import sys -import unittest - -from util import (build_path, RESET, FG_RED, FG_GREEN, executable_suffix) - - -class DenoTestCase(unittest.TestCase): - @classmethod - def setUpClass(cls): - args = parse_test_args() - - cls.build_dir = args.build_dir - cls.deno_exe = args.executable - - -# overload the test result class -class ColorTextTestResult(unittest.TextTestResult): - @contextlib.contextmanager - def color(self, code): - self.stream.write(code) - try: - yield - finally: - self.stream.write(RESET) - - def getDescription(self, test): - name = str(test) - if name.startswith("test_"): - name = name[5:] - return name - - def addSuccess(self, test): - with self.color(FG_GREEN): - super(ColorTextTestResult, self).addSuccess(test) - - def addError(self, test, err): - with self.color(FG_RED): - super(ColorTextTestResult, self).addError(test, err) - - def addFailure(self, test, err): - with self.color(FG_RED): - super(ColorTextTestResult, self).addFailure(test, err) - - -class ColorTextTestRunner(unittest.TextTestRunner): - resultclass = ColorTextTestResult - - -def create_test_arg_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--failfast', '-f', action='store_true', help='Stop on first failure') - parser.add_argument( - '--verbose', '-v', action='store_true', help='Verbose output') - parser.add_argument("--executable", help="Use external executable of Deno") - parser.add_argument( - '--release', - action='store_true', - help='Test against release executable') - parser.add_argument( - '--pattern', '-p', help='Run tests that match provided pattern') - parser.add_argument( - '--build-dir', dest="build_dir", help='Deno build directory') - return parser - - -TestArgParser = create_test_arg_parser() - - -def parse_test_args(argv=None): - if argv is None: - argv = sys.argv[1:] - - args = TestArgParser.parse_args(argv) - - if args.executable and args.release: - raise argparse.ArgumentError( - None, "Path to executable is inferred from " - "--release, cannot provide both.") - - if not args.build_dir: - args.build_dir = build_path() - - if not args.executable: - args.executable = os.path.join(args.build_dir, - "deno" + executable_suffix) - - if not os.path.isfile(args.executable): - raise argparse.ArgumentError( - None, "deno executable not found at {}".format(args.executable)) - - return args - - -def filter_test_suite(suite, pattern): - filtered_tests = [] - - for test_case in suite: - if isinstance(test_case, unittest.TestSuite): - filtered_tests += filter_test_suite(test_case, pattern) - else: - if pattern in str(test_case): - filtered_tests.append(test_case) - - return filtered_tests - - -def run_tests(test_cases=None): - args = parse_test_args() - - loader = unittest.TestLoader() - - # if suite was not explicitly passed load test - # cases from calling module - if test_cases is None: - import __main__ - suite = loader.loadTestsFromModule(__main__) - else: - suite = unittest.TestSuite() - for test_case in test_cases: - suite.addTests(loader.loadTestsFromTestCase(test_case)) - - if args.pattern: - filtered_tests = filter_test_suite(suite, args.pattern) - suite = unittest.TestSuite(filtered_tests) - - runner = ColorTextTestRunner( - verbosity=args.verbose + 2, failfast=args.failfast) - - result = runner.run(suite) - if not result.wasSuccessful(): - sys.exit(1) diff --git a/tools/third_party.py b/tools/third_party.py deleted file mode 100644 index 3464e148ea..0000000000 --- a/tools/third_party.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# This script contains helper functions to work with the third_party subrepo. - -import os -import re -import site -import sys -from util import add_env_path, executable_suffix, make_env, third_party_path - -prebuilt_path = os.path.join(third_party_path, "prebuilt") -python_packages_path = os.path.join(third_party_path, "python_packages") - -python_site_env = None - - -# Creates/modifies an environment so python can find packages that are bundled -# in the 'third_party' directory. -def python_env(env=None, merge_env=None): - if merge_env is None: - merge_env = {} - global python_site_env - - # Use site.addsitedir() to determine which search paths would be considered - # if 'third_party/python_packages' was a site-packages directory. - # PATH is also updated, so windows can find the DLLs that ship with pywin32. - if python_site_env is None: - python_site_env = {} - temp = os.environ["PATH"], sys.path - os.environ["PATH"], sys.path = "", [] - site.addsitedir(python_packages_path) # Modifies PATH and sys.path. - python_site_env = {"PATH": os.environ["PATH"], "PYTHONPATH": sys.path} - os.environ["PATH"], sys.path = temp - - # Make a new environment object. - env = make_env(env=env, merge_env=merge_env) - # Apply PATH and PYTHONPATH from the site-packages environment. - add_env_path(python_site_env["PATH"], env=env, key="PATH") - add_env_path(python_site_env["PYTHONPATH"], env=env, key="PYTHONPATH") - - return env - - -def get_platform_dir_name(): - if sys.platform == "win32": - return "win" - elif sys.platform == "darwin": - return "mac" - elif sys.platform.startswith("linux"): - return "linux64" - - -def get_prebuilt_tool_path(tool): - return os.path.join(prebuilt_path, get_platform_dir_name(), - tool + executable_suffix) diff --git a/tools/util.js b/tools/util.js new file mode 100644 index 0000000000..41236e5794 --- /dev/null +++ b/tools/util.js @@ -0,0 +1,114 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { + dirname, + fromFileUrl, + join, +} from "https://deno.land/std@0.76.0/path/mod.ts"; +export { dirname, join }; +export { existsSync } from "https://deno.land/std@0.76.0/fs/mod.ts"; + +export const ROOT_PATH = dirname(dirname(fromFileUrl(import.meta.url))); + +async function getFilesFromGit(baseDir, cmd) { + const p = Deno.run({ + cmd, + stdout: "piped", + }); + const { success } = await p.status(); + if (!success) { + throw new Error("gitLsFiles failed"); + } + + const output = new TextDecoder().decode(await p.output()); + p.close(); + + const files = output.split("\0").filter((line) => line.length > 0).map( + (filePath) => { + return Deno.realPathSync(join(baseDir, filePath)); + }, + ); + + return files; +} + +async function gitLsFiles(baseDir, patterns) { + baseDir = Deno.realPathSync(baseDir); + const cmd = [ + "git", + "-C", + baseDir, + "ls-files", + "-z", + "--exclude-standard", + "--cached", + "--modified", + "--others", + "--", + ...patterns, + ]; + return getFilesFromGit(baseDir, cmd); +} + +/** List all files staged for commit */ +async function gitStaged(baseDir, patterns) { + baseDir = Deno.realPathSync(baseDir); + const cmd = [ + "git", + "-C", + baseDir, + "diff", + "--staged", + "--diff-filter=ACMR", + "--name-only", + "-z", + "--", + ...patterns, + ]; + return getFilesFromGit(baseDir, cmd); +} + +/** + * Recursively list all files in (a subdirectory of) a git worktree. + * * Optionally, glob patterns may be specified to e.g. only list files with a + * certain extension. + * * Untracked files are included, unless they're listed in .gitignore. + * * Directory names themselves are not listed (but the files inside are). + * * Submodules and their contents are ignored entirely. + * * This function fails if the query matches no files. + * + * If --staged argument was provided when program is run + * only staged sources will be returned. + */ +export async function getSources(baseDir, patterns) { + const stagedOnly = Deno.args.includes("--staged"); + + if (stagedOnly) { + return await gitStaged(baseDir, patterns); + } else { + return await gitLsFiles(baseDir, patterns); + } +} + +export function buildMode() { + if (Deno.args.includes("--release")) { + return "release"; + } + + return "debug"; +} + +export function buildPath() { + return join(ROOT_PATH, "target", buildMode()); +} + +export function getPrebuiltToolPath(toolName) { + const PREBUILT_PATH = join(ROOT_PATH, "third_party", "prebuilt"); + + const platformDirName = { + "windows": "win", + "darwin": "mac", + "linux": "linux64", + }[Deno.build.os]; + const executableSuffix = Deno.build.os === "windows" ? ".exe" : ""; + return join(PREBUILT_PATH, platformDirName, toolName + executableSuffix); +} diff --git a/tools/util.py b/tools/util.py deleted file mode 100644 index f1dc138c7e..0000000000 --- a/tools/util.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import collections -import os -import re -import shutil -import select -import stat -import sys -import subprocess -import tempfile -import time - -if os.environ.get("NO_COLOR", None): - RESET = FG_READ = FG_GREEN = "" -else: - RESET = "\x1b[0m" - FG_RED = "\x1b[31m" - FG_GREEN = "\x1b[32m" - -executable_suffix = ".exe" if os.name == "nt" else "" - -root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -third_party_path = os.path.join(root_path, "third_party") - - -def make_env(merge_env=None, env=None): - if env is None: - env = os.environ - env = env.copy() - if merge_env is None: - merge_env = {} - for key in merge_env.keys(): - env[key] = merge_env[key] - return env - - -def add_env_path(add, env, key="PATH", prepend=False): - dirs_left = env[key].split(os.pathsep) if key in env else [] - dirs_right = add.split(os.pathsep) if isinstance(add, str) else add - - if prepend: - dirs_left, dirs_right = dirs_right, dirs_left - - for d in dirs_right: - if not d in dirs_left: - dirs_left += [d] - - env[key] = os.pathsep.join(dirs_left) - - -def run(args, quiet=False, cwd=None, env=None, merge_env=None, shell=None): - args[0] = os.path.normpath(args[0]) - env = make_env(env=env, merge_env=merge_env) - if shell is None: - # Use the default value for 'shell' parameter. - # - Posix: do not use shell. - # - Windows: use shell; this makes .bat/.cmd files work. - shell = os.name == "nt" - if not quiet: - print(" ".join([shell_quote(arg) for arg in args])) - rc = subprocess.call(args, cwd=cwd, env=env, shell=shell) - if rc != 0: - sys.exit(rc) - - -CmdResult = collections.namedtuple('CmdResult', ['out', 'err', 'code']) - - -def run_output(args, - quiet=False, - cwd=None, - env=None, - merge_env=None, - exit_on_fail=False): - if merge_env is None: - merge_env = {} - args[0] = os.path.normpath(args[0]) - if not quiet: - print(" ".join(args)) - env = make_env(env=env, merge_env=merge_env) - shell = os.name == "nt" # Run through shell to make .bat/.cmd files work. - p = subprocess.Popen( - args, - cwd=cwd, - env=env, - shell=shell, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: - out, err = p.communicate() - except subprocess.CalledProcessError as e: - p.kill() - p.wait() - raise e - retcode = p.poll() - if retcode and exit_on_fail: - sys.exit(retcode) - # Ignore Windows CRLF (\r\n). - return CmdResult( - out.replace('\r\n', '\n'), err.replace('\r\n', '\n'), retcode) - - -def shell_quote_win(arg): - if re.search(r'[\x00-\x20"^%~!@&?*<>|()=]', arg): - # Double all " quote characters. - arg = arg.replace('"', '""') - # Wrap the entire string in " quotes. - arg = '"' + arg + '"' - # Double any N backslashes that are immediately followed by a " quote. - arg = re.sub(r'(\\+)(?=")', r'\1\1', arg) - return arg - - -def shell_quote(arg): - if os.name == "nt": - return shell_quote_win(arg) - else: - # Python 2 has posix shell quoting built in, albeit in a weird place. - from pipes import quote - return quote(arg) - - -# Recursively list all files in (a subdirectory of) a git worktree. -# * Optionally, glob patterns may be specified to e.g. only list files with a -# certain extension. -# * Untracked files are included, unless they're listed in .gitignore. -# * Directory names themselves are not listed (but the files inside are). -# * Submodules and their contents are ignored entirely. -# * This function fails if the query matches no files. -def git_ls_files(base_dir, patterns=None): - base_dir = os.path.abspath(base_dir) - args = [ - "git", "-C", base_dir, "ls-files", "-z", "--exclude-standard", - "--cached", "--modified", "--others" - ] - if patterns: - args += ["--"] + patterns - output = subprocess.check_output(args) - files = [ - os.path.normpath(os.path.join(base_dir, f)) for f in output.split("\0") - if f != "" - ] - if not files: - raise RuntimeError("git_ls_files: no files in '%s'" % base_dir + - (" matching %s" % patterns if patterns else "")) - return files - - -# list all files staged for commit -def git_staged(base_dir, patterns=None): - base_dir = os.path.abspath(base_dir) - args = [ - "git", "-C", base_dir, "diff", "--staged", "--diff-filter=ACMR", - "--name-only", "-z" - ] - if patterns: - args += ["--"] + patterns - output = subprocess.check_output(args) - files = [ - os.path.normpath(os.path.join(base_dir, f)) for f in output.split("\0") - if f != "" - ] - return files - - -def build_mode(): - if "--release" in sys.argv: - return "release" - else: - return "debug" - - -# E.G. "target/debug" -def build_path(): - return os.path.join(root_path, "target", build_mode()) - - -# Attempts to enable ANSI escape code support. -# Returns True if successful, False if not supported. -def enable_ansi_colors(): - if os.name != 'nt': - return True # On non-windows platforms this just works. - return enable_ansi_colors_win10() - - -# The windows 10 implementation of enable_ansi_colors. -def enable_ansi_colors_win10(): - import ctypes - - # Function factory for errcheck callbacks that raise WinError on failure. - def raise_if(error_result): - def check(result, _func, args): - if result == error_result: - raise ctypes.WinError(ctypes.get_last_error()) - return args - - return check - - # Windows API types. - from ctypes.wintypes import BOOL, DWORD, HANDLE, LPCWSTR, LPVOID - LPDWORD = ctypes.POINTER(DWORD) - - # Generic constants. - NULL = ctypes.c_void_p(0).value - INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value - ERROR_INVALID_PARAMETER = 87 - - # CreateFile flags. - # yapf: disable - GENERIC_READ = 0x80000000 - GENERIC_WRITE = 0x40000000 - FILE_SHARE_READ = 0x01 - FILE_SHARE_WRITE = 0x02 - OPEN_EXISTING = 3 - # yapf: enable - - # Get/SetConsoleMode flags. - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 - - kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - - # HANDLE CreateFileW(...) - CreateFileW = kernel32.CreateFileW - CreateFileW.restype = HANDLE - CreateFileW.errcheck = raise_if(INVALID_HANDLE_VALUE) - # yapf: disable - CreateFileW.argtypes = (LPCWSTR, # lpFileName - DWORD, # dwDesiredAccess - DWORD, # dwShareMode - LPVOID, # lpSecurityAttributes - DWORD, # dwCreationDisposition - DWORD, # dwFlagsAndAttributes - HANDLE) # hTemplateFile - # yapf: enable - - # BOOL CloseHandle(HANDLE hObject) - CloseHandle = kernel32.CloseHandle - CloseHandle.restype = BOOL - CloseHandle.errcheck = raise_if(False) - CloseHandle.argtypes = (HANDLE, ) - - # BOOL GetConsoleMode(HANDLE hConsoleHandle, LPDWORD lpMode) - GetConsoleMode = kernel32.GetConsoleMode - GetConsoleMode.restype = BOOL - GetConsoleMode.errcheck = raise_if(False) - GetConsoleMode.argtypes = (HANDLE, LPDWORD) - - # BOOL SetConsoleMode(HANDLE hConsoleHandle, DWORD dwMode) - SetConsoleMode = kernel32.SetConsoleMode - SetConsoleMode.restype = BOOL - SetConsoleMode.errcheck = raise_if(False) - SetConsoleMode.argtypes = (HANDLE, DWORD) - - # Open the console output device. - conout = CreateFileW("CONOUT$", GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, - OPEN_EXISTING, 0, 0) - - # Get the current mode. - mode = DWORD() - GetConsoleMode(conout, ctypes.byref(mode)) - - # Try to set the flag that controls ANSI escape code support. - try: - SetConsoleMode(conout, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) - except WindowsError as e: # pylint:disable=undefined-variable - if e.winerror == ERROR_INVALID_PARAMETER: - return False # Not supported, likely an older version of Windows. - raise - finally: - CloseHandle(conout) - - return True - - -def print_command(cmd, files): - noun = "file" if len(files) == 1 else "files" - print("%s (%d %s)" % (cmd, len(files), noun)) diff --git a/tools/util_test.py b/tools/util_test.py deleted file mode 100755 index 7f5433114f..0000000000 --- a/tools/util_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import os - -from test_util import DenoTestCase, run_tests -from util import (shell_quote_win, root_path) - - -class TestUtil(DenoTestCase): - def test_shell_quote_win(self): - assert shell_quote_win('simple') == 'simple' - assert shell_quote_win( - 'roof/\\isoprojection') == 'roof/\\isoprojection' - assert shell_quote_win('with space') == '"with space"' - assert shell_quote_win('embedded"quote') == '"embedded""quote"' - assert shell_quote_win( - 'a"b""c\\d\\"e\\\\') == '"a""b""""c\\d\\\\""e\\\\\\\\"' - - def test_executable_exists(self): - assert os.path.exists(self.deno_exe) - - -if __name__ == '__main__': - run_tests()