feat(unstable): single checksum per JSR package in the lockfile (#22421)

This changes the lockfile to not store JSR specifiers in the "remote"
section. Instead a single JSR integrity is stored per package in the
lockfile, which is a hash of the version's `x.x.x_meta.json` file, which
contains hashes for every file in the package. The hashes in this file
are then compared against when loading.

Additionally, when using `{ "vendor": true }` in a deno.json, the files
can be modified without causing lockfile errors—the checksum is only
checked when copying into the vendor folder and not afterwards
(eventually we should add this behaviour for non-jsr specifiers as
well). As part of this change, the `vendor` folder creation is not
always automatic in the LSP and running an explicit cache command is
necessary. The code required to track checksums in the LSP would have
been too complex for this PR, so that all goes through deno_graph now.
The vendoring is still automatic when running from the CLI.
This commit is contained in:
David Sherret 2024-02-15 14:49:35 -05:00 committed by GitHub
parent 052b7d8bbd
commit 4f80d83774
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 766 additions and 242 deletions

288
Cargo.lock generated
View file

@ -128,6 +128,19 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
[[package]]
name = "ammonia"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170"
dependencies = [
"html5ever",
"maplit",
"once_cell",
"tendril",
"url",
]
[[package]]
name = "android_system_properties"
version = "0.1.5"
@ -1061,7 +1074,7 @@ dependencies = [
"open",
"os_pipe",
"percent-encoding",
"phf",
"phf 0.11.2",
"pin-project",
"pretty_assertions",
"quick-junit",
@ -1097,9 +1110,9 @@ dependencies = [
[[package]]
name = "deno_ast"
version = "0.33.2"
version = "0.33.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fdafff817ae3ad89672d54cd8daebc86dc352065ccc18691605043e6b845d00"
checksum = "7f61944e781d268799bf65857e664d3c09a37590043d4b0ed10facefc9bea473"
dependencies = [
"anyhow",
"base64",
@ -1172,9 +1185,9 @@ dependencies = [
[[package]]
name = "deno_cache_dir"
version = "0.6.1"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bbb245d9a3719b5eb2b5195aaaa25108c3c93d1762b181a20fb1af1c7703eaf"
checksum = "6cf517bddfd22d79d0f284500318e3f9aea193536c2b61cbf6ce7b50a85f1b6a"
dependencies = [
"anyhow",
"deno_media_type",
@ -1182,9 +1195,9 @@ dependencies = [
"log",
"once_cell",
"parking_lot 0.12.1",
"ring",
"serde",
"serde_json",
"sha2",
"thiserror",
"url",
]
@ -1310,10 +1323,11 @@ dependencies = [
[[package]]
name = "deno_doc"
version = "0.103.0"
version = "0.107.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73fe6bd8144456ca3f01b8d1cd1b668b974c84dc94cb642936c0938348b17017"
checksum = "f82478f27de7958eb6a1e48e447b8cb030a1294097ef510eec190d29e81f330f"
dependencies = [
"ammonia",
"anyhow",
"cfg-if",
"comrak",
@ -1328,15 +1342,14 @@ dependencies = [
"regex",
"serde",
"serde_json",
"syntect",
"termcolor",
]
[[package]]
name = "deno_emit"
version = "0.36.0"
version = "0.37.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5002f2c25489fb993132dc0cb0dabd41bae70a8629168db4bd726ee2e296ac"
checksum = "a670c56f233f85f18f1d4a3288c5241505d8aea559fe3870b45e00d4c0e731dc"
dependencies = [
"anyhow",
"base64",
@ -1404,9 +1417,9 @@ dependencies = [
[[package]]
name = "deno_graph"
version = "0.65.3"
version = "0.66.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d12c87f92df950ad0eed3ea8951f30bf9c54f69b3a903b805950d6761b35002"
checksum = "8c67c7c05d70e43560b1dfa38ee385d2d0153ccd4ea16fdc6a706881fd60f3c5"
dependencies = [
"anyhow",
"async-trait",
@ -1424,6 +1437,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
"sha2",
"thiserror",
"url",
]
@ -1455,7 +1469,7 @@ dependencies = [
"mime",
"once_cell",
"percent-encoding",
"phf",
"phf 0.11.2",
"pin-project",
"rand",
"ring",
@ -1531,13 +1545,13 @@ dependencies = [
[[package]]
name = "deno_lockfile"
version = "0.18.2"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f348633cc4425b2a9011436e256b1ae8f6c8026ec2705d852baee8643dc5562"
checksum = "8835418ae924f25ab20f508bf6240193b22d893519d44432b670a27b8fb1efeb"
dependencies = [
"ring",
"serde",
"serde_json",
"sha2",
"thiserror",
]
@ -1659,9 +1673,9 @@ dependencies = [
[[package]]
name = "deno_npm"
version = "0.16.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "376262760b173ff01f8f5d05d58a64f6d863472396afb5582590fa0949342854"
checksum = "53a333104d3fb6aa52e499384e523aefc09d3ac8ecd05ca7f65f856044fbcb09"
dependencies = [
"anyhow",
"async-trait",
@ -2414,9 +2428,9 @@ dependencies = [
[[package]]
name = "eszip"
version = "0.62.0"
version = "0.63.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a26aa6791e6021e9e3ffc6bc8ab00ff2d0d748c64a75b7333076d973ce32f6b"
checksum = "731a0e44e886cb8efbbd63b8121341d505e9dab855fe487249d70c362a6bd774"
dependencies = [
"anyhow",
"base64",
@ -2648,6 +2662,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "futf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
dependencies = [
"mac",
"new_debug_unreachable",
]
[[package]]
name = "futures"
version = "0.3.29"
@ -3059,7 +3083,7 @@ checksum = "de90d3db62411eb62eddabe402d706ac4970f7ac8d088c05f11069cad9be9857"
dependencies = [
"new_debug_unreachable",
"once_cell",
"phf",
"phf 0.11.2",
"rustc-hash",
"smallvec",
]
@ -3073,6 +3097,20 @@ dependencies = [
"utf8-width",
]
[[package]]
name = "html5ever"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"
dependencies = [
"log",
"mac",
"markup5ever",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "http"
version = "0.2.11"
@ -3731,6 +3769,12 @@ dependencies = [
"url",
]
[[package]]
name = "mac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "malloc_buf"
version = "0.0.6"
@ -3740,6 +3784,26 @@ dependencies = [
"libc",
]
[[package]]
name = "maplit"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
[[package]]
name = "markup5ever"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"
dependencies = [
"log",
"phf 0.10.1",
"phf_codegen",
"string_cache",
"string_cache_codegen",
"tendril",
]
[[package]]
name = "match_cfg"
version = "0.1.0"
@ -4147,28 +4211,6 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "onig"
version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
dependencies = [
"bitflags 1.3.2",
"libc",
"once_cell",
"onig_sys",
]
[[package]]
name = "onig_sys"
version = "69.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
dependencies = [
"cc",
"pkg-config",
]
[[package]]
name = "opaque-debug"
version = "0.3.0"
@ -4433,6 +4475,15 @@ dependencies = [
"indexmap",
]
[[package]]
name = "phf"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
dependencies = [
"phf_shared 0.10.0",
]
[[package]]
name = "phf"
version = "0.11.2"
@ -4440,7 +4491,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_macros",
"phf_shared",
"phf_shared 0.11.2",
]
[[package]]
name = "phf_codegen"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
]
[[package]]
name = "phf_generator"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
dependencies = [
"phf_shared 0.10.0",
"rand",
]
[[package]]
@ -4449,7 +4520,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
dependencies = [
"phf_shared",
"phf_shared 0.11.2",
"rand",
]
@ -4459,13 +4530,22 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
dependencies = [
"phf_generator",
"phf_shared",
"phf_generator 0.11.2",
"phf_shared 0.11.2",
"proc-macro2",
"quote",
"syn 2.0.48",
]
[[package]]
name = "phf_shared"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.11.2"
@ -4588,6 +4668,12 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "presser"
version = "0.3.1"
@ -4923,7 +5009,7 @@ dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax 0.8.2",
"regex-syntax",
]
[[package]]
@ -4934,15 +5020,9 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.2",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "regex-syntax"
version = "0.8.2"
@ -5703,6 +5783,32 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "string_cache"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
dependencies = [
"new_debug_unreachable",
"once_cell",
"parking_lot 0.12.1",
"phf_shared 0.10.0",
"precomputed-hash",
"serde",
]
[[package]]
name = "string_cache_codegen"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
"proc-macro2",
"quote",
]
[[package]]
name = "string_enum"
version = "0.4.2"
@ -5772,9 +5878,9 @@ dependencies = [
[[package]]
name = "swc_bundler"
version = "0.225.3"
version = "0.225.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26491762e84ae1d0a2e179fe48066072834777a1b12e8e88a7f07c8f92cc0188"
checksum = "feb8b6f3ad184a5ae21544411491bf136635237fc097d7c93ccd915449ebb2ba"
dependencies = [
"anyhow",
"crc",
@ -5875,7 +5981,7 @@ dependencies = [
"bitflags 2.4.1",
"is-macro",
"num-bigint",
"phf",
"phf 0.11.2",
"scoped-tls",
"serde",
"string_enum",
@ -5939,7 +6045,7 @@ dependencies = [
"new_debug_unreachable",
"num-bigint",
"num-traits",
"phf",
"phf 0.11.2",
"serde",
"smallvec",
"smartstring",
@ -5953,15 +6059,15 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_base"
version = "0.137.3"
version = "0.137.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfd47dd9ccb73a1f5d8d7eff9518554b752b1733b56503af090e78859abb42dd"
checksum = "803bb435fdd532d5c931f0d487e48dbc94750d26c9336d79a6f1c04c62f08d93"
dependencies = [
"better_scoped_tls",
"bitflags 2.4.1",
"indexmap",
"once_cell",
"phf",
"phf 0.11.2",
"rustc-hash",
"serde",
"smallvec",
@ -5976,9 +6082,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_classes"
version = "0.126.3"
version = "0.126.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb31417e0d415d7f0ff026f1e7c909427e386b7d0af9a2a78678507e4d9d79"
checksum = "486479e75907547d4c65ca6deed8faa465a2e9475cc9605be36a0e5eb609f578"
dependencies = [
"swc_atoms",
"swc_common",
@ -6002,9 +6108,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_optimization"
version = "0.198.3"
version = "0.198.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3920268ac8972b494067d0b7c088964b21d08f5d1f58d7151bd1eb7054a137b0"
checksum = "f3c6fffe4d6e3609fdd6c768cc063dbc9f5101f9be0db1168ec76ace979cf616"
dependencies = [
"dashmap",
"indexmap",
@ -6026,9 +6132,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_proposal"
version = "0.171.3"
version = "0.171.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "448c40c2a2b224cb5101cc6cdee81837c281a34f2a2aa6dd18d6d5cd8d492e60"
checksum = "e2822bc6c28bb1a96090a3b2caa28f45efbaecb333714d30868a2390eaae943f"
dependencies = [
"either",
"rustc-hash",
@ -6046,9 +6152,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_react"
version = "0.183.3"
version = "0.183.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2394dc3abceada246feeb709b8c4d23392973f49a24fcc59b2ee21737cb6c8"
checksum = "8984ebb8955116c426457a0c70b0aa9a08a06656e245781ff617a9cd1e289697"
dependencies = [
"base64",
"dashmap",
@ -6070,9 +6176,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_typescript"
version = "0.188.3"
version = "0.188.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cff231437173e041e5a3be9b8c782fd297ffcb53ed16d805f853e4a68315c45"
checksum = "a2e898fbab993abb60fb67009521908f2317f1d33f804e5b38d769f52e58572e"
dependencies = [
"ryu-js",
"serde",
@ -6087,9 +6193,9 @@ dependencies = [
[[package]]
name = "swc_ecma_utils"
version = "0.127.3"
version = "0.127.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd185161161dfc65ee0d6f3044c901b766c3abb4efcd0b35c9e76c833724896"
checksum = "8ff9e77ea18468895d26bd38656885860fede2acd24d1687f64363aaf8910441"
dependencies = [
"indexmap",
"num_cpus",
@ -6222,25 +6328,6 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "syntect"
version = "5.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e02b4b303bf8d08bfeb0445cba5068a3d306b6baece1d5582171a9bf49188f91"
dependencies = [
"bincode",
"bitflags 1.3.2",
"flate2",
"fnv",
"once_cell",
"onig",
"regex-syntax 0.7.5",
"serde",
"serde_json",
"thiserror",
"walkdir",
]
[[package]]
name = "tar"
version = "0.4.40"
@ -6265,6 +6352,17 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "tendril"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
dependencies = [
"futf",
"mac",
"utf-8",
]
[[package]]
name = "termcolor"
version = "1.4.0"
@ -6324,12 +6422,12 @@ dependencies = [
"prost-build",
"regex",
"reqwest",
"ring",
"rustls-pemfile",
"rustls-tokio-stream",
"semver 1.0.14",
"serde",
"serde_json",
"sha2",
"tar",
"tempfile",
"termcolor",

View file

@ -46,7 +46,7 @@ deno_ast = { version = "0.33.2", features = ["transpiling"] }
deno_core = { version = "0.262.0" }
deno_bench_util = { version = "0.132.0", path = "./bench_util" }
deno_lockfile = "0.18.2"
deno_lockfile = "0.19.0"
deno_media_type = { version = "0.1.1", features = ["module_specifier"] }
deno_runtime = { version = "0.146.0", path = "./runtime" }
deno_terminal = "0.1.1"
@ -97,7 +97,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde"
console_static_text = "=0.8.1"
data-encoding = "2.3.3"
data-url = "=0.3.0"
deno_cache_dir = "=0.6.1"
deno_cache_dir = "=0.7.1"
dlopen2 = "0.6.1"
ecb = "=0.1.2"
elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem"] }

View file

@ -66,17 +66,17 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa
deno_cache_dir = { workspace = true }
deno_config = "=0.9.2"
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_doc = { version = "=0.103.0", features = ["html"] }
deno_emit = "=0.36.0"
deno_graph = "=0.65.3"
deno_doc = { version = "=0.107.0", features = ["html"] }
deno_emit = "=0.37.0"
deno_graph = "=0.66.0"
deno_lint = { version = "=0.56.0", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm = "=0.16.0"
deno_npm = "=0.17.0"
deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_semver = "=0.5.4"
deno_task_shell = "=0.14.3"
deno_terminal.workspace = true
eszip = "=0.62.0"
eszip = "=0.63.0"
napi_sym.workspace = true
async-trait.workspace = true

8
cli/cache/mod.rs vendored
View file

@ -196,8 +196,7 @@ impl Loader for FetchCacher {
fn load(
&mut self,
specifier: &ModuleSpecifier,
_is_dynamic: bool,
cache_setting: deno_graph::source::CacheSetting,
options: deno_graph::source::LoadOptions,
) -> LoadFuture {
use deno_graph::source::CacheSetting as LoaderCacheSetting;
@ -222,7 +221,7 @@ impl Loader for FetchCacher {
let specifier = specifier.clone();
async move {
let maybe_cache_setting = match cache_setting {
let maybe_cache_setting = match options.cache_setting {
LoaderCacheSetting::Use => None,
LoaderCacheSetting::Reload => {
if matches!(file_fetcher.cache_setting(), CacheSetting::Only) {
@ -240,6 +239,7 @@ impl Loader for FetchCacher {
permissions,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
maybe_checksum: options.maybe_checksum,
})
.await
.map(|file| {
@ -269,7 +269,7 @@ impl Loader for FetchCacher {
let error_class_name = get_error_class_name(&err);
match error_class_name {
"NotFound" => Ok(None),
"NotCached" if cache_setting == LoaderCacheSetting::Only => Ok(None),
"NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None),
_ => Err(err),
}
})

View file

@ -66,7 +66,6 @@ use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::inspector_server::InspectorServer;
use deno_semver::package::PackageNv;
use import_map::ImportMap;
use log::warn;
use std::future::Future;
@ -381,16 +380,6 @@ impl CliFactory {
no_npm,
no_config: self.options.no_config(),
config,
nv_to_jsr_url: |nv| {
let nv = PackageNv::from_str(nv).ok()?;
Some(
deno_graph::source::recommended_registry_package_url(
crate::args::jsr_url(),
&nv,
)
.to_string(),
)
},
},
);
}

View file

@ -24,6 +24,7 @@ use deno_core::futures::future::FutureExt;
use deno_core::parking_lot::Mutex;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_graph::source::LoaderChecksum;
use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
use deno_runtime::deno_fetch::reqwest::header::ACCEPT;
use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
@ -146,6 +147,7 @@ pub struct FetchOptions<'a> {
pub permissions: PermissionsContainer,
pub maybe_accept: Option<&'a str>,
pub maybe_cache_setting: Option<&'a CacheSetting>,
pub maybe_checksum: Option<LoaderChecksum>,
}
/// A structure for resolving, fetching and caching source files.
@ -199,6 +201,7 @@ impl FileFetcher {
pub fn fetch_cached(
&self,
specifier: &ModuleSpecifier,
maybe_checksum: Option<LoaderChecksum>,
redirect_limit: i64,
) -> Result<Option<File>, AnyError> {
debug!("FileFetcher::fetch_cached - specifier: {}", specifier);
@ -207,16 +210,22 @@ impl FileFetcher {
}
let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once
let Some(metadata) = self.http_cache.read_metadata(&cache_key)? else {
let Some(headers) = self.http_cache.read_headers(&cache_key)? else {
return Ok(None);
};
let headers = metadata.headers;
if let Some(redirect_to) = headers.get("location") {
let redirect =
deno_core::resolve_import(redirect_to, specifier.as_str())?;
return self.fetch_cached(&redirect, redirect_limit - 1);
return self.fetch_cached(&redirect, maybe_checksum, redirect_limit - 1);
}
let Some(bytes) = self.http_cache.read_file_bytes(&cache_key)? else {
let Some(bytes) = self.http_cache.read_file_bytes(
&cache_key,
maybe_checksum
.as_ref()
.map(|c| deno_cache_dir::Checksum::new(c.as_str())),
deno_cache_dir::GlobalToLocalCopy::Allow,
)?
else {
return Ok(None);
};
@ -282,6 +291,7 @@ impl FileFetcher {
redirect_limit: i64,
maybe_accept: Option<String>,
cache_setting: &CacheSetting,
maybe_checksum: Option<LoaderChecksum>,
) -> Pin<Box<dyn Future<Output = Result<File, AnyError>> + Send>> {
debug!("FileFetcher::fetch_remote() - specifier: {}", specifier);
if redirect_limit < 0 {
@ -294,7 +304,8 @@ impl FileFetcher {
}
if self.should_use_cache(specifier, cache_setting) {
match self.fetch_cached(specifier, redirect_limit) {
match self.fetch_cached(specifier, maybe_checksum.clone(), redirect_limit)
{
Ok(Some(file)) => {
return futures::future::ok(file).boxed();
}
@ -331,8 +342,8 @@ impl FileFetcher {
.http_cache
.cache_item_key(specifier)
.ok()
.and_then(|key| self.http_cache.read_metadata(&key).ok().flatten())
.and_then(|metadata| metadata.headers.get("etag").cloned());
.and_then(|key| self.http_cache.read_headers(&key).ok().flatten())
.and_then(|headers| headers.get("etag").cloned());
let maybe_auth_token = self.auth_tokens.get(specifier);
let specifier = specifier.clone();
let client = self.http_client.clone();
@ -376,7 +387,9 @@ impl FileFetcher {
.await?
{
FetchOnceResult::NotModified => {
let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap();
let file = file_fetcher
.fetch_cached(&specifier, maybe_checksum, 10)?
.unwrap();
Ok(file)
}
FetchOnceResult::Redirect(redirect_url, headers) => {
@ -388,6 +401,7 @@ impl FileFetcher {
redirect_limit - 1,
maybe_accept,
&cache_setting,
maybe_checksum,
)
.await
}
@ -395,6 +409,9 @@ impl FileFetcher {
file_fetcher
.http_cache
.set(&specifier, headers.clone(), &bytes)?;
if let Some(checksum) = &maybe_checksum {
checksum.check_source(&bytes)?;
}
Ok(File {
specifier,
maybe_headers: Some(headers),
@ -438,15 +455,16 @@ impl FileFetcher {
let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else {
return false;
};
let Ok(Some(metadata)) = self.http_cache.read_metadata(&cache_key)
let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else {
return false;
};
let Ok(Some(download_time)) =
self.http_cache.read_download_time(&cache_key)
else {
return false;
};
let cache_semantics = CacheSemantics::new(
metadata.headers,
metadata.time,
SystemTime::now(),
);
let cache_semantics =
CacheSemantics::new(headers, download_time, SystemTime::now());
cache_semantics.should_use()
}
CacheSetting::ReloadSome(list) => {
@ -482,6 +500,7 @@ impl FileFetcher {
permissions,
maybe_accept: None,
maybe_cache_setting: None,
maybe_checksum: None,
})
.await
}
@ -517,6 +536,7 @@ impl FileFetcher {
10,
options.maybe_accept.map(String::from),
options.maybe_cache_setting.unwrap_or(&self.cache_setting),
options.maybe_checksum,
)
.await
}
@ -728,6 +748,7 @@ mod tests {
1,
None,
&file_fetcher.cache_setting,
None,
)
.await;
let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap();
@ -735,10 +756,9 @@ mod tests {
result.unwrap(),
file_fetcher
.http_cache
.read_metadata(&cache_key)
.read_headers(&cache_key)
.unwrap()
.unwrap()
.headers,
.unwrap(),
)
}
@ -899,18 +919,11 @@ mod tests {
let cache_item_key =
file_fetcher.http_cache.cache_item_key(&specifier).unwrap();
let mut metadata = file_fetcher
.http_cache
.read_metadata(&cache_item_key)
.unwrap()
.unwrap();
metadata.headers = HashMap::new();
metadata
.headers
.insert("content-type".to_string(), "text/javascript".to_string());
let mut headers = HashMap::new();
headers.insert("content-type".to_string(), "text/javascript".to_string());
file_fetcher
.http_cache
.set(&specifier, metadata.headers.clone(), file.source.as_bytes())
.set(&specifier, headers.clone(), file.source.as_bytes())
.unwrap();
let result = file_fetcher_01
@ -926,20 +939,17 @@ mod tests {
// the value above.
assert_eq!(file.media_type, MediaType::JavaScript);
let headers = file_fetcher_02
let headers2 = file_fetcher_02
.http_cache
.read_metadata(&cache_item_key)
.read_headers(&cache_item_key)
.unwrap()
.unwrap()
.headers;
assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
metadata.headers = HashMap::new();
metadata
.headers
.insert("content-type".to_string(), "application/json".to_string());
.unwrap();
assert_eq!(headers2.get("content-type").unwrap(), "text/javascript");
headers = HashMap::new();
headers.insert("content-type".to_string(), "application/json".to_string());
file_fetcher_02
.http_cache
.set(&specifier, metadata.headers.clone(), file.source.as_bytes())
.set(&specifier, headers.clone(), file.source.as_bytes())
.unwrap();
let result = file_fetcher_02
@ -1013,7 +1023,12 @@ mod tests {
.unwrap(),
file_fetcher
.http_cache
.read_metadata(&cache_key)
.read_headers(&cache_key)
.unwrap()
.unwrap(),
file_fetcher
.http_cache
.read_download_time(&cache_key)
.unwrap()
.unwrap(),
)
@ -1045,7 +1060,12 @@ mod tests {
.unwrap(),
file_fetcher
.http_cache
.read_metadata(&cache_key)
.read_headers(&cache_key)
.unwrap()
.unwrap(),
file_fetcher
.http_cache
.read_download_time(&cache_key)
.unwrap()
.unwrap(),
)
@ -1182,7 +1202,12 @@ mod tests {
.unwrap(),
file_fetcher
.http_cache
.read_metadata(&cache_key)
.read_headers(&cache_key)
.unwrap()
.unwrap(),
file_fetcher
.http_cache
.read_download_time(&cache_key)
.unwrap()
.unwrap(),
)
@ -1216,7 +1241,12 @@ mod tests {
.unwrap(),
file_fetcher
.http_cache
.read_metadata(&cache_key)
.read_headers(&cache_key)
.unwrap()
.unwrap(),
file_fetcher
.http_cache
.read_download_time(&cache_key)
.unwrap()
.unwrap(),
)
@ -1240,6 +1270,7 @@ mod tests {
2,
None,
&file_fetcher.cache_setting,
None,
)
.await;
assert!(result.is_ok());
@ -1251,14 +1282,15 @@ mod tests {
1,
None,
&file_fetcher.cache_setting,
None,
)
.await;
assert!(result.is_err());
let result = file_fetcher.fetch_cached(&specifier, 2);
let result = file_fetcher.fetch_cached(&specifier, None, 2);
assert!(result.is_ok());
let result = file_fetcher.fetch_cached(&specifier, 1);
let result = file_fetcher.fetch_cached(&specifier, None, 1);
assert!(result.is_err());
}
@ -2072,7 +2104,11 @@ mod tests {
let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap();
let bytes = file_fetcher
.http_cache
.read_file_bytes(&cache_key)
.read_file_bytes(
&cache_key,
None,
deno_cache_dir::GlobalToLocalCopy::Allow,
)
.unwrap()
.unwrap();
String::from_utf8(bytes).unwrap()
@ -2086,10 +2122,9 @@ mod tests {
let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap();
file_fetcher
.http_cache
.read_metadata(&cache_key)
.read_headers(&cache_key)
.unwrap()
.unwrap()
.headers
.remove("location")
}
}

View file

@ -1,5 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::args::jsr_url;
use crate::args::CliOptions;
use crate::args::Lockfile;
use crate::args::TsTypeLib;
@ -174,6 +175,18 @@ pub fn graph_lock_or_exit(graph: &ModuleGraph, lockfile: &mut Lockfile) {
Module::Json(module) => &module.source,
Module::Node(_) | Module::Npm(_) | Module::External(_) => continue,
};
// skip over any specifiers in JSR packages because those
// are enforced via the integrity
if deno_graph::source::recommended_registry_package_url_to_nv(
jsr_url(),
module.specifier(),
)
.is_some()
{
continue;
}
if !lockfile.check_or_insert_remote(module.specifier().as_str(), source) {
let err = format!(
concat!(
@ -475,6 +488,19 @@ impl ModuleGraphBuilder {
}
}
}
for (nv, value) in &lockfile.content.packages.jsr {
if let Ok(nv) = PackageNv::from_str(nv) {
graph
.packages
.add_manifest_checksum(nv, value.integrity.clone())
.map_err(|err| deno_lockfile::IntegrityCheckFailedError {
package_display_id: format!("jsr:{}", err.nv),
actual: err.actual,
expected: err.expected,
filename: lockfile.filename.display().to_string(),
})?;
}
}
}
}
@ -504,9 +530,14 @@ impl ModuleGraphBuilder {
format!("jsr:{}", to),
);
}
for (name, deps) in graph.packages.package_deps() {
lockfile
.insert_package_deps(name.to_string(), deps.map(|s| s.to_string()));
for (name, checksum, deps) in
graph.packages.packages_with_checksum_and_deps()
{
lockfile.insert_package(
name.to_string(),
checksum.clone(),
deps.map(|s| s.to_string()),
);
}
}
}

View file

@ -11,6 +11,16 @@ use std::path::Path;
use std::sync::Arc;
use std::time::SystemTime;
/// In the LSP, we disallow the cache from automatically copying from
/// the global cache to the local cache for technical reasons.
///
/// 1. We need to verify the checksums from the lockfile are correct when
/// moving from the global to the local cache.
/// 2. We need to verify the checksums for JSR https specifiers match what
/// is found in the package's manifest.
pub const LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY: deno_cache_dir::GlobalToLocalCopy =
deno_cache_dir::GlobalToLocalCopy::Disallow;
pub fn calculate_fs_version(
cache: &Arc<dyn HttpCache>,
specifier: &ModuleSpecifier,
@ -123,8 +133,8 @@ impl CacheMetadata {
return None;
}
let cache_key = self.cache.cache_item_key(specifier).ok()?;
let specifier_metadata = self.cache.read_metadata(&cache_key).ok()??;
let values = Arc::new(parse_metadata(&specifier_metadata.headers));
let headers = self.cache.read_headers(&cache_key).ok()??;
let values = Arc::new(parse_metadata(&headers));
let version = calculate_fs_version_in_cache(&self.cache, specifier);
let mut metadata_map = self.metadata.lock();
let metadata = Metadata { values, version };

View file

@ -2,6 +2,7 @@
use super::cache::calculate_fs_version;
use super::cache::calculate_fs_version_at_path;
use super::cache::LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY;
use super::jsr_resolver::JsrResolver;
use super::language_server::StateNpmSnapshot;
use super::text::LineIndex;
@ -736,12 +737,7 @@ impl RedirectResolver {
) -> Option<ModuleSpecifier> {
if redirect_limit > 0 {
let cache_key = self.cache.cache_item_key(specifier).ok()?;
let headers = self
.cache
.read_metadata(&cache_key)
.ok()
.flatten()
.map(|m| m.headers)?;
let headers = self.cache.read_headers(&cache_key).ok().flatten()?;
if let Some(location) = headers.get("location") {
let redirect =
deno_core::resolve_import(location, specifier.as_str()).ok()?;
@ -822,12 +818,14 @@ impl FileSystemDocuments {
} else {
let fs_version = calculate_fs_version(cache, specifier)?;
let cache_key = cache.cache_item_key(specifier).ok()?;
let bytes = cache.read_file_bytes(&cache_key).ok()??;
let specifier_metadata = cache.read_metadata(&cache_key).ok()??;
let bytes = cache
.read_file_bytes(&cache_key, None, LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY)
.ok()??;
let specifier_headers = cache.read_headers(&cache_key).ok()??;
let (_, maybe_charset) =
deno_graph::source::resolve_media_type_and_charset_from_headers(
specifier,
Some(&specifier_metadata.headers),
Some(&specifier_headers),
);
let content = deno_graph::source::decode_owned_source(
specifier,
@ -835,7 +833,7 @@ impl FileSystemDocuments {
maybe_charset,
)
.ok()?;
let maybe_headers = Some(specifier_metadata.headers);
let maybe_headers = Some(specifier_headers);
Document::new(
specifier.clone(),
fs_version,
@ -1826,8 +1824,7 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> {
fn load(
&mut self,
specifier: &ModuleSpecifier,
is_dynamic: bool,
cache_setting: deno_graph::source::CacheSetting,
options: deno_graph::source::LoadOptions,
) -> deno_graph::source::LoadFuture {
let specifier = if self.unstable_sloppy_imports {
self
@ -1839,9 +1836,7 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> {
match self.load_from_docs(&specifier) {
Some(fut) => fut,
None => self
.inner_loader
.load(&specifier, is_dynamic, cache_setting),
None => self.inner_loader.load(&specifier, options),
}
}

View file

@ -15,6 +15,8 @@ use deno_semver::package::PackageReq;
use std::borrow::Cow;
use std::sync::Arc;
use super::cache::LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY;
#[derive(Debug)]
pub struct JsrResolver {
nv_by_req: DashMap<PackageReq, Option<PackageNv>>,
@ -111,7 +113,13 @@ fn read_cached_package_info(
) -> Option<JsrPackageInfo> {
let meta_url = jsr_url().join(&format!("{}/meta.json", name)).ok()?;
let meta_cache_item_key = cache.cache_item_key(&meta_url).ok()?;
let meta_bytes = cache.read_file_bytes(&meta_cache_item_key).ok()??;
let meta_bytes = cache
.read_file_bytes(
&meta_cache_item_key,
None,
LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY,
)
.ok()??;
serde_json::from_slice::<JsrPackageInfo>(&meta_bytes).ok()
}
@ -123,12 +131,19 @@ fn read_cached_package_version_info(
.join(&format!("{}/{}_meta.json", &nv.name, &nv.version))
.ok()?;
let meta_cache_item_key = cache.cache_item_key(&meta_url).ok()?;
let meta_bytes = cache.read_file_bytes(&meta_cache_item_key).ok()??;
let meta_bytes = cache
.read_file_bytes(
&meta_cache_item_key,
None,
LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY,
)
.ok()??;
// This is a roundabout way of deserializing `JsrPackageVersionInfo`,
// because we only want the `exports` field and `module_graph` is large.
let mut info =
serde_json::from_slice::<serde_json::Value>(&meta_bytes).ok()?;
Some(JsrPackageVersionInfo {
manifest: Default::default(), // not used by the LSP (only caching checks this in deno_graph)
exports: info.as_object_mut()?.remove("exports")?,
module_graph: None,
})

View file

@ -515,6 +515,7 @@ impl ModuleRegistry {
permissions: PermissionsContainer::allow_all(),
maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"),
maybe_cache_setting: None,
maybe_checksum: None,
})
.await;
// if there is an error fetching, we will cache an empty file, so that

View file

@ -523,7 +523,7 @@ pub async fn cover_files(
file_fetcher.get_source(&module_specifier)
} else {
file_fetcher
.fetch_cached(&module_specifier, 10)
.fetch_cached(&module_specifier, None, 10)
.with_context(|| {
format!("Failed to fetch \"{module_specifier}\" from cache.")
})?

View file

@ -211,9 +211,9 @@ impl deno_doc::html::HrefResolver for DocResolver {
fn resolve_usage(
&self,
_current_specifier: &ModuleSpecifier,
current_file: &str,
current_file: Option<&str>,
) -> Option<String> {
Some(current_file.to_string())
current_file.map(|f| f.to_string())
}
fn resolve_source(&self, location: &deno_doc::Location) -> Option<String> {

View file

@ -115,8 +115,7 @@ impl Loader for TestLoader {
fn load(
&mut self,
specifier: &ModuleSpecifier,
_is_dynamic: bool,
_cache_setting: deno_graph::source::CacheSetting,
_options: deno_graph::source::LoadOptions,
) -> LoadFuture {
let specifier = self.redirects.get(specifier).unwrap_or(specifier);
let result = self.files.get(specifier).map(|result| match result {

View file

@ -904,8 +904,7 @@ mod tests {
fn load(
&mut self,
specifier: &ModuleSpecifier,
_is_dynamic: bool,
_cache_setting: deno_graph::source::CacheSetting,
_options: deno_graph::source::LoadOptions,
) -> deno_graph::source::LoadFuture {
let specifier_text = specifier
.to_string()

View file

@ -42,12 +42,12 @@ pretty_assertions.workspace = true
prost.workspace = true
regex.workspace = true
reqwest.workspace = true
ring.workspace = true
rustls-pemfile.workspace = true
rustls-tokio-stream.workspace = true
semver = "=1.0.14"
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tar.workspace = true
tempfile.workspace = true
termcolor.workspace = true

View file

@ -280,6 +280,34 @@ impl TestContext {
.run()
.skip_output_check();
}
pub fn get_jsr_package_integrity(&self, sub_path: &str) -> String {
fn get_checksum(bytes: &[u8]) -> String {
use sha2::Digest;
let mut hasher = sha2::Sha256::new();
hasher.update(bytes);
format!("{:x}", hasher.finalize())
}
let url = url::Url::parse(self.envs.get("JSR_URL").unwrap()).unwrap();
let url = url.join(&format!("{}_meta.json", sub_path)).unwrap();
let bytes = sync_fetch(url);
get_checksum(&bytes)
}
}
fn sync_fetch(url: url::Url) -> bytes::Bytes {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.unwrap();
runtime.block_on(async move {
let client = reqwest::Client::new();
let response = client.get(url).send().await.unwrap();
assert!(response.status().is_success());
response.bytes().await.unwrap()
})
}
/// We can't clone an stdio, so if someone clones a DenoCmd,

View file

@ -64,9 +64,6 @@ impl CustomNpmPackageCache {
}
fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
use ring::digest::Context;
use ring::digest::SHA512;
let package_folder = testdata_path().join("npm/registry").join(package_name);
if !package_folder.exists() {
return Ok(None);
@ -103,10 +100,7 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
}
// get tarball hash
let mut hash_ctx = Context::new(&SHA512);
hash_ctx.update(&tarball_bytes);
let digest = hash_ctx.finish();
let tarball_checksum = BASE64_STANDARD.encode(digest.as_ref());
let tarball_checksum = get_tarball_checksum(&tarball_bytes);
// create the registry file JSON for this version
let mut dist = serde_json::Map::new();
@ -176,3 +170,10 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
tarballs,
}))
}
fn get_tarball_checksum(bytes: &[u8]) -> String {
use sha2::Digest;
let mut hasher = sha2::Sha512::new();
hasher.update(bytes);
BASE64_STANDARD.encode(hasher.finalize())
}

View file

@ -13,9 +13,14 @@ use hyper::body::Incoming;
use hyper::Request;
use hyper::Response;
use hyper::StatusCode;
use once_cell::sync::Lazy;
use serde_json::json;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::convert::Infallible;
use std::net::SocketAddr;
use std::path::Path;
use std::sync::Mutex;
pub async fn registry_server(port: u16) {
let registry_server_addr = SocketAddr::from(([127, 0, 0, 1], port));
@ -66,6 +71,27 @@ async fn registry_server_handler(
testdata_path().to_path_buf().join("jsr").join("registry");
file_path.push(&req.uri().path()[1..].replace("%2f", "/"));
if let Ok(body) = tokio::fs::read(&file_path).await {
let body = if let Some(version) = file_path
.file_name()
.unwrap()
.to_string_lossy()
.strip_suffix("_meta.json")
{
// fill the manifest with checksums found in the directory so that
// we don't need to maintain them manually in the testdata directory
let mut meta: serde_json::Value = serde_json::from_slice(&body)?;
let mut manifest =
manifest_sorted(meta.get("manifest").cloned().unwrap_or(json!({})));
let version_dir = file_path.parent().unwrap().join(version);
fill_manifest_at_dir(&mut manifest, &version_dir);
meta
.as_object_mut()
.unwrap()
.insert("manifest".to_string(), json!(manifest));
serde_json::to_string(&meta).unwrap().into_bytes()
} else {
body
};
return Ok(Response::new(UnsyncBoxBody::new(
http_body_util::Full::new(Bytes::from(body)),
)));
@ -77,3 +103,80 @@ async fn registry_server_handler(
.body(empty_body)?;
Ok(res)
}
fn manifest_sorted(
meta: serde_json::Value,
) -> BTreeMap<String, serde_json::Value> {
let mut manifest = BTreeMap::new();
if let serde_json::Value::Object(files) = meta {
for (file, checksum) in files {
manifest.insert(file.clone(), checksum.clone());
}
}
manifest
}
fn fill_manifest_at_dir(
manifest: &mut BTreeMap<String, serde_json::Value>,
dir: &Path,
) {
let file_system_manifest = get_manifest_entries_for_dir(dir);
for (file_path, value) in file_system_manifest {
manifest.entry(file_path).or_insert(value);
}
}
static DIR_MANIFEST_CACHE: Lazy<
Mutex<HashMap<String, BTreeMap<String, serde_json::Value>>>,
> = Lazy::new(Default::default);
fn get_manifest_entries_for_dir(
dir: &Path,
) -> BTreeMap<String, serde_json::Value> {
fn inner_fill(
root_dir: &Path,
dir: &Path,
manifest: &mut BTreeMap<String, serde_json::Value>,
) {
for entry in std::fs::read_dir(dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_file() {
let file_bytes = std::fs::read(&path).unwrap();
let checksum = format!("sha256-{}", get_checksum(&file_bytes));
let relative_path = path
.to_string_lossy()
.strip_prefix(&root_dir.to_string_lossy().to_string())
.unwrap()
.replace('\\', "/");
manifest.insert(
relative_path,
json!({
"size": file_bytes.len(),
"checksum": checksum,
}),
);
} else if path.is_dir() {
inner_fill(root_dir, &path, manifest);
}
}
}
DIR_MANIFEST_CACHE
.lock()
.unwrap()
.entry(dir.to_string_lossy().to_string())
.or_insert_with(|| {
let mut manifest = BTreeMap::new();
inner_fill(dir, dir, &mut manifest);
manifest
})
.clone()
}
fn get_checksum(bytes: &[u8]) -> String {
use sha2::Digest;
let mut hasher = sha2::Sha256::new();
hasher.update(bytes);
format!("{:x}", hasher.finalize())
}

View file

@ -1,5 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_lockfile::Lockfile;
use test_util as util;
@ -182,3 +183,167 @@ fn reload_info_not_found_cache_but_exists_remote() {
))
.assert_exit_code(0);
}
#[test]
fn lockfile_bad_package_integrity() {
let test_context = TestContextBuilder::for_jsr().use_temp_cwd().build();
let temp_dir = test_context.temp_dir();
temp_dir.write(
"main.ts",
r#"import version from "jsr:@denotest/no_module_graph@0.1";
console.log(version);"#,
);
temp_dir.write("deno.json", "{}"); // to automatically create a lockfile
test_context
.new_command()
.args("run --quiet main.ts")
.run()
.assert_matches_text("0.1.1\n");
let lockfile_path = temp_dir.path().join("deno.lock");
let mut lockfile = Lockfile::new(lockfile_path.to_path_buf(), false).unwrap();
let pkg_name = "@denotest/no_module_graph@0.1.1";
let original_integrity = get_lockfile_pkg_integrity(&lockfile, pkg_name);
set_lockfile_pkg_integrity(&mut lockfile, pkg_name, "bad_integrity");
lockfile_path.write(lockfile.as_json_string());
let actual_integrity =
test_context.get_jsr_package_integrity("@denotest/no_module_graph/0.1.1");
let integrity_check_failed_msg = format!("error: Integrity check failed for http://127.0.0.1:4250/@denotest/no_module_graph/0.1.1_meta.json
Actual: {}
Expected: bad_integrity
at file:///[WILDCARD]/main.ts:1:21
", actual_integrity);
test_context
.new_command()
.args("run --quiet main.ts")
.run()
.assert_matches_text(&integrity_check_failed_msg)
.assert_exit_code(1);
// now try with a vendor folder
temp_dir
.path()
.join("deno.json")
.write_json(&json!({ "vendor": true }));
// should fail again
test_context
.new_command()
.args("run --quiet main.ts")
.run()
.assert_matches_text(&integrity_check_failed_msg)
.assert_exit_code(1);
// now update to the correct integrity
set_lockfile_pkg_integrity(&mut lockfile, pkg_name, &original_integrity);
lockfile_path.write(lockfile.as_json_string());
// should pass now
test_context
.new_command()
.args("run --quiet main.ts")
.run()
.assert_matches_text("0.1.1\n")
.assert_exit_code(0);
// now update to a bad integrity again
set_lockfile_pkg_integrity(&mut lockfile, pkg_name, "bad_integrity");
lockfile_path.write(lockfile.as_json_string());
// shouldn't matter because we have a vendor folder
test_context
.new_command()
.args("run --quiet main.ts")
.run()
.assert_matches_text("0.1.1\n")
.assert_exit_code(0);
// now remove the vendor dir and it should fail again
temp_dir.path().join("vendor").remove_dir_all();
test_context
.new_command()
.args("run --quiet main.ts")
.run()
.assert_matches_text(&integrity_check_failed_msg)
.assert_exit_code(1);
}
#[test]
fn bad_manifest_checksum() {
let test_context = TestContextBuilder::for_jsr().use_temp_cwd().build();
let temp_dir = test_context.temp_dir();
temp_dir.write(
"main.ts",
r#"import { add } from "jsr:@denotest/bad-manifest-checksum@1.0.0";
console.log(add);"#,
);
// test it properly checks the checksum on download
test_context
.new_command()
.args("run main.ts")
.run()
.assert_matches_text(
"Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/meta.json
Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0_meta.json
Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0/mod.ts
error: Integrity check failed.
Actual: 9a30ac96b5d5c1b67eca69e1e2cf0798817d9578c8d7d904a81a67b983b35cba
Expected: bad-checksum
at file:///[WILDCARD]main.ts:1:21
",
)
.assert_exit_code(1);
// test it properly checks the checksum when loading from the cache
test_context
.new_command()
.args("run main.ts")
.run()
.assert_matches_text(
// ideally the two error messages would be the same... this one comes from
// deno_cache and the one above comes from deno_graph. The thing is, in deno_cache
// (source of this error) it makes sense to include the url in the error message
// because it's not always used in the context of deno_graph
"error: Integrity check failed for http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0/mod.ts
Actual: 9a30ac96b5d5c1b67eca69e1e2cf0798817d9578c8d7d904a81a67b983b35cba
Expected: bad-checksum
at file:///[WILDCARD]main.ts:1:21
",
)
.assert_exit_code(1);
}
fn get_lockfile_pkg_integrity(lockfile: &Lockfile, pkg_name: &str) -> String {
lockfile
.content
.packages
.jsr
.get(pkg_name)
.unwrap()
.integrity
.clone()
}
fn set_lockfile_pkg_integrity(
lockfile: &mut Lockfile,
pkg_name: &str,
integrity: &str,
) {
lockfile
.content
.packages
.jsr
.get_mut(pkg_name)
.unwrap()
.integrity = integrity.to_string();
}

View file

@ -693,11 +693,19 @@ fn lsp_format_vendor_path() {
.use_http_server()
.use_temp_cwd()
.build();
// put this dependency in the global cache
context
.new_command()
.args("cache http://localhost:4545/run/002_hello.ts")
.run()
.skip_output_check();
let temp_dir = context.temp_dir();
temp_dir.write("deno.json", json!({ "vendor": true }).to_string());
let mut client = context.new_lsp_command().build();
client.initialize_default();
client.did_open(json!({
let diagnostics = client.did_open(json!({
"textDocument": {
"uri": "file:///a/file.ts",
"languageId": "typescript",
@ -705,6 +713,18 @@ fn lsp_format_vendor_path() {
"text": r#"import "http://localhost:4545/run/002_hello.ts";"#,
},
}));
// copying from the global cache to the local cache requires explicitly
// running the cache command so that the checksums can be verified
assert_eq!(
diagnostics
.all()
.iter()
.map(|d| d.message.as_str())
.collect::<Vec<_>>(),
vec![
"Uncached or missing remote URL: http://localhost:4545/run/002_hello.ts"
]
);
client.write_request(
"workspace/executeCommand",
json!({
@ -4358,7 +4378,8 @@ fn lsp_code_actions() {
}])
);
let res = client
.write_request( "codeAction/resolve",
.write_request(
"codeAction/resolve",
json!({
"title": "Add all missing 'async' modifiers",
"kind": "quickfix",
@ -4378,8 +4399,7 @@ fn lsp_code_actions() {
"fixId": "fixAwaitInSyncFunction"
}
}),
)
;
);
assert_eq!(
res,
json!({
@ -4762,26 +4782,27 @@ fn lsp_code_actions_deno_cache_jsr() {
#[test]
fn lsp_jsr_lockfile() {
let context = TestContextBuilder::new()
.use_http_server()
.use_temp_cwd()
.build();
let context = TestContextBuilder::for_jsr().use_temp_cwd().build();
let temp_dir = context.temp_dir();
temp_dir.write("./deno.json", json!({}).to_string());
temp_dir.write(
"./deno.lock",
json!({
"version": "3",
"packages": {
"specifiers": {
// This is an old version of the package which exports `sum()` instead
// of `add()`.
"jsr:@denotest/add": "jsr:@denotest/add@0.2.0",
},
let lockfile = temp_dir.path().join("deno.lock");
let integrity = context.get_jsr_package_integrity("@denotest/add/0.2.0");
lockfile.write_json(&json!({
"version": "3",
"packages": {
"specifiers": {
// This is an old version of the package which exports `sum()` instead
// of `add()`.
"jsr:@denotest/add": "jsr:@denotest/add@0.2.0",
},
})
.to_string(),
);
"jsr": {
"@denotest/add@0.2.0": {
"integrity": integrity
}
}
},
"remote": {},
}));
let mut client = context.new_lsp_command().build();
client.initialize_default();
client.did_open(json!({
@ -4790,8 +4811,8 @@ fn lsp_jsr_lockfile() {
"languageId": "typescript",
"version": 1,
"text": r#"
import { add } from "jsr:@denotest/add";
console.log(add(1, 2));
import { sum } from "jsr:@denotest/add";
console.log(sum(1, 2));
"#,
},
}));
@ -10672,9 +10693,27 @@ fn lsp_vendor_dir() {
refresh_config(&mut client);
let diagnostics = client.read_diagnostics();
assert_eq!(diagnostics.all().len(), 0, "{:#?}", diagnostics); // cached
// won't be cached until a manual cache occurs
assert_eq!(
diagnostics
.all()
.iter()
.map(|d| d.message.as_str())
.collect::<Vec<_>>(),
vec![
"Uncached or missing remote URL: http://localhost:4545/subdir/mod1.ts"
]
);
// no caching necessary because it was already cached. It should exist now
assert!(!temp_dir
.path()
.join("vendor/http_localhost_4545/subdir/mod1.ts")
.exists());
// now cache
cache(&mut client);
let diagnostics = client.read_diagnostics();
assert_eq!(diagnostics.all().len(), 0, "{:#?}", diagnostics); // cached
assert!(temp_dir
.path()
.join("vendor/http_localhost_4545/subdir/mod1.ts")

View file

@ -1549,7 +1549,7 @@ fn auto_discover_lock_file() {
output
.assert_matches_text(
r#"Download http://localhost:4545/npm/registry/@denotest/bin
error: Integrity check failed for npm package: "@denotest/bin@1.0.0". Unable to verify that the package
error: Integrity check failed for package: "npm:@denotest/bin@1.0.0". Unable to verify that the package
is the same as when the lockfile was generated.
Actual: sha512-[WILDCARD]

View file

@ -1053,7 +1053,9 @@ fn lock_deno_json_package_json_deps() {
"npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0"
},
"jsr": {
"@denotest/module_graph@1.4.0": {}
"@denotest/module_graph@1.4.0": {
"integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b"
}
},
"npm": {
"@denotest/esm-basic@1.0.0": {
@ -1062,10 +1064,7 @@ fn lock_deno_json_package_json_deps() {
}
}
},
"remote": {
"http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699",
"http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade"
},
"remote": {},
"workspace": {
"dependencies": [
"jsr:@denotest/module_graph@1.4",
@ -1106,7 +1105,9 @@ fn lock_deno_json_package_json_deps() {
"npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0"
},
"jsr": {
"@denotest/module_graph@1.4.0": {}
"@denotest/module_graph@1.4.0": {
"integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b"
}
},
"npm": {
"@denotest/esm-basic@1.0.0": {
@ -1115,10 +1116,7 @@ fn lock_deno_json_package_json_deps() {
}
}
},
"remote": {
"http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699",
"http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade"
},
"remote": {},
"workspace": {
"dependencies": [
"jsr:@denotest/module_graph@1.4"
@ -1147,13 +1145,12 @@ fn lock_deno_json_package_json_deps() {
"jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0",
},
"jsr": {
"@denotest/module_graph@1.4.0": {}
"@denotest/module_graph@1.4.0": {
"integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b"
}
}
},
"remote": {
"http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699",
"http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade"
},
"remote": {},
"workspace": {
"dependencies": [
"jsr:@denotest/module_graph@1.4"

View file

@ -0,0 +1,3 @@
export function add(a: number, b: number): number {
return a + b;
}

View file

@ -0,0 +1,11 @@
{
"exports": {
".": "./mod.ts"
},
"manifest": {
"/mod.ts": {
"size": 0,
"checksum": "sha256-bad-checksum"
}
}
}

View file

@ -0,0 +1,5 @@
{
"versions": {
"1.0.0": {}
}
}

View file

@ -1,5 +1,5 @@
Download [WILDCARD]
error: Integrity check failed for npm package: "@babel/parser@7.19.0". Unable to verify that the package
error: Integrity check failed for package: "npm:@babel/parser@7.19.0". Unable to verify that the package
is the same as when the lockfile was generated.
Actual: sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw==