From 69a94e412bcb2f71772884bc11556720fa9bca8f Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Thu, 17 Feb 2022 22:24:41 +0100 Subject: [PATCH] docs: use rust libraries for downloading and unzipping tldr --- .../workspace.wordlist.txt | 1 + Cargo.lock | 302 ++++++++++++++++++ Cargo.toml | 2 + docs/.gitignore | 1 - docs/Makefile | 2 - src/bin/uudoc.rs | 103 +++--- 6 files changed, 364 insertions(+), 47 deletions(-) diff --git a/.vscode/cspell.dictionaries/workspace.wordlist.txt b/.vscode/cspell.dictionaries/workspace.wordlist.txt index e41aba979..99ac20ea2 100644 --- a/.vscode/cspell.dictionaries/workspace.wordlist.txt +++ b/.vscode/cspell.dictionaries/workspace.wordlist.txt @@ -44,6 +44,7 @@ termsize termwidth textwrap thiserror +ureq walkdir winapi xattr diff --git a/Cargo.lock b/Cargo.lock index 90b71d2a7..244c56baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.4.7" @@ -73,6 +79,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bigdecimal" version = "0.3.0" @@ -167,6 +179,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byte-unit" version = "4.0.13" @@ -228,6 +246,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "clang-sys" version = "1.3.0" @@ -329,6 +353,7 @@ dependencies = [ "time", "unindent", "unix_socket", + "ureq", "users", "uu_arch", "uu_base32", @@ -432,6 +457,7 @@ dependencies = [ "uu_yes", "uucore", "walkdir", + "zip", ] [[package]] @@ -546,6 +572,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.2" @@ -792,12 +827,34 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.2.0" @@ -927,6 +984,17 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "if_rust_version" version = "1.0.0" @@ -967,6 +1035,15 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.0" @@ -1044,6 +1121,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "md5" version = "0.3.8" @@ -1089,6 +1172,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.14" @@ -1375,6 +1468,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "phf" version = "0.10.1" @@ -1655,6 +1754,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rlimit" version = "0.4.0" @@ -1681,6 +1795,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustls" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b323592e3164322f5b193dc4302e4e36cd8d37158a712d664efae1a5c2791700" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1696,6 +1822,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "selinux" version = "0.2.5" @@ -1838,6 +1974,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2009,6 +2151,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "toml" version = "0.5.8" @@ -2024,6 +2181,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + [[package]] name = "unicode-linebreak" version = "0.1.2" @@ -2033,6 +2196,15 @@ dependencies = [ "regex", ] +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -2073,6 +2245,41 @@ dependencies = [ "libc", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "users" version = "0.10.0" @@ -3171,6 +3378,89 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote 1.0.14", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +dependencies = [ + "webpki", +] + [[package]] name = "which" version = "4.2.2" @@ -3248,3 +3538,15 @@ name = "z85" version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af896e93db81340b74b65f74276a99b210c086f3d34ed0abf433182a462af856" + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "crc32fast", + "flate2", + "thiserror", +] diff --git a/Cargo.toml b/Cargo.toml index 336729813..4222f1749 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,8 @@ lazy_static = { version="1.3" } textwrap = { version="0.14", features=["terminal_size"] } uucore = { version=">=0.0.11", package="uucore", path="src/uucore" } selinux = { version="0.2", optional = true } +ureq = "2.4.0" +zip = { version = "0.5.13", default_features=false, features=["deflate"] } # * uutils uu_test = { optional=true, version="0.0.12", package="uu_test", path="src/uu/test" } # diff --git a/docs/.gitignore b/docs/.gitignore index c836306f5..f2b5c7168 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,4 +1,3 @@ book src/utils src/SUMMARY.md -tldr/ diff --git a/docs/Makefile b/docs/Makefile index 23901b755..7372b3868 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,6 +1,4 @@ -# spell-checker:ignore tldr clean: rm -rf book rm -f src/SUMMARY.md rm -f src/utils/* - rm -rf tldr diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 5658f491c..9e075ccd4 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -7,27 +7,21 @@ use clap::App; use std::ffi::OsString; use std::fs::File; -use std::io::{self, Read, Write}; -use std::process::Command; +use std::io::Cursor; +use std::io::{self, Read, Seek, Write}; +use zip::ZipArchive; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn main() -> io::Result<()> { - let _ = std::fs::create_dir("docs/tldr"); println!("Downloading tldr archive"); - Command::new("curl") - .arg("https://tldr.sh/assets/tldr.zip") - .arg("--output") - .arg("docs/tldr/tldr.zip") - .output()?; - - println!("Unzipping tldr archive"); - Command::new("unzip") - .arg("-o") - .arg("docs/tldr/tldr.zip") - .arg("-d") - .arg("docs/tldr") - .output()?; + let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip") + .call() + .unwrap() + .into_reader(); + let mut buffer = Vec::new(); + zip_reader.read_to_end(&mut buffer).unwrap(); + let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap(); let utils = util_map::>>(); match std::fs::create_dir("docs/src/utils/") { @@ -58,7 +52,7 @@ fn main() -> io::Result<()> { } let p = format!("docs/src/utils/{}.md", name); if let Ok(f) = File::create(&p) { - write_markdown(f, &mut app(), name)?; + write_markdown(f, &mut app(), name, &mut tldr_zip)?; println!("Wrote to '{}'", p); } else { println!("Error writing to {}", p); @@ -68,12 +62,17 @@ fn main() -> io::Result<()> { Ok(()) } -fn write_markdown(mut w: impl Write, app: &mut App, name: &str) -> io::Result<()> { +fn write_markdown( + mut w: impl Write, + app: &mut App, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { write!(w, "# {}\n\n", name)?; write_version(&mut w, app)?; write_usage(&mut w, app, name)?; write_description(&mut w, app)?; - write_examples(&mut w, name)?; + write_examples(&mut w, name, tldr_zip)?; write_options(&mut w, app) } @@ -101,33 +100,49 @@ fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> { } } -fn write_examples(w: &mut impl Write, name: &str) -> io::Result<()> { - if let Ok(mut file) = std::fs::File::open(format!("docs/tldr/pages/common/{}.md", name)) - .or_else(|_| std::fs::File::open(format!("docs/tldr/pages/linux/{}.md", name))) - { - let mut content = String::new(); - file.read_to_string(&mut content)?; - - writeln!(w, "## Examples")?; - writeln!(w)?; - for line in content.lines().skip_while(|l| !l.starts_with('-')) { - if let Some(l) = line.strip_prefix("- ") { - writeln!(w, "{}", l)?; - } else if line.starts_with('`') { - writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; - } else if line.is_empty() { - writeln!(w)?; - } else { - println!("Not sure what to do with this line:"); - println!("{}", line); - } - } - writeln!(w)?; - writeln!(w, "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md).")?; +fn write_examples( + w: &mut impl Write, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { + let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) { + f + } else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) { + f } else { - println!("No examples found for: {}", name); + return Ok(()); + }; + + writeln!(w, "## Examples")?; + writeln!(w)?; + for line in content.lines().skip_while(|l| !l.starts_with('-')) { + if let Some(l) = line.strip_prefix("- ") { + writeln!(w, "{}", l)?; + } else if line.starts_with('`') { + writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; + } else if line.is_empty() { + writeln!(w)?; + } else { + println!("Not sure what to do with this line:"); + println!("{}", line); + } } - Ok(()) + writeln!(w)?; + writeln!( + w, + "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)." + )?; + writeln!(w, ">")?; + writeln!( + w, + "> Please note that, as uutils is a work in progress, some examples might fail." + ) +} + +fn get_zip_content(archive: &mut ZipArchive, name: &str) -> Option { + let mut s = String::new(); + archive.by_name(name).ok()?.read_to_string(&mut s).unwrap(); + Some(s) } fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> {