From c08f113c2d6cd0036ec1373a64d0ec62d0dfa7c0 Mon Sep 17 00:00:00 2001 From: Aaron Power Date: Mon, 2 Jan 2017 22:47:10 +0000 Subject: [PATCH] version 5: optimised stats, language generation --- .gitattributes | 1 + Cargo.lock | 24 ++--- Cargo.toml | 55 ++++++----- build.rs | 5 +- cli.yml | 2 +- languages.json | 9 +- src/language/language.rs | 51 ++++------ src/language/language_type.hbs.rs | 22 +++-- src/language/languages.rs | 159 +++--------------------------- src/main.rs | 74 +++++--------- src/serde_types.in.rs | 5 +- src/stats.rs | 68 ++++++++----- src/utils/fs.rs | 20 ++-- src/utils/multi_line.rs | 4 +- tests/accuracy.rs | 59 +++++++++++ tests/data/d.d | 8 ++ tests/data/fsharp.fs | 13 +++ tests/data/rust.rs | 38 +++++++ 18 files changed, 298 insertions(+), 319 deletions(-) create mode 100644 .gitattributes create mode 100644 tests/accuracy.rs create mode 100644 tests/data/d.d create mode 100644 tests/data/fsharp.fs create mode 100644 tests/data/rust.rs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..918bc07 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/data/* linguist-documentation diff --git a/Cargo.lock b/Cargo.lock index c356a10..5801b48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,18 +1,18 @@ [root] name = "tokei" -version = "4.5.4" +version = "5.0.3" dependencies = [ "clap 2.19.1 (registry+https://github.com/rust-lang/crates.io-index)", "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "errln 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "handlebars 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)", + "handlebars 0.24.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "ignore 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "maplit 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde_cbor 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -171,14 +171,15 @@ dependencies = [ [[package]] name = "handlebars" -version = "0.21.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "pest 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -251,7 +252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "num_cpus" -version = "0.2.13" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -282,11 +283,12 @@ dependencies = [ [[package]] name = "rayon" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -577,7 +579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum errln 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6aba822d3275762d55fd527b698dcf8f233a488885ea5e75c683fb5fd94af946" "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344" "checksum globset 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a9853491e657bd919f5a7e7c3dd1dfcdd2ba674b4d2465c042be2bfb36b642d9" -"checksum handlebars 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "937e9d49d65ffb5f70e95710a6c8539addf40200275ad8b6cdba0f0a59d5814d" +"checksum handlebars 0.24.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd60b0a45ee5f649490244d804b1e8ead7687493938325211b4825761ea06ef5" "checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" "checksum ignore 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "038e0fe6111065719d5ae5b724284d9505be4bbe6195f3418b1ba98995b9e181" "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1" @@ -588,12 +590,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum maplit 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "be384c560e0c3ad868b590ffb88d2c0a1effde6f59885234e4ea811c1202bfea" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum num-traits 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "a16a42856a256b39c6d3484f097f6713e14feacd9bfb02290917904fae46c81c" -"checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3" +"checksum num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "55aabf4e2d6271a2e4e4c0f2ea1f5b07cc589cc1a9e9213013b54a76678ca4f3" "checksum pest 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0a6dda33d67c26f0aac90d324ab2eb7239c819fc7b2552fe9faa4fe88441edc8" "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" "checksum quote 0.3.10 (registry+https://github.com/rust-lang/crates.io-index)" = "6732e32663c9c271bfc7c1823486b471f18c47a2dbf87c066897b7b51afc83be" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" -"checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea" +"checksum rayon 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50c575b58c2b109e2fbc181820cbe177474f35610ff9e357dc75f6bac854ffbf" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" "checksum rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "237546c689f20bb44980270c73c3b9edd0891c1be49cc1274406134a66d3957b" diff --git a/Cargo.toml b/Cargo.toml index d063bc8..cef6036 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,67 +8,70 @@ license = "MIT/Apache-2.0" name = "tokei" readme = "README.md" repository = "https://github.com/Aaronepower/tokei.git" -version = "4.5.4" +version = "5.0.3" + +[profile.release] +panic="abort" [build-dependencies] -serde = "~0.8.19" -serde_json = "~0.8.4" -errln = "0.1.0" +serde = "0.8" +serde_json = "0.8" +errln = "0.1" [build-dependencies.handlebars] features = ["serde_type"] -version = "0.21.1" +version = "~0.24" [build-dependencies.serde_codegen] optional = true -version = "0.8.19" +version = "0.8" [dependencies] -encoding = "~0.2.33" -errln = "0.1.0" -ignore = "~0.1.5" -lazy_static = "~0.2.1" -log = "~0.3.6" -maplit = "~0.1.3" -rayon = "=0.4.2" -regex = "~0.1.80" +encoding = "0.2" +errln = "0.1" +ignore = "0.1" +lazy_static = "0.2" +log = "0.3" +maplit = "0.1" +rayon = "0.6" +regex = "0.1" [dependencies.clap] features = ["yaml"] -version = "~2.19.1" +version = "2.19" [dependencies.env_logger] features = [] -version = "~0.3.5" +version = "0.3" + +[dependencies.hex] +version = "0.2" +optional = true [dependencies.serde] optional = true -version = "~0.8.19" +version = "0.8" [dependencies.serde_cbor] optional = true -version = "~0.4.0" +version = "0.4" [dependencies.serde_json] optional = true -version = "~0.8.4" +version = "0.8" [dependencies.serde_yaml] optional = true -version = "~0.4.0" +version = "0.4" [dependencies.toml] default-features = false features = ["serde"] optional = true -version = "~0.2.1" +version = "0.2" [dev-dependencies] -tempdir = "~0.3.5" - -[dependencies.hex] -version = "~0.2.0" -optional = true +tempdir = "0.3" [features] all = ["json", "cbor", "toml-io", "yaml"] diff --git a/build.rs b/build.rs index 4be869f..1d68627 100644 --- a/build.rs +++ b/build.rs @@ -5,7 +5,7 @@ extern crate handlebars; #[macro_use] extern crate errln; use serde_json::Value; -use handlebars::{Context, Handlebars}; +use handlebars::Handlebars; use std::fs::File; use std::env; use std::path::{Path, PathBuf}; @@ -62,11 +62,10 @@ fn render_handlebars(out_dir: &OsString) -> PathBuf { let mut handlebars = Handlebars::new(); handlebars.register_escape_fn(handlebars::no_escape); - let raw_data: Value = serde_json::from_reader( + let data: Value = serde_json::from_reader( File::open(&"languages.json").expect("Can't open JSON") ).expect("Can't parse JSON"); - let data = Context::wraps(&raw_data); let out = Path::new(&out_dir).join("language_type.rs"); let mut source_template = File::open(&"src/language/language_type.hbs.rs") .expect("Can't find Template"); diff --git a/cli.yml b/cli.yml index 9aa791e..6d59ffb 100644 --- a/cli.yml +++ b/cli.yml @@ -5,7 +5,7 @@ about: Count Code, Quickly. author: Aaron P. bin_name: Tokei name: Tokei -version: 4.5.4 +version: '5' args: - exclude: help: Ignore all files & directories containing the word. diff --git a/languages.json b/languages.json index 10042f4..a1dd669 100644 --- a/languages.json +++ b/languages.json @@ -85,7 +85,7 @@ ] }, "Sh":{ - "name":"sh", + "name":"Shell", "base":"hash", "quotes":[ [ @@ -452,10 +452,9 @@ "single":[ "//" ], - "multi":[[ - "(*", - "*)" - ]], + "multi":[ + ["(*", "*)"] + ], "extensions":[ "fs", "fsi", diff --git a/src/language/language.rs b/src/language/language.rs index 14f73a7..130db94 100644 --- a/src/language/language.rs +++ b/src/language/language.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use std::ops::AddAssign; use std::path::PathBuf; +use std::mem; use regex::{self, Regex}; @@ -24,10 +25,15 @@ fn generate_regex(multi_line: &[(&'static str, &'static str)]) -> Cow<'static, R Cow::Owned(Regex::new(&raw_regex).unwrap()) } -lazy_static! { - static ref C_REGEX: Regex = Regex::new(r"/\*").unwrap(); +fn get_c_regex() -> &'static Regex { + lazy_static! { + static ref C_REGEX: Regex = Regex::new(r"/\*").unwrap(); + } + + &*C_REGEX } + impl Language { /// Constructs a new empty Language with the comments provided. /// @@ -43,6 +49,7 @@ impl Language { line_comment: line_comment, regex: Some(generate_regex(&multi_line)), multi_line: multi_line, + quotes: vec![("\"", "\"")], ..Self::default() } } @@ -76,7 +83,7 @@ impl Language { line_comment: vec!["//"], multi_line: vec![("/*", "*/")], quotes: vec![("\"", "\"")], - regex: Some(Cow::Borrowed(&*C_REGEX)), + regex: Some(Cow::Borrowed(get_c_regex())), ..Self::default() } } @@ -94,7 +101,7 @@ impl Language { /// ``` pub fn new_func() -> Self { lazy_static! { - static ref FUNC_REGEX: Regex = Regex::new(r"\(\*").unwrap(); + static ref FUNC_REGEX: Regex = Regex::new(®ex::quote(r"\(\*")).unwrap(); } Language { multi_line: vec![("(*", "*)")], @@ -155,7 +162,7 @@ impl Language { /// ``` pub fn new_haskell() -> Self { lazy_static! { - static ref HASKELL_REGEX: Regex = Regex::new(r"\{-").unwrap(); + static ref HASKELL_REGEX: Regex = Regex::new(®ex::quote(r"\{-")).unwrap(); } Language { @@ -199,7 +206,7 @@ impl Language { line_comment: vec!["%"], multi_line: vec![("/*", "*/")], quotes: vec![("\"", "\"")], - regex: Some(Cow::Borrowed(&*C_REGEX)), + regex: Some(Cow::Borrowed(get_c_regex())), ..Self::default() } } @@ -302,10 +309,12 @@ impl Language { /// panic!'s if given the wrong category. /// /// ``` - /// # use tokei::*; + /// use tokei::{Language, Stats, Sort}; + /// use std::path::PathBuf; + /// /// let mut rust = Language::new_c(); - /// let mut foo_stats = Stats::new("foo"); - /// let mut bar_stats = Stats::new("bar"); + /// let mut foo_stats = Stats::new(PathBuf::from("foo")); + /// let mut bar_stats = Stats::new(PathBuf::from("bar")); /// /// foo_stats.code += 20; /// bar_stats.code += 10; @@ -332,32 +341,12 @@ impl Language { } impl AddAssign for Language { - fn add_assign(&mut self, rhs: Self) { + fn add_assign(&mut self, mut rhs: Self) { self.lines += rhs.lines; self.comments += rhs.comments; self.blanks += rhs.blanks; self.code += rhs.code; - self.stats.extend_from_slice(&*rhs.stats); - } -} - -impl<'a> AddAssign<&'a Language> for Language { - fn add_assign(&mut self, rhs: &'a Self) { - self.lines += rhs.lines; - self.comments += rhs.comments; - self.blanks += rhs.blanks; - self.code += rhs.code; - self.stats.extend_from_slice(&*rhs.stats); - } -} - -impl<'a> AddAssign<&'a mut Language> for Language { - fn add_assign(&mut self, rhs: &mut Self) { - self.lines += rhs.lines; - self.comments += rhs.comments; - self.blanks += rhs.blanks; - self.code += rhs.code; - self.stats.extend_from_slice(&*rhs.stats); + self.stats.extend(mem::replace(&mut rhs.stats, Vec::new())); } } diff --git a/src/language/language_type.hbs.rs b/src/language/language_type.hbs.rs index 43231ac..34ead2d 100644 --- a/src/language/language_type.hbs.rs +++ b/src/language/language_type.hbs.rs @@ -7,7 +7,6 @@ use std::fmt; use std::path::Path; use std::fs::File; use std::io::{BufRead, BufReader}; -use std::collections::BTreeMap; use utils::fs; use self::LanguageType::*; @@ -45,6 +44,15 @@ impl LanguageType { } } + /// Provides every variant in a Vec + pub fn list() -> Vec { + return vec! [ + {{#each languages}} + {{@key}}, + {{~/each}} + ] + } + /// Get language from it's file extension. /// /// ```no_run @@ -74,10 +82,9 @@ impl LanguageType { } impl Languages { - #[inline] - pub fn generate_languages() -> BTreeMap { - btreemap! { - {{~#each languages}} + pub fn generate_language(language: LanguageType) -> Language { + match language { + {{#each languages}} {{~@key}} => {{~#if this.base}} Language::new_{{this.base}}() @@ -192,11 +199,14 @@ impl<'a> From<&'a LanguageType> for Cow<'a, LanguageType> { /// This is for getting the file type from the first line of a file -pub fn get_filetype_from_shebang>(file: P) -> Option<&'static str> { +pub fn get_filetype_from_shebang

(file: P) -> Option<&'static str> + where P: AsRef +{ let file = match File::open(file) { Ok(file) => file, _ => return None, }; + let mut buf = BufReader::new(file); let mut line = String::new(); let _ = buf.read_line(&mut line); diff --git a/src/language/languages.rs b/src/language/languages.rs index 45d5ac4..67a7e69 100644 --- a/src/language/languages.rs +++ b/src/language/languages.rs @@ -7,6 +7,7 @@ use std::collections::{btree_map, BTreeMap}; use std::fs::File; use std::io::Read; use std::iter::IntoIterator; +use std::mem; use std::ops::{AddAssign, Deref, DerefMut}; use std::sync::{mpsc, Mutex}; @@ -25,31 +26,22 @@ use super::LanguageType::*; use super::{Language, LanguageType}; use utils::{fs, multi_line}; -fn count_files(mut language_tuple: (&LanguageType, &mut Language)) { - - let (name, ref mut language) = language_tuple; - - if language.files.is_empty() { - return; - } +fn count_files((name, ref mut language): (&LanguageType, &mut Language)) { let is_fortran = name == &FortranModern || name == &FortranLegacy; - let (tx, rx) = mpsc::channel(); - let has_multi_line = !language.multi_line.is_empty() && - !language.nested_comments.is_empty(); let synced_tx = Mutex::new(tx); + let has_multi_line = !language.multi_line.is_empty() || !language.nested_comments.is_empty(); let is_blank = language.is_blank(); - language.files.par_iter().for_each(|file| { - let mut stats = Stats::new( - opt_ret_error!(file.to_str(), "Couldn't convert path to String.") - ); + let files = mem::replace(&mut language.files, Vec::new()); + + files.into_par_iter().for_each(|file| { let mut stack = Vec::new(); let mut contents = Vec::new(); let mut quote = None; - rs_ret_error!(rs_ret_error!(File::open(file)).read_to_end(&mut contents)); + rs_ret_error!(rs_ret_error!(File::open(&file)).read_to_end(&mut contents)); let text = match encoding::decode(&contents, Replace, UTF_8) { (Ok(string), _) => Cow::Owned(string), @@ -57,7 +49,7 @@ fn count_files(mut language_tuple: (&LanguageType, &mut Language)) { }; let lines = text.lines(); - + let mut stats = Stats::new(file); if is_blank { let count = lines.count(); stats.lines += count; @@ -120,7 +112,7 @@ fn count_files(mut language_tuple: (&LanguageType, &mut Language)) { } if let &mut Some(quote_str) = &mut quote { - if window.starts_with(&*b"\\") { + if window.starts_with(&*br"\") { skip = 1; } else if window.starts_with(quote_str.as_bytes()) { quote = None; @@ -139,7 +131,10 @@ fn count_files(mut language_tuple: (&LanguageType, &mut Language)) { } } - if no_stack { + let starts_with_comment = language.multi_line.iter().chain(&language.nested_comments) + .any(|&(start, _)| line.starts_with(start)); + + if no_stack && !starts_with_comment { stats.code += 1; } else { stats.comments += 1; @@ -300,8 +295,7 @@ impl Languages { /// let languages = Languages::new(); /// ``` pub fn new() -> Self { - let map = Self::generate_languages(); - Languages { inner: map } + Languages { inner: BTreeMap::new() } } /// Creates a new map that only contains non empty languages. @@ -409,7 +403,7 @@ impl Languages { /// /// ```no_run /// use tokei::*; - /// + /// /// let yaml = r#" /// --- /// "Rust": @@ -479,31 +473,6 @@ impl AddAssign> for Languages { } } -impl<'a> AddAssign<&'a BTreeMap> for Languages { - fn add_assign(&mut self, rhs: &'a BTreeMap) { - - for (name, language) in rhs { - - if let Some(result) = self.inner.get_mut(&name) { - *result += language; - } - } - } -} - -impl<'a> AddAssign<&'a mut BTreeMap> for Languages { - fn add_assign(&mut self, rhs: &'a mut BTreeMap) { - - for (name, language) in rhs { - - if let Some(result) = self.inner.get_mut(&name) { - *result += language; - } - } - } -} - - impl Deref for Languages { type Target = BTreeMap; @@ -511,105 +480,9 @@ impl Deref for Languages { &self.inner } } + impl DerefMut for Languages { fn deref_mut(&mut self) -> &mut BTreeMap { &mut self.inner } } - - -#[cfg(test)] -mod accuracy_tests { - extern crate tempdir; - use super::*; - use std::io::Write; - use std::fs::File; - use language::LanguageType; - use self::tempdir::TempDir; - - - fn test_accuracy(file_name: &'static str, - expected: usize, - contents: &'static str) - { - let tmp_dir = TempDir::new("test").expect("Couldn't create temp dir"); - let file_name = tmp_dir.path().join(file_name); - let mut file = File::create(&file_name).expect("Couldn't create file"); - file.write(contents.as_bytes()).expect("couldn't write to file"); - - let mut l = Languages::new(); - let l_type = LanguageType::from_extension(&file_name) - .expect("Can't find language type"); - l.get_statistics(vec![file_name.to_str().unwrap()], vec![]); - let language = l.get_mut(&l_type).expect("Couldn't find language"); - - assert_eq!(expected, language.code); - } - - #[test] - fn inside_quotes() { - test_accuracy("inside_quotes.rs", - 8, - r#"fn main() { - let start = "/*"; - loop { - if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */ - break; - } - } - }"#) - } - - #[test] - fn shouldnt_panic() { - test_accuracy("shouldnt_panic.rs", - 9, - r#"fn foo() { - let this_ends = "a \"test/*."; - call1(); - call2(); - let this_does_not = /* a /* nested */ comment " */ - "*/another /*test - call3(); - */"; - }"#) - } - - #[test] - fn all_quotes_no_comment() { - test_accuracy("all_quotes_no_comment.rs", - 10, - r#"fn foobar() { - let does_not_start = // " - "until here, - test/* - test"; // a quote: " - let also_doesnt_start = /* " */ - "until here, - test,*/ - test"; // another quote: " -}"#) - } - - #[test] - fn commenting_on_comments() { - test_accuracy("commenting_on_comments.rs", - 5, - r#"fn foo() { - let a = 4; // /* - let b = 5; - let c = 6; // */ -}"#) - } - - #[test] - fn nesting_with_nesting_comments() { - test_accuracy("nesting_with_nesting_comments.d", - 5, - r#"void main() { - auto x = 5; /+ a /+ nested +/ comment /* +/ - writefln("hello"); - auto y = 4; // */ -}"#) - } -} diff --git a/src/main.rs b/src/main.rs index 0cf9b55..ad2b96c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,9 +12,6 @@ mod input; use input::*; use std::borrow::Cow; -use std::sync::mpsc::channel; -use std::thread; -use std::time::Duration; use clap::App; use env_logger::LogBuilder; @@ -43,11 +40,9 @@ fn main() { let verbose_option = matches.occurrences_of("verbose"); let sort_option = matches.value_of("sort"); let ignored_directories = { - let mut ignored_directories: Vec<&str> = vec![".git"]; + let mut ignored_directories: Vec<&str> = Vec::new(); if let Some(user_ignored) = matches.values_of("exclude") { - for ignored in user_ignored { - ignored_directories.push(ignored); - } + ignored_directories.extend(user_ignored); } ignored_directories }; @@ -65,7 +60,7 @@ fn main() { let mut languages = Languages::new(); if language_option { - for key in languages.keys() { + for key in LanguageType::list() { println!("{:<25}", key); } return; @@ -77,31 +72,9 @@ fn main() { add_input(input, &mut languages); } - let mut total = Language::new_blank(); - - let print_animation = output_option == None; - let (tx, rx) = channel(); - let child = thread::spawn(move || { - let time = 100; - loop { - if let Ok(_) = rx.try_recv() { - break; - } - - if print_animation { - print!(" Counting files. \r"); - thread::sleep(Duration::from_millis(time)); - print!(" Counting files..\r"); - thread::sleep(Duration::from_millis(time)); - print!(" Counting files...\r"); - thread::sleep(Duration::from_millis(time)); - } - } - }); - languages.get_statistics(paths, ignored_directories); - if output_option == None { + if output_option.is_none() { println!("{}", ROW); println!(" {:<12} {:>12} {:>12} {:>12} {:>12} {:>12}", "Language", @@ -111,32 +84,24 @@ fn main() { "Comments", "Blanks"); println!("{}", ROW); - } - for (name, language) in &languages { - if !language.is_empty() && sort_option == None && output_option == None { - if files_option { - print_language(language, name); - println!("{}", ROW); + if sort_option.is_none() { + for (name, language) in languages.iter().filter(isnt_empty) { + if files_option { + print_language(language, name); + println!("{}", ROW); - for stat in &language.stats { - println!("{}", stat); + for stat in &language.stats { + println!("{}", stat); + } + println!("{}", ROW); + } else if output_option.is_none() { + print_language(language, name); } - println!("{}", ROW); - } else if output_option == None { - print_language(language, name); } } } - let _ = tx.send(()); - let _ = child.join(); - - for (_, language) in &languages { - if !language.is_empty() { - total += language; - } - } if let Some(format) = output_option { match_output(format, languages); @@ -153,7 +118,7 @@ fn main() { } } - let mut languages: Vec<_> = languages.into_iter().collect(); + let mut languages: Vec<_> = languages.iter().collect(); match &*sort_category { BLANKS => languages.sort_by(|a, b| b.1.blanks.cmp(&a.1.blanks)), @@ -184,6 +149,10 @@ fn main() { if !files_option { println!("{}", ROW); } + let mut total = Language::new_blank(); + for (_, language) in languages { + total += language; + } println!(" {: <18} {: >6} {:>12} {:>12} {:>12} {:>12}", "Total", total.stats.len(), @@ -195,6 +164,9 @@ fn main() { } } +fn isnt_empty(&(_, language): &(&LanguageType, &Language)) -> bool { + !language.is_empty() +} fn print_language<'a, C>(language: &'a Language, name: C) where C: Into> diff --git a/src/serde_types.in.rs b/src/serde_types.in.rs index 6e9932a..2f1b703 100644 --- a/src/serde_types.in.rs +++ b/src/serde_types.in.rs @@ -1,6 +1,7 @@ +use std::path::PathBuf; /// A struct representing the statistics of a file. #[cfg_attr(feature = "io", derive(Deserialize, Serialize))] -#[derive(Clone, Default, Debug, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct Stats { /// Number of blank lines within the file. pub blanks: usize, @@ -11,6 +12,6 @@ pub struct Stats { /// Total number of lines within the file. pub lines: usize, /// File name. - pub name: String, + pub name: PathBuf, } diff --git a/src/stats.rs b/src/stats.rs index a616e67..8906714 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -10,11 +10,27 @@ impl Stats { /// Create a new `Stats` from a file path. /// /// ``` - /// # use tokei::*; - /// let stats = Stats::new("src/main.rs"); + /// use std::path::PathBuf; + /// use tokei::Stats; + /// + /// let path = PathBuf::from("src/main.rs"); + /// + /// let stats = Stats::new(path); /// ``` - pub fn new>(name: S) -> Self { - Stats { name: name.into(), ..Self::default() } + pub fn new(name: PathBuf) -> Self { + Stats { name: name, ..Self::default() } + } +} + +impl Default for Stats { + fn default() -> Self { + Stats { + name: PathBuf::new(), + lines: usize::default(), + code: usize::default(), + comments: usize::default(), + blanks: usize::default(), + } } } @@ -27,26 +43,30 @@ fn find_char_boundary(s: &str, index: usize) -> usize { unreachable!(); } -impl fmt::Display for Stats { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let name_length = self.name.len(); - - let name = if name_length == 25 { - self.name.clone() - } else if self.name.len() > 24 { - let mut name = String::from("|"); - let from = find_char_boundary(&self.name, self.name.len() - 24); - name.push_str(&self.name[from..]); - name - } else { - self.name.clone() - }; - write!(f, +macro_rules! display_stats { + ($f:expr, $this:expr, $name:expr) => { + write!($f, " {: <25} {:>12} {:>12} {:>12} {:>12}", - name, - self.lines, - self.code, - self.comments, - self.blanks) + $name, + $this.lines, + $this.code, + $this.comments, + $this.blanks) + } +} + +impl fmt::Display for Stats { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let name = self.name.to_string_lossy(); + let name_length = name.len(); + + if name_length == 25 || name_length <= 24 { + display_stats!(f, self, name) + } else { + let mut formatted = String::from("|"); + let from = find_char_boundary(&name, name_length - 24); + formatted.push_str(&name[from..]); + display_stats!(f, self, formatted) + } } } diff --git a/src/utils/fs.rs b/src/utils/fs.rs index 7dd2058..537eaec 100644 --- a/src/utils/fs.rs +++ b/src/utils/fs.rs @@ -11,20 +11,10 @@ use ignore::WalkBuilder; use ignore::overrides::OverrideBuilder; use ignore::WalkState::*; -use language::{Language, LanguageType}; +use language::{Language, Languages, LanguageType}; use language::LanguageType::*; pub use language::get_filetype_from_shebang; -macro_rules! get_language { - ($languages:expr, $entry:expr) => { - if let Some(language_type) = LanguageType::from_extension($entry) { - opt_error!($languages.get_mut(&language_type), "Unknown Language? Shouldn't happen.") - } else { - continue; - } - } -} - pub fn get_all_files(paths: Vec<&str>, ignored_directories: Vec<&str>, languages: &mut BTreeMap) @@ -84,7 +74,9 @@ pub fn get_all_files(paths: Vec<&str>, }); for (language_type, pathbuf) in rx { - opt_error!(languages.get_mut(&language_type), "??!").files.push(pathbuf); + languages.entry(language_type) + .or_insert(Languages::generate_language(language_type)) + .files.push(pathbuf); } } @@ -124,8 +116,8 @@ mod test { create_dir(&path_name).expect("Couldn't create directory.rs within temp"); let mut l = Languages::new(); - get_all_files(vec![tmp_dir.into_path().to_str().unwrap()].into(), vec![].into(), &mut l); + get_all_files(vec![tmp_dir.into_path().to_str().unwrap()], vec![], &mut l); - assert_eq!(0, l.get(&LanguageType::Rust).unwrap().files.len()); + assert!(l.get(&LanguageType::Rust).is_none()); } } diff --git a/src/utils/multi_line.rs b/src/utils/multi_line.rs index 7a677fb..5350588 100644 --- a/src/utils/multi_line.rs +++ b/src/utils/multi_line.rs @@ -18,11 +18,11 @@ pub fn handle_multi_line(line: &str, 'window: for window in line.as_bytes().windows(window_size) { while skip != 0 { skip -= 1; - continue; + continue 'window; } if let &mut Some(quote_str) = quote { - if window.starts_with(b"\\") { + if window.starts_with(br"\") { skip = 1; } else if window.starts_with(quote_str.as_bytes()) { *quote = None; diff --git a/tests/accuracy.rs b/tests/accuracy.rs new file mode 100644 index 0000000..be664ed --- /dev/null +++ b/tests/accuracy.rs @@ -0,0 +1,59 @@ +#[macro_use] extern crate lazy_static; +extern crate regex; +extern crate tokei; +extern crate ignore; + +use std::io::Read; +use std::fs::File; + +use regex::Regex; +use tokei::Languages; + +lazy_static! { + static ref LINES: Regex = Regex::new(r"\d+ lines").unwrap(); + static ref CODE: Regex = Regex::new(r"\d+ code").unwrap(); + static ref COMMENTS: Regex = Regex::new(r"\d+ comments").unwrap(); + static ref BLANKS: Regex = Regex::new(r"\d+ blanks").unwrap(); +} + +macro_rules! get_digit { + ($regex:expr, $text:expr) => {{ + let (begin, end) = $regex.find(&$text).expect("Couldn't find category"); + $text[begin..end].split_whitespace() + .next() + .unwrap() + .parse::() + .unwrap() + }} +} + +#[test] +fn languages() { + use ignore::Walk; + let walker = Walk::new("./tests/data/").filter(|p| { + match p { + &Ok(ref p) => !p.metadata().unwrap().is_dir(), + _ => false, + } + }); + for path in walker { + let path = path.unwrap(); + let path = path.path().to_str().unwrap(); + let mut languages = Languages::new(); + languages.get_statistics(vec![path], vec![]); + let mut contents = String::new(); + File::open(path).unwrap().read_to_string(&mut contents).unwrap(); + + + for (name, language) in languages { + assert_eq!(get_digit!(LINES, contents), language.lines); + println!("{} LINES MATCH", name); + assert_eq!(get_digit!(CODE, contents), language.code); + println!("{} CODE MATCH", name); + assert_eq!(get_digit!(COMMENTS, contents), language.comments); + println!("{} COMMENTS MATCH", name); + assert_eq!(get_digit!(BLANKS, contents), language.blanks); + println!("{} BLANKS MATCH", name); + } + } +} diff --git a/tests/data/d.d b/tests/data/d.d new file mode 100644 index 0000000..af91f07 --- /dev/null +++ b/tests/data/d.d @@ -0,0 +1,8 @@ +/* 8 lines 5 code 1 comments 2 blanks */ + +void main() { + auto x = 5; /+ a /+ nested +/ comment /* +/ + writefln("hello"); + auto y = 4; // */ +} + diff --git a/tests/data/fsharp.fs b/tests/data/fsharp.fs new file mode 100644 index 0000000..27bb394 --- /dev/null +++ b/tests/data/fsharp.fs @@ -0,0 +1,13 @@ +(* 13 lines 5 code 4 comments 4 blanks *) + +// Comment + +let foo = (* + Comment +*) +5 + +let bar = "(* + Code +*)" + diff --git a/tests/data/rust.rs b/tests/data/rust.rs new file mode 100644 index 0000000..a098390 --- /dev/null +++ b/tests/data/rust.rs @@ -0,0 +1,38 @@ +// 38 lines 32 code 1 comments 5 blanks + +fn main() { + let start = "/*"; + loop { + if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */ + break; + } + } +} + +fn foo() { + let this_ends = "a \"test/*."; + call1(); + call2(); + let this_does_not = /* a /* nested */ comment " */ + "*/another /*test + call3(); + */"; +} + +fn foobar() { + let does_not_start = // " + "until here, + test/* + test"; // a quote: " + let also_doesnt_start = /* " */ + "until here, + test,*/ + test"; // another quote: " +} + +fn foo() { + let a = 4; // /* + let b = 5; + let c = 6; // */ +} +