version 5: optimised stats, language generation

.gitattributes

@@ -0,0 +1 @@
+tests/data/* linguist-documentation

Cargo.lock

@@ -1,18 +1,18 @@
 [root]
 name = "tokei"
-version = "4.5.4"
+version = "5.0.3"
 dependencies = [
  "clap 2.19.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "errln 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "handlebars 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "handlebars 0.24.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ignore 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "maplit 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rayon 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_cbor 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -171,14 +171,15 @@ dependencies = [
 
 [[package]]
 name = "handlebars"
-version = "0.21.1"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
+ "lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "pest 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -251,7 +252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "num_cpus"
-version = "0.2.13"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -282,11 +283,12 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "0.4.2"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -577,7 +579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum errln 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6aba822d3275762d55fd527b698dcf8f233a488885ea5e75c683fb5fd94af946"
 "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
 "checksum globset 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a9853491e657bd919f5a7e7c3dd1dfcdd2ba674b4d2465c042be2bfb36b642d9"
-"checksum handlebars 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "937e9d49d65ffb5f70e95710a6c8539addf40200275ad8b6cdba0f0a59d5814d"
+"checksum handlebars 0.24.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd60b0a45ee5f649490244d804b1e8ead7687493938325211b4825761ea06ef5"
 "checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa"
 "checksum ignore 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "038e0fe6111065719d5ae5b724284d9505be4bbe6195f3418b1ba98995b9e181"
 "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1"
@@ -588,12 +590,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum maplit 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "be384c560e0c3ad868b590ffb88d2c0a1effde6f59885234e4ea811c1202bfea"
 "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
 "checksum num-traits 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "a16a42856a256b39c6d3484f097f6713e14feacd9bfb02290917904fae46c81c"
-"checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3"
+"checksum num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "55aabf4e2d6271a2e4e4c0f2ea1f5b07cc589cc1a9e9213013b54a76678ca4f3"
 "checksum pest 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0a6dda33d67c26f0aac90d324ab2eb7239c819fc7b2552fe9faa4fe88441edc8"
 "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
 "checksum quote 0.3.10 (registry+https://github.com/rust-lang/crates.io-index)" = "6732e32663c9c271bfc7c1823486b471f18c47a2dbf87c066897b7b51afc83be"
 "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
-"checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea"
+"checksum rayon 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50c575b58c2b109e2fbc181820cbe177474f35610ff9e357dc75f6bac854ffbf"
 "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f"
 "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957"
 "checksum rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "237546c689f20bb44980270c73c3b9edd0891c1be49cc1274406134a66d3957b"

Cargo.toml

@@ -8,67 +8,70 @@ license = "MIT/Apache-2.0"
 name = "tokei"
 readme = "README.md"
 repository = "https://github.com/Aaronepower/tokei.git"
-version = "4.5.4"
+version = "5.0.3"
+
+[profile.release]
+panic="abort"
 
 [build-dependencies]
-serde = "~0.8.19"
+serde = "0.8"
-serde_json = "~0.8.4"
+serde_json = "0.8"
-errln = "0.1.0"
+errln = "0.1"
 
 [build-dependencies.handlebars]
 features = ["serde_type"]
-version = "0.21.1"
+version = "~0.24"
 
 [build-dependencies.serde_codegen]
 optional = true
-version = "0.8.19"
+version = "0.8"
 
 [dependencies]
-encoding = "~0.2.33"
+encoding = "0.2"
-errln = "0.1.0"
+errln = "0.1"
-ignore = "~0.1.5"
+ignore = "0.1"
-lazy_static = "~0.2.1"
+lazy_static = "0.2"
-log = "~0.3.6"
+log = "0.3"
-maplit = "~0.1.3"
+maplit = "0.1"
-rayon = "=0.4.2"
+rayon = "0.6"
-regex = "~0.1.80"
+regex = "0.1"
 
 [dependencies.clap]
 features = ["yaml"]
-version = "~2.19.1"
+version = "2.19"
 
 [dependencies.env_logger]
 features = []
-version = "~0.3.5"
+version = "0.3"
+
+[dependencies.hex]
+version = "0.2"
+optional = true
 
 [dependencies.serde]
 optional = true
-version = "~0.8.19"
+version = "0.8"
 
 [dependencies.serde_cbor]
 optional = true
-version = "~0.4.0"
+version = "0.4"
 
 [dependencies.serde_json]
 optional = true
-version = "~0.8.4"
+version = "0.8"
 
 [dependencies.serde_yaml]
 optional = true
-version = "~0.4.0"
+version = "0.4"
 
 [dependencies.toml]
 default-features = false
 features = ["serde"]
 optional = true
-version = "~0.2.1"
+version = "0.2"
 
 [dev-dependencies]
-tempdir = "~0.3.5"
+tempdir = "0.3"
-
-[dependencies.hex]
-version = "~0.2.0"
-optional = true
 
 [features]
 all = ["json", "cbor", "toml-io", "yaml"]

build.rs

@@ -5,7 +5,7 @@ extern crate handlebars;
 #[macro_use] extern crate errln;
 
 use serde_json::Value;
-use handlebars::{Context, Handlebars};
+use handlebars::Handlebars;
 use std::fs::File;
 use std::env;
 use std::path::{Path, PathBuf};
@@ -62,11 +62,10 @@ fn render_handlebars(out_dir: &OsString) -> PathBuf {
     let mut handlebars = Handlebars::new();
     handlebars.register_escape_fn(handlebars::no_escape);
 
-    let raw_data: Value = serde_json::from_reader(
+    let data: Value = serde_json::from_reader(
         File::open(&"languages.json").expect("Can't open JSON")
     ).expect("Can't parse JSON");
 
-    let data = Context::wraps(&raw_data);
     let out = Path::new(&out_dir).join("language_type.rs");
     let mut source_template = File::open(&"src/language/language_type.hbs.rs")
         .expect("Can't find Template");

cli.yml

@@ -5,7 +5,7 @@ about: Count Code, Quickly.
 author: Aaron P. <theaaronepower@gmail.com>
 bin_name: Tokei
 name: Tokei
-version: 4.5.4
+version: '5'
 args:
     - exclude:
         help: Ignore all files & directories containing the word.

languages.json

@@ -85,7 +85,7 @@
             ]
         },
         "Sh":{
-            "name":"sh",
+            "name":"Shell",
             "base":"hash",
             "quotes":[
                 [
@@ -452,10 +452,9 @@
             "single":[
                 "//"
             ],
-            "multi":[[
+            "multi":[
-                "(*",
+                ["(*", "*)"]
-                "*)"
+            ],
-            ]],
             "extensions":[
                 "fs",
                 "fsi",

src/language/language.rs

@@ -1,6 +1,7 @@
 use std::borrow::Cow;
 use std::ops::AddAssign;
 use std::path::PathBuf;
+use std::mem;
 
 use regex::{self, Regex};
 
@@ -24,10 +25,15 @@ fn generate_regex(multi_line: &[(&'static str, &'static str)]) -> Cow<'static, R
     Cow::Owned(Regex::new(&raw_regex).unwrap())
 }
 
-lazy_static! {
+fn get_c_regex() -> &'static Regex {
-    static ref C_REGEX: Regex = Regex::new(r"/\*").unwrap();
+    lazy_static! {
+        static ref C_REGEX: Regex = Regex::new(r"/\*").unwrap();
+    }
+
+    &*C_REGEX
 }
 
+
 impl Language {
     /// Constructs a new  empty Language with the comments provided.
     ///
@@ -43,6 +49,7 @@ impl Language {
             line_comment: line_comment,
             regex: Some(generate_regex(&multi_line)),
             multi_line: multi_line,
+            quotes: vec![("\"", "\"")],
             ..Self::default()
         }
     }
@@ -76,7 +83,7 @@ impl Language {
             line_comment: vec!["//"],
             multi_line: vec![("/*", "*/")],
             quotes: vec![("\"", "\"")],
-            regex: Some(Cow::Borrowed(&*C_REGEX)),
+            regex: Some(Cow::Borrowed(get_c_regex())),
             ..Self::default()
         }
     }
@@ -94,7 +101,7 @@ impl Language {
     /// ```
     pub fn new_func() -> Self {
         lazy_static! {
-            static ref FUNC_REGEX: Regex = Regex::new(r"\(\*").unwrap();
+            static ref FUNC_REGEX: Regex = Regex::new(&regex::quote(r"\(\*")).unwrap();
         }
         Language {
             multi_line: vec![("(*", "*)")],
@@ -155,7 +162,7 @@ impl Language {
     /// ```
     pub fn new_haskell() -> Self {
         lazy_static! {
-            static ref HASKELL_REGEX: Regex = Regex::new(r"\{-").unwrap();
+            static ref HASKELL_REGEX: Regex = Regex::new(&regex::quote(r"\{-")).unwrap();
         }
 
         Language {
@@ -199,7 +206,7 @@ impl Language {
             line_comment: vec!["%"],
             multi_line: vec![("/*", "*/")],
             quotes: vec![("\"", "\"")],
-            regex: Some(Cow::Borrowed(&*C_REGEX)),
+            regex: Some(Cow::Borrowed(get_c_regex())),
             ..Self::default()
         }
     }
@@ -302,10 +309,12 @@ impl Language {
     /// panic!'s if given the wrong category.
     ///
     /// ```
-    /// # use tokei::*;
+    /// use tokei::{Language, Stats, Sort};
+    /// use std::path::PathBuf;
+    ///
     /// let mut rust = Language::new_c();
-    /// let mut foo_stats = Stats::new("foo");
+    /// let mut foo_stats = Stats::new(PathBuf::from("foo"));
-    /// let mut bar_stats = Stats::new("bar");
+    /// let mut bar_stats = Stats::new(PathBuf::from("bar"));
     ///
     /// foo_stats.code += 20;
     /// bar_stats.code += 10;
@@ -332,32 +341,12 @@ impl Language {
 }
 
 impl AddAssign for Language {
-    fn add_assign(&mut self, rhs: Self) {
+    fn add_assign(&mut self, mut rhs: Self) {
         self.lines += rhs.lines;
         self.comments += rhs.comments;
         self.blanks += rhs.blanks;
         self.code += rhs.code;
-        self.stats.extend_from_slice(&*rhs.stats);
+        self.stats.extend(mem::replace(&mut rhs.stats, Vec::new()));
-    }
-}
-
-impl<'a> AddAssign<&'a Language> for Language {
-    fn add_assign(&mut self, rhs: &'a Self) {
-        self.lines += rhs.lines;
-        self.comments += rhs.comments;
-        self.blanks += rhs.blanks;
-        self.code += rhs.code;
-        self.stats.extend_from_slice(&*rhs.stats);
-    }
-}
-
-impl<'a> AddAssign<&'a mut Language> for Language {
-    fn add_assign(&mut self, rhs: &mut Self) {
-        self.lines += rhs.lines;
-        self.comments += rhs.comments;
-        self.blanks += rhs.blanks;
-        self.code += rhs.code;
-        self.stats.extend_from_slice(&*rhs.stats);
     }
 }
 

src/language/language_type.hbs.rs

@@ -7,7 +7,6 @@ use std::fmt;
 use std::path::Path;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
-use std::collections::BTreeMap;
 
 use utils::fs;
 use self::LanguageType::*;
@@ -45,6 +44,15 @@ impl LanguageType {
         }
     }
 
+    /// Provides every variant in a Vec
+    pub fn list() -> Vec<Self> {
+        return vec! [
+            {{#each languages}}
+                {{@key}},
+            {{~/each}}
+        ]
+    }
+
     /// Get language from it's file extension.
     ///
     /// ```no_run
@@ -74,10 +82,9 @@ impl LanguageType {
 }
 
 impl Languages {
-    #[inline]
+    pub fn generate_language(language: LanguageType) -> Language {
-    pub fn generate_languages() -> BTreeMap<LanguageType, Language> {
+        match language {
-        btreemap! {
+            {{#each languages}}
-            {{~#each languages}}
                 {{~@key}} =>
                 {{~#if this.base}}
                     Language::new_{{this.base}}()
@@ -192,11 +199,14 @@ impl<'a> From<&'a LanguageType> for Cow<'a, LanguageType> {
 
 
 /// This is for getting the file type from the first line of a file
-pub fn get_filetype_from_shebang<P: AsRef<Path>>(file: P) -> Option<&'static str> {
+pub fn get_filetype_from_shebang<P>(file: P) -> Option<&'static str>
+    where P: AsRef<Path>
+{
     let file = match File::open(file) {
         Ok(file) => file,
         _ => return None,
     };
+
     let mut buf = BufReader::new(file);
     let mut line = String::new();
     let _ = buf.read_line(&mut line);

src/language/languages.rs

@@ -7,6 +7,7 @@ use std::collections::{btree_map, BTreeMap};
 use std::fs::File;
 use std::io::Read;
 use std::iter::IntoIterator;
+use std::mem;
 use std::ops::{AddAssign, Deref, DerefMut};
 use std::sync::{mpsc, Mutex};
 
@@ -25,31 +26,22 @@ use super::LanguageType::*;
 use super::{Language, LanguageType};
 use utils::{fs, multi_line};
 
-fn count_files(mut language_tuple: (&LanguageType, &mut Language)) {
+fn count_files((name, ref mut language): (&LanguageType, &mut Language)) {
-
-    let (name, ref mut language) = language_tuple;
-
-    if language.files.is_empty() {
-        return;
-    }
 
     let is_fortran = name == &FortranModern || name == &FortranLegacy;
-
     let (tx, rx) = mpsc::channel();
-    let has_multi_line = !language.multi_line.is_empty() &&
-        !language.nested_comments.is_empty();
     let synced_tx = Mutex::new(tx);
+    let has_multi_line = !language.multi_line.is_empty() || !language.nested_comments.is_empty();
     let is_blank = language.is_blank();
 
-    language.files.par_iter().for_each(|file| {
+    let files = mem::replace(&mut language.files, Vec::new());
-        let mut stats = Stats::new(
+
-            opt_ret_error!(file.to_str(), "Couldn't convert path to String.")
+    files.into_par_iter().for_each(|file| {
-        );
         let mut stack = Vec::new();
         let mut contents = Vec::new();
         let mut quote = None;
 
-        rs_ret_error!(rs_ret_error!(File::open(file)).read_to_end(&mut contents));
+        rs_ret_error!(rs_ret_error!(File::open(&file)).read_to_end(&mut contents));
 
         let text = match encoding::decode(&contents, Replace, UTF_8) {
             (Ok(string), _) => Cow::Owned(string),
@@ -57,7 +49,7 @@ fn count_files(mut language_tuple: (&LanguageType, &mut Language)) {
         };
 
         let lines = text.lines();
-
+        let mut stats = Stats::new(file);
         if is_blank {
             let count = lines.count();
             stats.lines += count;
@@ -120,7 +112,7 @@ fn count_files(mut language_tuple: (&LanguageType, &mut Language)) {
                     }
 
                     if let &mut Some(quote_str) = &mut quote {
-                        if window.starts_with(&*b"\\") {
+                        if window.starts_with(&*br"\") {
                             skip = 1;
                         } else if window.starts_with(quote_str.as_bytes()) {
                             quote = None;
@@ -139,7 +131,10 @@ fn count_files(mut language_tuple: (&LanguageType, &mut Language)) {
                 }
             }
 
-            if no_stack {
+            let starts_with_comment = language.multi_line.iter().chain(&language.nested_comments)
+                .any(|&(start, _)| line.starts_with(start));
+
+            if no_stack && !starts_with_comment {
                 stats.code += 1;
             } else {
                 stats.comments += 1;
@@ -300,8 +295,7 @@ impl Languages {
     /// let languages = Languages::new();
     /// ```
     pub fn new() -> Self {
-        let map = Self::generate_languages();
+        Languages { inner: BTreeMap::new() }
-        Languages { inner: map }
     }
 
     /// Creates a new map that only contains non empty languages.
@@ -409,7 +403,7 @@ impl Languages {
     ///
     /// ```no_run
     /// use tokei::*;
-    /// 
+    ///
     /// let yaml = r#"
     ///     ---
     ///     "Rust":
@@ -479,31 +473,6 @@ impl AddAssign<BTreeMap<LanguageType, Language>> for Languages {
     }
 }
 
-impl<'a> AddAssign<&'a BTreeMap<LanguageType, Language>> for Languages {
-    fn add_assign(&mut self, rhs: &'a BTreeMap<LanguageType, Language>) {
-
-        for (name, language) in rhs {
-
-            if let Some(result) = self.inner.get_mut(&name) {
-                *result += language;
-            }
-        }
-    }
-}
-
-impl<'a> AddAssign<&'a mut BTreeMap<LanguageType, Language>> for Languages {
-    fn add_assign(&mut self, rhs: &'a mut BTreeMap<LanguageType, Language>) {
-
-        for (name, language) in rhs {
-
-            if let Some(result) = self.inner.get_mut(&name) {
-                *result += language;
-            }
-        }
-    }
-}
-
-
 impl Deref for Languages {
     type Target = BTreeMap<LanguageType, Language>;
 
@@ -511,105 +480,9 @@ impl Deref for Languages {
         &self.inner
     }
 }
+
 impl DerefMut for Languages {
     fn deref_mut(&mut self) -> &mut BTreeMap<LanguageType, Language> {
         &mut self.inner
     }
 }
-
-
-#[cfg(test)]
-mod accuracy_tests {
-    extern crate tempdir;
-    use super::*;
-    use std::io::Write;
-    use std::fs::File;
-    use language::LanguageType;
-    use self::tempdir::TempDir;
-
-
-    fn test_accuracy(file_name: &'static str,
-                     expected: usize,
-                     contents: &'static str)
-    {
-        let tmp_dir = TempDir::new("test").expect("Couldn't create temp dir");
-        let file_name = tmp_dir.path().join(file_name);
-        let mut file = File::create(&file_name).expect("Couldn't create file");
-        file.write(contents.as_bytes()).expect("couldn't write to file");
-
-        let mut l = Languages::new();
-        let l_type = LanguageType::from_extension(&file_name)
-            .expect("Can't find language type");
-        l.get_statistics(vec![file_name.to_str().unwrap()], vec![]);
-        let language = l.get_mut(&l_type).expect("Couldn't find language");
-
-        assert_eq!(expected, language.code);
-    }
-
-    #[test]
-    fn inside_quotes() {
-        test_accuracy("inside_quotes.rs",
-                      8,
-                      r#"fn main() {
-            let start = "/*";
-            loop {
-                if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */
-                break;
-                }
-            }
-        }"#)
-    }
-
-    #[test]
-    fn shouldnt_panic() {
-        test_accuracy("shouldnt_panic.rs",
-                      9,
-                      r#"fn foo() {
-            let this_ends = "a \"test/*.";
-            call1();
-            call2();
-            let this_does_not = /* a /* nested */ comment " */
-                                "*/another /*test
-            call3();
-            */";
-        }"#)
-    }
-
-    #[test]
-    fn all_quotes_no_comment() {
-        test_accuracy("all_quotes_no_comment.rs",
-                      10,
-                      r#"fn foobar() {
-    let does_not_start = // "
-        "until here,
-        test/*
-        test"; // a quote: "
-    let also_doesnt_start = /* " */
-        "until here,
-        test,*/
-        test"; // another quote: "
-}"#)
-    }
-
-    #[test]
-    fn commenting_on_comments() {
-        test_accuracy("commenting_on_comments.rs",
-                      5,
-                      r#"fn foo() {
-    let a = 4; // /*
-    let b = 5;
-    let c = 6; // */
-}"#)
-    }
-
-    #[test]
-    fn nesting_with_nesting_comments() {
-        test_accuracy("nesting_with_nesting_comments.d",
-                      5,
-                      r#"void main() {
-    auto x = 5; /+ a /+ nested +/ comment /* +/
-    writefln("hello");
-    auto y = 4; // */
-}"#)
-    }
-}

src/main.rs

@@ -12,9 +12,6 @@ mod input;
 use input::*;
 
 use std::borrow::Cow;
-use std::sync::mpsc::channel;
-use std::thread;
-use std::time::Duration;
 
 use clap::App;
 use env_logger::LogBuilder;
@@ -43,11 +40,9 @@ fn main() {
     let verbose_option = matches.occurrences_of("verbose");
     let sort_option = matches.value_of("sort");
     let ignored_directories = {
-        let mut ignored_directories: Vec<&str> = vec![".git"];
+        let mut ignored_directories: Vec<&str> = Vec::new();
         if let Some(user_ignored) = matches.values_of("exclude") {
-            for ignored in user_ignored {
+            ignored_directories.extend(user_ignored);
-                ignored_directories.push(ignored);
-            }
         }
         ignored_directories
     };
@@ -65,7 +60,7 @@ fn main() {
     let mut languages = Languages::new();
 
     if language_option {
-        for key in languages.keys() {
+        for key in LanguageType::list() {
             println!("{:<25}", key);
         }
         return;
@@ -77,31 +72,9 @@ fn main() {
         add_input(input, &mut languages);
     }
 
-    let mut total = Language::new_blank();
-
-    let print_animation = output_option == None;
-    let (tx, rx) = channel();
-    let child = thread::spawn(move || {
-        let time = 100;
-        loop {
-            if let Ok(_) = rx.try_recv() {
-                break;
-            }
-
-            if print_animation {
-                print!(" Counting files.  \r");
-                thread::sleep(Duration::from_millis(time));
-                print!(" Counting files..\r");
-                thread::sleep(Duration::from_millis(time));
-                print!(" Counting files...\r");
-                thread::sleep(Duration::from_millis(time));
-            }
-        }
-    });
-
     languages.get_statistics(paths, ignored_directories);
 
-    if output_option == None {
+    if output_option.is_none() {
         println!("{}", ROW);
         println!(" {:<12} {:>12} {:>12} {:>12} {:>12} {:>12}",
                  "Language",
@@ -111,32 +84,24 @@ fn main() {
                  "Comments",
                  "Blanks");
         println!("{}", ROW);
-    }
 
-    for (name, language) in &languages {
+        if sort_option.is_none() {
-        if !language.is_empty() && sort_option == None && output_option == None {
+            for (name, language) in languages.iter().filter(isnt_empty) {
-            if files_option {
+                if files_option {
-                print_language(language, name);
+                    print_language(language, name);
-                println!("{}", ROW);
+                    println!("{}", ROW);
 
-                for stat in &language.stats {
+                    for stat in &language.stats {
-                    println!("{}", stat);
+                        println!("{}", stat);
+                    }
+                    println!("{}", ROW);
+                } else if output_option.is_none() {
+                    print_language(language, name);
                 }
-                println!("{}", ROW);
-            } else if output_option == None {
-                print_language(language, name);
             }
         }
     }
 
-    let _ = tx.send(());
-    let _ = child.join();
-
-    for (_, language) in &languages {
-        if !language.is_empty() {
-            total += language;
-        }
-    }
 
     if let Some(format) = output_option {
         match_output(format, languages);
@@ -153,7 +118,7 @@ fn main() {
             }
         }
 
-        let mut languages: Vec<_> = languages.into_iter().collect();
+        let mut languages: Vec<_> = languages.iter().collect();
 
         match &*sort_category {
             BLANKS => languages.sort_by(|a, b| b.1.blanks.cmp(&a.1.blanks)),
@@ -184,6 +149,10 @@ fn main() {
         if !files_option {
             println!("{}", ROW);
         }
+        let mut total = Language::new_blank();
+        for (_, language) in languages {
+            total += language;
+        }
         println!(" {: <18} {: >6} {:>12} {:>12} {:>12} {:>12}",
                  "Total",
                  total.stats.len(),
@@ -195,6 +164,9 @@ fn main() {
     }
 }
 
+fn isnt_empty(&(_, language): &(&LanguageType, &Language)) -> bool {
+    !language.is_empty()
+}
 
 fn print_language<'a, C>(language: &'a Language, name: C)
     where C: Into<Cow<'a, LanguageType>>

src/serde_types.in.rs

@@ -1,6 +1,7 @@
+use std::path::PathBuf;
 /// A struct representing the statistics of a file.
 #[cfg_attr(feature = "io", derive(Deserialize, Serialize))]
-#[derive(Clone, Default, Debug, Eq, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
 pub struct Stats {
     /// Number of blank lines within the file.
     pub blanks: usize,
@@ -11,6 +12,6 @@ pub struct Stats {
     /// Total number of lines within the file.
     pub lines: usize,
     /// File name.
-    pub name: String,
+    pub name: PathBuf,
 }
 

src/stats.rs

@@ -10,11 +10,27 @@ impl Stats {
     /// Create a new `Stats` from a file path.
     ///
     /// ```
-    /// # use tokei::*;
+    /// use std::path::PathBuf;
-    /// let stats = Stats::new("src/main.rs");
+    /// use tokei::Stats;
+    ///
+    /// let path = PathBuf::from("src/main.rs");
+    ///
+    /// let stats = Stats::new(path);
     /// ```
-    pub fn new<S: Into<String>>(name: S) -> Self {
+    pub fn new(name: PathBuf) -> Self {
-        Stats { name: name.into(), ..Self::default() }
+        Stats { name: name, ..Self::default() }
+    }
+}
+
+impl Default for Stats {
+    fn default() -> Self {
+        Stats {
+            name: PathBuf::new(),
+            lines: usize::default(),
+            code: usize::default(),
+            comments: usize::default(),
+            blanks: usize::default(),
+        }
     }
 }
 
@@ -27,26 +43,30 @@ fn find_char_boundary(s: &str, index: usize) -> usize {
     unreachable!();
 }
 
-impl fmt::Display for Stats {
+macro_rules! display_stats {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    ($f:expr, $this:expr, $name:expr) => {
-        let name_length = self.name.len();
+        write!($f,
-
-        let name = if name_length == 25 {
-            self.name.clone()
-        } else if self.name.len() > 24 {
-            let mut name = String::from("|");
-            let from = find_char_boundary(&self.name, self.name.len() - 24);
-            name.push_str(&self.name[from..]);
-            name
-        } else {
-            self.name.clone()
-        };
-        write!(f,
                " {: <25} {:>12} {:>12} {:>12} {:>12}",
-               name,
+               $name,
-               self.lines,
+               $this.lines,
-               self.code,
+               $this.code,
-               self.comments,
+               $this.comments,
-               self.blanks)
+               $this.blanks)
+    }
+}
+
+impl fmt::Display for Stats {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let name = self.name.to_string_lossy();
+        let name_length = name.len();
+
+        if name_length == 25 || name_length <= 24 {
+            display_stats!(f, self, name)
+        } else {
+            let mut formatted = String::from("|");
+            let from = find_char_boundary(&name, name_length - 24);
+            formatted.push_str(&name[from..]);
+            display_stats!(f, self, formatted)
+        }
     }
 }

src/utils/fs.rs

@@ -11,20 +11,10 @@ use ignore::WalkBuilder;
 use ignore::overrides::OverrideBuilder;
 use ignore::WalkState::*;
 
-use language::{Language, LanguageType};
+use language::{Language, Languages, LanguageType};
 use language::LanguageType::*;
 pub use language::get_filetype_from_shebang;
 
-macro_rules! get_language {
-    ($languages:expr, $entry:expr) => {
-        if let Some(language_type) = LanguageType::from_extension($entry) {
-            opt_error!($languages.get_mut(&language_type), "Unknown Language? Shouldn't happen.")
-        } else {
-            continue;
-        }
-    }
-}
-
 pub fn get_all_files(paths: Vec<&str>,
                      ignored_directories: Vec<&str>,
                      languages: &mut BTreeMap<LanguageType, Language>)
@@ -84,7 +74,9 @@ pub fn get_all_files(paths: Vec<&str>,
     });
 
     for (language_type, pathbuf) in rx {
-        opt_error!(languages.get_mut(&language_type), "??!").files.push(pathbuf);
+        languages.entry(language_type)
+                 .or_insert(Languages::generate_language(language_type))
+                 .files.push(pathbuf);
     }
 }
 
@@ -124,8 +116,8 @@ mod test {
         create_dir(&path_name).expect("Couldn't create directory.rs within temp");
 
         let mut l = Languages::new();
-        get_all_files(vec![tmp_dir.into_path().to_str().unwrap()].into(), vec![].into(), &mut l);
+        get_all_files(vec![tmp_dir.into_path().to_str().unwrap()], vec![], &mut l);
 
-        assert_eq!(0, l.get(&LanguageType::Rust).unwrap().files.len());
+        assert!(l.get(&LanguageType::Rust).is_none());
     }
 }

src/utils/multi_line.rs

@@ -18,11 +18,11 @@ pub fn handle_multi_line(line: &str,
     'window: for window in line.as_bytes().windows(window_size) {
         while skip != 0 {
             skip -= 1;
-            continue;
+            continue 'window;
         }
 
         if let &mut Some(quote_str) = quote {
-            if window.starts_with(b"\\") {
+            if window.starts_with(br"\") {
                 skip = 1;
             } else if window.starts_with(quote_str.as_bytes()) {
                 *quote = None;

tests/accuracy.rs

@@ -0,0 +1,59 @@
+#[macro_use] extern crate lazy_static;
+extern crate regex;
+extern crate tokei;
+extern crate ignore;
+
+use std::io::Read;
+use std::fs::File;
+
+use regex::Regex;
+use tokei::Languages;
+
+lazy_static! {
+    static ref LINES: Regex = Regex::new(r"\d+ lines").unwrap();
+    static ref CODE: Regex = Regex::new(r"\d+ code").unwrap();
+    static ref COMMENTS: Regex = Regex::new(r"\d+ comments").unwrap();
+    static ref BLANKS: Regex = Regex::new(r"\d+ blanks").unwrap();
+}
+
+macro_rules! get_digit {
+    ($regex:expr, $text:expr) => {{
+        let (begin, end) = $regex.find(&$text).expect("Couldn't find category");
+        $text[begin..end].split_whitespace()
+            .next()
+            .unwrap()
+            .parse::<usize>()
+            .unwrap()
+    }}
+}
+
+#[test]
+fn languages() {
+    use ignore::Walk;
+    let walker = Walk::new("./tests/data/").filter(|p| {
+        match p {
+            &Ok(ref p) => !p.metadata().unwrap().is_dir(),
+            _ => false,
+        }
+    });
+    for path in walker {
+        let path = path.unwrap();
+        let path = path.path().to_str().unwrap();
+        let mut languages = Languages::new();
+        languages.get_statistics(vec![path], vec![]);
+        let mut contents = String::new();
+        File::open(path).unwrap().read_to_string(&mut contents).unwrap();
+
+
+        for (name, language) in languages {
+            assert_eq!(get_digit!(LINES, contents), language.lines);
+            println!("{} LINES MATCH", name);
+            assert_eq!(get_digit!(CODE, contents), language.code);
+            println!("{} CODE MATCH", name);
+            assert_eq!(get_digit!(COMMENTS, contents), language.comments);
+            println!("{} COMMENTS MATCH", name);
+            assert_eq!(get_digit!(BLANKS, contents), language.blanks);
+            println!("{} BLANKS MATCH", name);
+        }
+    }
+}

tests/data/d.d

@@ -0,0 +1,8 @@
+/* 8 lines 5 code 1 comments 2 blanks */
+
+void main() {
+    auto x = 5; /+ a /+ nested +/ comment /* +/
+    writefln("hello");
+    auto y = 4; // */
+}
+

tests/data/fsharp.fs

@@ -0,0 +1,13 @@
+(* 13 lines 5 code 4 comments 4 blanks *)
+
+// Comment
+
+let foo = (*
+    Comment
+*)
+5
+
+let bar = "(*
+    Code
+*)"
+

tests/data/rust.rs

@@ -0,0 +1,38 @@
+// 38 lines 32 code 1 comments 5 blanks
+
+fn main() {
+    let start = "/*";
+    loop {
+        if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */
+            break;
+        }
+    }
+}
+
+fn foo() {
+    let this_ends = "a \"test/*.";
+    call1();
+    call2();
+    let this_does_not = /* a /* nested */ comment " */
+        "*/another /*test
+            call3();
+            */";
+}
+
+fn foobar() {
+    let does_not_start = // "
+        "until here,
+        test/*
+        test"; // a quote: "
+    let also_doesnt_start = /* " */
+        "until here,
+        test,*/
+        test"; // another quote: "
+}
+
+fn foo() {
+    let a = 4; // /*
+    let b = 5;
+    let c = 6; // */
+}
+