mirror of
https://github.com/XAMPPRocky/tokei
synced 2024-08-28 03:39:37 +00:00
4.2.0 fixes #51
This commit is contained in:
parent
e1a4529e2d
commit
68c62f8dc2
17
.travis.yml
17
.travis.yml
|
@ -1,18 +1,19 @@
|
||||||
os:
|
os:
|
||||||
- linux
|
- linux
|
||||||
- osx
|
- osx
|
||||||
|
env:
|
||||||
|
- RUST_TEST_THREADS=1
|
||||||
language: rust
|
language: rust
|
||||||
rust:
|
rust:
|
||||||
- stable
|
- stable
|
||||||
- beta
|
- beta
|
||||||
- nightly
|
- nightly
|
||||||
script:
|
script:
|
||||||
- cargo build --verbose --release
|
- cargo build --verbose --release
|
||||||
- cargo build --verbose --release --features all
|
- cargo build --verbose --release --features all
|
||||||
- cargo test --verbose --release
|
- cargo test --verbose --release
|
||||||
- cargo test --verbose --release --features all
|
- cargo test --verbose --release --features all
|
||||||
matrix:
|
matrix:
|
||||||
allow_failure:
|
allow_failure:
|
||||||
- rust: nightly
|
- rust: nightly
|
||||||
sudo: false
|
sudo: false
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,7 @@ license = "MIT/Apache-2.0"
|
||||||
name = "tokei"
|
name = "tokei"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
repository = "https://github.com/Aaronepower/tokei.git"
|
repository = "https://github.com/Aaronepower/tokei.git"
|
||||||
version = "4.1.0"
|
version = "4.2.0"
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
doc = false
|
doc = false
|
||||||
|
|
2
cli.yml
2
cli.yml
|
@ -5,7 +5,7 @@ about: Count Code, Quickly.
|
||||||
author: Aaron P. <theaaronepower@gmail.com>
|
author: Aaron P. <theaaronepower@gmail.com>
|
||||||
bin_name: Tokei
|
bin_name: Tokei
|
||||||
name: Tokei
|
name: Tokei
|
||||||
version: 4.1.0
|
version: 4.2.0
|
||||||
args:
|
args:
|
||||||
- exclude:
|
- exclude:
|
||||||
help: Ignore all files & directories containing the word.
|
help: Ignore all files & directories containing the word.
|
||||||
|
|
|
@ -31,6 +31,12 @@ pub struct Language {
|
||||||
/// Whether the language supports nested multi line comments or not.
|
/// Whether the language supports nested multi line comments or not.
|
||||||
#[serde(skip_deserializing, skip_serializing)]
|
#[serde(skip_deserializing, skip_serializing)]
|
||||||
pub nested: bool,
|
pub nested: bool,
|
||||||
|
/// A list of specific nested comments if this is empty all `multi_line` comments count.
|
||||||
|
#[serde(skip_deserializing, skip_serializing)]
|
||||||
|
pub nested_comments: Vec<(&'static str, &'static str)>,
|
||||||
|
/// A list of quotes by default it is `""`.
|
||||||
|
#[serde(skip_deserializing, skip_serializing)]
|
||||||
|
pub quotes: Vec<(&'static str, &'static str)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "io"))]
|
#[cfg(not(feature = "io"))]
|
||||||
|
@ -54,6 +60,10 @@ pub struct Language {
|
||||||
pub multi_line: Vec<(&'static str, &'static str)>,
|
pub multi_line: Vec<(&'static str, &'static str)>,
|
||||||
/// Whether the language supports nested multi line comments or not.
|
/// Whether the language supports nested multi line comments or not.
|
||||||
pub nested: bool,
|
pub nested: bool,
|
||||||
|
/// A list of specific nested comments if this is empty all `multi_line` comments count.
|
||||||
|
pub nested_comments: Vec<(&'static str, &'static str)>,
|
||||||
|
/// A list of quotes by default it is `""`.
|
||||||
|
pub quotes: Vec<(&'static str, &'static str)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -101,6 +111,7 @@ impl Language {
|
||||||
Language {
|
Language {
|
||||||
line_comment: vec!["//"],
|
line_comment: vec!["//"],
|
||||||
multi_line: vec![("/*", "*/")],
|
multi_line: vec![("/*", "*/")],
|
||||||
|
quotes: vec![("\"", "\"")],
|
||||||
..Self::default()
|
..Self::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -116,7 +127,11 @@ impl Language {
|
||||||
/// assert_eq!(ocaml.multi_line, coq.multi_line);
|
/// assert_eq!(ocaml.multi_line, coq.multi_line);
|
||||||
/// ```
|
/// ```
|
||||||
pub fn new_func() -> Self {
|
pub fn new_func() -> Self {
|
||||||
Language { multi_line: vec![("(*", "*)")], ..Self::default() }
|
Language {
|
||||||
|
multi_line: vec![("(*", "*)")],
|
||||||
|
quotes: vec![("\"", "\"")],
|
||||||
|
..Self::default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Convience constructor for creating a language that has the same commenting syntax as HTML like languages.
|
/// Convience constructor for creating a language that has the same commenting syntax as HTML like languages.
|
||||||
|
@ -130,7 +145,11 @@ impl Language {
|
||||||
/// assert_eq!(xml.multi_line, html.multi_line);
|
/// assert_eq!(xml.multi_line, html.multi_line);
|
||||||
/// ```
|
/// ```
|
||||||
pub fn new_html() -> Self {
|
pub fn new_html() -> Self {
|
||||||
Language { multi_line: vec![("<!--", "-->")], ..Self::default() }
|
Language {
|
||||||
|
multi_line: vec![("<!--", "-->")],
|
||||||
|
quotes: vec![("\"", "\"")],
|
||||||
|
..Self::default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Convience constructor for creating a language that has the same commenting syntax as Bash.
|
/// Convience constructor for creating a language that has the same commenting syntax as Bash.
|
||||||
|
@ -154,7 +173,11 @@ impl Language {
|
||||||
/// let mustache = Language::new_multi(vec![("{{!", "}}")]);
|
/// let mustache = Language::new_multi(vec![("{{!", "}}")]);
|
||||||
/// ```
|
/// ```
|
||||||
pub fn new_multi(multi_line: Vec<(&'static str, &'static str)>) -> Self {
|
pub fn new_multi(multi_line: Vec<(&'static str, &'static str)>) -> Self {
|
||||||
Language { multi_line: multi_line, ..Self::default() }
|
Language {
|
||||||
|
multi_line: multi_line,
|
||||||
|
quotes: vec![("\"", "\"")],
|
||||||
|
..Self::default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Convience constructor for creating a language that has the same commenting syntax as Prolog.
|
/// Convience constructor for creating a language that has the same commenting syntax as Prolog.
|
||||||
|
@ -171,6 +194,7 @@ impl Language {
|
||||||
Language {
|
Language {
|
||||||
line_comment: vec!["%"],
|
line_comment: vec!["%"],
|
||||||
multi_line: vec![("/*", "*/")],
|
multi_line: vec![("/*", "*/")],
|
||||||
|
quotes: vec![("\"", "\"")],
|
||||||
..Self::default()
|
..Self::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -182,7 +206,11 @@ impl Language {
|
||||||
/// let haskell = Language::new_single(vec!["--"]);
|
/// let haskell = Language::new_single(vec!["--"]);
|
||||||
/// ```
|
/// ```
|
||||||
pub fn new_single(line_comment: Vec<&'static str>) -> Self {
|
pub fn new_single(line_comment: Vec<&'static str>) -> Self {
|
||||||
Language { line_comment: line_comment, ..Self::default() }
|
Language {
|
||||||
|
line_comment: line_comment,
|
||||||
|
quotes: vec![("\"", "\"")],
|
||||||
|
..Self::default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks if the language is empty. Empty meaning it doesn't have any statistics.
|
/// Checks if the language is empty. Empty meaning it doesn't have any statistics.
|
||||||
|
@ -221,7 +249,40 @@ impl Language {
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sorts each of the `Stats` structs contained in the language based on what category is provided
|
/// Specify if the the language supports nested multi line comments.
|
||||||
|
/// And which are nested. If this is specified there is no need to
|
||||||
|
/// call the `nested` function.
|
||||||
|
///
|
||||||
|
/// ```
|
||||||
|
/// # use tokei::*;
|
||||||
|
/// let mut d = Language::new(vec!["//"], vec![("/*", "*/")])
|
||||||
|
/// .nested_comments(vec![("/+", "+/")]);
|
||||||
|
/// assert!(d.nested);
|
||||||
|
/// assert_eq!(d.nested_comments, vec![("/+", "+/")]);
|
||||||
|
/// ```
|
||||||
|
pub fn nested_comments(mut self, nested_comments: Vec<(&'static str, &'static str)>) -> Self {
|
||||||
|
self.nested = true;
|
||||||
|
self.nested_comments = nested_comments;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Specifies if the language has a quotes to define a string where
|
||||||
|
/// the commenting syntax would be ignored. By default it is only
|
||||||
|
/// `""` quotes that are ignored.
|
||||||
|
///
|
||||||
|
/// ```
|
||||||
|
/// # use tokei::*;
|
||||||
|
/// let mut javascript = Language::new(vec!["//"], vec![("/*", "*/")])
|
||||||
|
/// .set_quotes(vec![("\"", "\""), ("'", "'")]);
|
||||||
|
/// assert!(!javascript.quotes.is_empty());
|
||||||
|
/// ```
|
||||||
|
pub fn set_quotes(mut self, quotes: Vec<(&'static str, &'static str)>) -> Self {
|
||||||
|
self.quotes = quotes;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sorts each of the `Stats` structs contained in the language based
|
||||||
|
/// on what category is provided
|
||||||
/// panic!'s if given the wrong category.
|
/// panic!'s if given the wrong category.
|
||||||
///
|
///
|
||||||
/// ```
|
/// ```
|
||||||
|
|
|
@ -7,7 +7,7 @@ use std::borrow::Cow;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use utils::*;
|
use utils::fs;
|
||||||
use self::LanguageType::*;
|
use self::LanguageType::*;
|
||||||
|
|
||||||
#[cfg_attr(feature = "io", derive(Deserialize, Serialize))]
|
#[cfg_attr(feature = "io", derive(Deserialize, Serialize))]
|
||||||
|
@ -269,7 +269,7 @@ impl LanguageType {
|
||||||
/// assert_eq!(rust, Some(LanguageType::Rust));
|
/// assert_eq!(rust, Some(LanguageType::Rust));
|
||||||
/// ```
|
/// ```
|
||||||
pub fn from_extension<P: AsRef<Path>>(entry: P) -> Option<Self> {
|
pub fn from_extension<P: AsRef<Path>>(entry: P) -> Option<Self> {
|
||||||
if let Some(extension) = get_extension(entry) {
|
if let Some(extension) = fs::get_extension(entry) {
|
||||||
match &*extension {
|
match &*extension {
|
||||||
"as" => Some(ActionScript),
|
"as" => Some(ActionScript),
|
||||||
"ada" | "adb" | "ads" => Some(Ada),
|
"ada" | "adb" | "ads" => Some(Ada),
|
||||||
|
|
|
@ -19,7 +19,7 @@ use serde_yaml;
|
||||||
use toml;
|
use toml;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
|
|
||||||
use utils::*;
|
use utils::{fs, multi_line};
|
||||||
use super::{Language, LanguageType};
|
use super::{Language, LanguageType};
|
||||||
use super::LanguageType::*;
|
use super::LanguageType::*;
|
||||||
use stats::Stats;
|
use stats::Stats;
|
||||||
|
@ -31,6 +31,87 @@ const TOML_ERROR: &'static str = "Tokei was not compiled with the `toml-io` flag
|
||||||
#[cfg(not(feature = "yaml"))]
|
#[cfg(not(feature = "yaml"))]
|
||||||
const YAML_ERROR: &'static str = "Tokei was not compiled with the `yaml` flag.";
|
const YAML_ERROR: &'static str = "Tokei was not compiled with the `yaml` flag.";
|
||||||
|
|
||||||
|
fn count_files(language_tuple: &mut (&LanguageType, &mut Language)) {
|
||||||
|
|
||||||
|
let &mut (name, ref mut language) = language_tuple;
|
||||||
|
|
||||||
|
if language.files.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let is_fortran = name == &FortranModern || name == &FortranLegacy;
|
||||||
|
|
||||||
|
let files: Vec<_> = language.files.drain(..).collect();
|
||||||
|
let mut contents = String::new();
|
||||||
|
let mut stack = vec![];
|
||||||
|
let mut quote = None;
|
||||||
|
|
||||||
|
for file in files {
|
||||||
|
let mut stats = Stats::new(opt_or_cont!(file.to_str()));
|
||||||
|
stack.clear();
|
||||||
|
contents.clear();
|
||||||
|
|
||||||
|
rs_or_cont!(rs_or_cont!(File::open(file)).read_to_string(&mut contents));
|
||||||
|
|
||||||
|
let lines = contents.lines();
|
||||||
|
|
||||||
|
if language.is_blank() {
|
||||||
|
stats.code += lines.count();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
'line: for line in lines {
|
||||||
|
stats.lines += 1;
|
||||||
|
// FORTRAN has a rule where it only counts as a comment if it's the first
|
||||||
|
// character in the column, so removing starting whitespace could cause a
|
||||||
|
// miscount.
|
||||||
|
let line = if is_fortran {
|
||||||
|
line
|
||||||
|
} else {
|
||||||
|
line.trim()
|
||||||
|
};
|
||||||
|
|
||||||
|
if line.trim().is_empty() {
|
||||||
|
stats.blanks += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for single in &language.line_comment {
|
||||||
|
if line.starts_with(single) {
|
||||||
|
stats.comments += 1;
|
||||||
|
continue 'line;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut started_with = false;
|
||||||
|
if quote.is_none() {
|
||||||
|
let chain = language.multi_line.iter().chain(language.nested_comments.iter());
|
||||||
|
|
||||||
|
for &(start, _) in chain {
|
||||||
|
if line.starts_with(start) {
|
||||||
|
started_with = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
multi_line::handle_multi_line(line, &language, &mut stack, &mut quote);
|
||||||
|
|
||||||
|
if !stack.is_empty() {
|
||||||
|
if started_with {
|
||||||
|
stats.code += 1;
|
||||||
|
} else {
|
||||||
|
stats.comments += 1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
stats.code += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
**language += stats;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// A collection of existing languages([_List of Languages_](https://github.com/Aaronepower/tokei#supported-languages))
|
/// A collection of existing languages([_List of Languages_](https://github.com/Aaronepower/tokei#supported-languages))
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Languages {
|
pub struct Languages {
|
||||||
|
@ -168,97 +249,11 @@ impl Languages {
|
||||||
where I: Into<Cow<'a, [&'a str]>>
|
where I: Into<Cow<'a, [&'a str]>>
|
||||||
{
|
{
|
||||||
|
|
||||||
get_all_files(paths.into(), ignored.into(), &mut self.inner);
|
fs::get_all_files(paths.into(), ignored.into(), &mut self.inner);
|
||||||
|
|
||||||
let mut language_iter: Vec<_> = self.inner.iter_mut().collect();
|
let mut language_iter: Vec<_> = self.inner.iter_mut().collect();
|
||||||
|
|
||||||
language_iter.par_iter_mut().for_each(|&mut (name, ref mut language)| {
|
language_iter.par_iter_mut().for_each(count_files);
|
||||||
if language.files.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let is_fortran = name == &FortranModern || name == &FortranLegacy;
|
|
||||||
|
|
||||||
let files: Vec<_> = language.files.drain(..).collect();
|
|
||||||
let mut contents = String::new();
|
|
||||||
|
|
||||||
for file in files {
|
|
||||||
let mut is_in_comments = false;
|
|
||||||
let mut previous_comment_start = "";
|
|
||||||
let mut comment_depth: usize = 0;
|
|
||||||
let mut stats = Stats::new(opt_or_cont!(file.to_str()));
|
|
||||||
|
|
||||||
let contents = {
|
|
||||||
contents.clear();
|
|
||||||
let _ = rs_or_cont!(rs_or_cont!(File::open(file))
|
|
||||||
.read_to_string(&mut contents));
|
|
||||||
contents
|
|
||||||
};
|
|
||||||
|
|
||||||
let lines = contents.lines();
|
|
||||||
|
|
||||||
if language.is_blank() {
|
|
||||||
stats.code += lines.count();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
'line: for line in lines {
|
|
||||||
stats.lines += 1;
|
|
||||||
|
|
||||||
// FORTRAN has a rule where it only counts as a comment if it's the first
|
|
||||||
// character in the column, so removing starting whitespace could cause a
|
|
||||||
// miscount.
|
|
||||||
let line = if is_fortran {
|
|
||||||
line
|
|
||||||
} else {
|
|
||||||
line.trim()
|
|
||||||
};
|
|
||||||
|
|
||||||
if line.trim().is_empty() {
|
|
||||||
stats.blanks += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if line.starts_with(multi_line) {
|
|
||||||
if let Some(multi_line) = has_trailing_comments(line, &language) {
|
|
||||||
previous_comment_start = multi_line;
|
|
||||||
is_in_comments = true;
|
|
||||||
if language.nested {
|
|
||||||
comment_depth += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if is_in_comments {
|
|
||||||
for &(multi_line, multi_line_end) in &language.multi_line {
|
|
||||||
if multi_line == previous_comment_start {
|
|
||||||
if let Some(pos) = line.find(multi_line_end) {
|
|
||||||
if language.nested {
|
|
||||||
comment_depth -= 1;
|
|
||||||
if comment_depth == 0 {
|
|
||||||
is_in_comments = false;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
is_in_comments = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stats.comments += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
for single in &language.line_comment {
|
|
||||||
if line.starts_with(single) {
|
|
||||||
stats.comments += 1;
|
|
||||||
continue 'line;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stats.code += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
**language += stats;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Constructs a new, blank `Languages`.
|
/// Constructs a new, blank `Languages`.
|
||||||
|
@ -279,7 +274,8 @@ impl Languages {
|
||||||
C => Language::new_c(),
|
C => Language::new_c(),
|
||||||
CHeader => Language::new_c(),
|
CHeader => Language::new_c(),
|
||||||
Clojure => Language::new_single(vec![";","#"]),
|
Clojure => Language::new_single(vec![";","#"]),
|
||||||
CoffeeScript => Language::new(vec!["#"], vec![("###", "###")]),
|
CoffeeScript => Language::new(vec!["#"], vec![("###", "###")])
|
||||||
|
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||||
ColdFusion => Language::new_multi(vec![("<!---", "--->")]),
|
ColdFusion => Language::new_multi(vec![("<!---", "--->")]),
|
||||||
ColdFusionScript => Language::new_c(),
|
ColdFusionScript => Language::new_c(),
|
||||||
Coq => Language::new_func(),
|
Coq => Language::new_func(),
|
||||||
|
@ -287,8 +283,9 @@ impl Languages {
|
||||||
CppHeader => Language::new_c(),
|
CppHeader => Language::new_c(),
|
||||||
CSharp => Language::new_c(),
|
CSharp => Language::new_c(),
|
||||||
CShell => Language::new_hash(),
|
CShell => Language::new_hash(),
|
||||||
Css => Language::new_c(),
|
Css => Language::new_c()
|
||||||
D => Language::new(vec!["//"], vec![("/*", "*/"), ("/+", "+/")]).nested(),
|
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||||
|
D => Language::new(vec!["//"], vec![("/*", "*/")]).nested_comments(vec![("/+", "+/")]),
|
||||||
Dart => Language::new_c(),
|
Dart => Language::new_c(),
|
||||||
DeviceTree => Language::new_c(),
|
DeviceTree => Language::new_c(),
|
||||||
Erlang => Language::new_single(vec!["%"]),
|
Erlang => Language::new_single(vec!["%"]),
|
||||||
|
@ -296,9 +293,11 @@ impl Languages {
|
||||||
FortranLegacy => Language::new_single(vec!["c","C","!","*"]),
|
FortranLegacy => Language::new_single(vec!["c","C","!","*"]),
|
||||||
FortranModern => Language::new_single(vec!["!"]),
|
FortranModern => Language::new_single(vec!["!"]),
|
||||||
Go => Language::new_c(),
|
Go => Language::new_c(),
|
||||||
Handlebars => Language::new_multi(vec![("<!--", "-->"), ("{{!", "}}")]),
|
Handlebars => Language::new_multi(vec![("<!--", "-->"), ("{{!", "}}")])
|
||||||
|
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||||
Haskell => Language::new_single(vec!["--"]),
|
Haskell => Language::new_single(vec!["--"]),
|
||||||
Html => Language::new_html(),
|
Html => Language::new_html()
|
||||||
|
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||||
Idris => Language::new(vec!["--"], vec![("{-", "-}")]),
|
Idris => Language::new(vec!["--"], vec![("{-", "-}")]),
|
||||||
Isabelle => Language::new(
|
Isabelle => Language::new(
|
||||||
vec!["--"],
|
vec!["--"],
|
||||||
|
@ -310,9 +309,11 @@ impl Languages {
|
||||||
),
|
),
|
||||||
Jai => Language::new_c(),
|
Jai => Language::new_c(),
|
||||||
Java => Language::new_c(),
|
Java => Language::new_c(),
|
||||||
JavaScript => Language::new_c(),
|
JavaScript => Language::new_c()
|
||||||
|
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||||
Json => Language::new_blank(),
|
Json => Language::new_blank(),
|
||||||
Jsx => Language::new_c(),
|
Jsx => Language::new_c()
|
||||||
|
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||||
Julia => Language::new(vec!["#"], vec![("#=", "=#")]),
|
Julia => Language::new(vec!["#"], vec![("#=", "=#")]),
|
||||||
Kotlin => Language::new_c(),
|
Kotlin => Language::new_c(),
|
||||||
Less => Language::new_c(),
|
Less => Language::new_c(),
|
||||||
|
@ -580,12 +581,14 @@ impl DerefMut for Languages {
|
||||||
mod accuracy_tests {
|
mod accuracy_tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use std::io;
|
use std::io;
|
||||||
|
use language::LanguageType;
|
||||||
|
|
||||||
fn write(contents: &'static str, extension: &'static str) -> io::Result<()> {
|
fn write(contents: &'static str, file_name: &str) -> io::Result<()> {
|
||||||
use std::io::prelude::*;
|
use std::io::prelude::*;
|
||||||
use std::fs::File;
|
use std::fs::{File, create_dir};
|
||||||
|
|
||||||
let mut f = try!(File::create(format!("./_temp/_temp_file.{}", extension)));
|
let _ = create_dir("./_temp/");
|
||||||
|
let mut f = try!(File::create(file_name));
|
||||||
|
|
||||||
try!(f.write_all(&contents.as_bytes()));
|
try!(f.write_all(&contents.as_bytes()));
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -596,93 +599,84 @@ mod accuracy_tests {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn test_accuracy(ext: &'static str, num: u32, contents: &'static str) {
|
fn test_accuracy(file_name: &'static str, expected: u32, contents: &'static str) {
|
||||||
write(contents, ext).unwrap();
|
let file_name = format!("./_temp/{}", file_name);
|
||||||
|
write(contents, &*file_name).unwrap();
|
||||||
let mut l = Languages::new();
|
let mut l = Languages::new();
|
||||||
|
let l_type = LanguageType::from_extension(file_name).expect("Can't find language type");
|
||||||
|
|
||||||
l.get_statistics(vec!["./temp/"], vec![]);
|
l.get_statistics(vec!["./_temp/"], vec![]);
|
||||||
|
|
||||||
assert_eq!(num as usize,
|
|
||||||
l.get_mut(&::language::LanguageType::from(ext)).unwrap().code);
|
|
||||||
let _ = cleanup();
|
let _ = cleanup();
|
||||||
|
let language = l.get_mut(&l_type).expect("Couldn't find language");
|
||||||
|
|
||||||
|
assert_eq!(expected as usize, language.code);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn inside_quotes() {
|
fn inside_quotes() {
|
||||||
test_accuracy("rs",
|
test_accuracy("inside_quotes.rs",
|
||||||
8,
|
8,
|
||||||
"fn main() {
|
r#"fn main() {
|
||||||
let start = \"/*\";
|
let start = "/*";
|
||||||
loop {
|
loop {
|
||||||
\
|
if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */
|
||||||
if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */
|
break;
|
||||||
\
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}"#)
|
||||||
")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn shouldnt_panic() {
|
fn shouldnt_panic() {
|
||||||
test_accuracy("rs",
|
test_accuracy("shouldnt_panic.rs",
|
||||||
9,
|
9,
|
||||||
"fn foo() {
|
r#"fn foo() {
|
||||||
let this_ends = \"a \\\"test/*.\";
|
let this_ends = "a \"test/*.";
|
||||||
\
|
call1();
|
||||||
call1();
|
|
||||||
call2();
|
call2();
|
||||||
let this_does_not = /* a /* \
|
let this_does_not = /* a /* nested */ comment " */
|
||||||
nested */ comment \" */
|
"*/another /*test
|
||||||
\"*/another /*test
|
call3();
|
||||||
\
|
*/";
|
||||||
call3();
|
}"#)
|
||||||
*/\";
|
|
||||||
}")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn all_quotes_no_comment() {
|
fn all_quotes_no_comment() {
|
||||||
test_accuracy("rs",
|
test_accuracy("all_quotes_no_comment.rs",
|
||||||
10,
|
10,
|
||||||
"fn foobar() {
|
r#"fn foobar() {
|
||||||
let does_not_start = // \"
|
let does_not_start = // "
|
||||||
\"until here,
|
"until here,
|
||||||
\
|
test/*
|
||||||
test/*
|
test"; // a quote: "
|
||||||
test\"; // a quote: \"
|
let also_doesnt_start = /* " */
|
||||||
let also_doesnt_start = \
|
"until here,
|
||||||
/* \" */
|
test,*/
|
||||||
\"until here,
|
test"; // another quote: "
|
||||||
test,*/
|
}"#)
|
||||||
test\"; \
|
|
||||||
// another quote: \"
|
|
||||||
}")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn commenting_on_comments() {
|
fn commenting_on_comments() {
|
||||||
test_accuracy("rs",
|
test_accuracy("commenting_on_comments.rs",
|
||||||
5,
|
5,
|
||||||
"fn foo() {
|
r#"fn foo() {
|
||||||
let a = 4; // /*
|
let a = 4; // /*
|
||||||
let b = 5;
|
let b = 5;
|
||||||
\
|
let c = 6; // */
|
||||||
let c = 6; // */
|
}"#)
|
||||||
}")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn deez_nesting_comments() {
|
fn nesting_with_nesting_comments() {
|
||||||
test_accuracy("d",
|
test_accuracy("nesting_with_nesting_comments.d",
|
||||||
5,
|
5,
|
||||||
"void main() {
|
r#"void main() {
|
||||||
auto x = 5; /+ a /+ nested +/ comment /* +/
|
auto x = 5; /+ a /+ nested +/ comment /* +/
|
||||||
\
|
writefln("hello");
|
||||||
writefln(\"hello\");
|
auto y = 4; // */
|
||||||
auto y = 4; // */
|
}"#)
|
||||||
}")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,78 +14,6 @@ use walkdir::{WalkDir, WalkDirIterator};
|
||||||
use language::{Language, LanguageType};
|
use language::{Language, LanguageType};
|
||||||
use language::LanguageType::*;
|
use language::LanguageType::*;
|
||||||
|
|
||||||
pub fn handle_multi_line() -> usize {
|
|
||||||
unreachable!()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// This is used to catch lines like "let x = 5; /* Comment */"
|
|
||||||
pub fn has_trailing_comments(line: &str, language: &Language) -> bool {
|
|
||||||
let mut is_in_comments = 0u64;
|
|
||||||
|
|
||||||
line = if !language.single.is_empty() {
|
|
||||||
let found = None;
|
|
||||||
for single in &language.line_comment {
|
|
||||||
if let Some(pos) = line.find(single) {
|
|
||||||
found = Some(pos);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(pos) = found {
|
|
||||||
&line[0..pos]
|
|
||||||
} else {
|
|
||||||
line
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
line
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut unfinished_comments = vec![];
|
|
||||||
for &(comment, comment_end) in &language {
|
|
||||||
let start = match line.find(comment) {
|
|
||||||
Some(start) => start,
|
|
||||||
None => continue,
|
|
||||||
};
|
|
||||||
|
|
||||||
// This should short circuit 99% of languages.
|
|
||||||
if !language.nested && language.multi_line.len() == 1 {
|
|
||||||
if let Some(end) = line.rfind(comment_end) {
|
|
||||||
if let Some(end_check) = line.rfind(comment) {
|
|
||||||
if end_check > end {
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut chars = line[start..end + comment_end.len()].chars();
|
|
||||||
loop {
|
|
||||||
let window = chars.as_str();
|
|
||||||
|
|
||||||
if window.starts_with(comment) {
|
|
||||||
if nested {
|
|
||||||
is_in_comments += 1;
|
|
||||||
} else {
|
|
||||||
is_in_comments = 1;
|
|
||||||
}
|
|
||||||
} else if window.starts_with(comment_end) {
|
|
||||||
is_in_comments = is_in_comments.saturating_sub(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if chars.next().is_none() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
is_in_comments != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_all_files<'a>(paths: Cow<'a, [&'a str]>,
|
pub fn get_all_files<'a>(paths: Cow<'a, [&'a str]>,
|
||||||
ignored_directories: Cow<'a, [&'a str]>,
|
ignored_directories: Cow<'a, [&'a str]>,
|
||||||
languages: &mut BTreeMap<LanguageType, Language>) {
|
languages: &mut BTreeMap<LanguageType, Language>) {
|
||||||
|
@ -186,42 +114,3 @@ pub fn get_filetype_from_shebang<P: AsRef<Path>>(file: P) -> Option<&'static str
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn both_comments_in_line() {
|
|
||||||
assert!(!has_trailing_comments("Hello /* /* */ World", "//", ("/*", "*/"), false));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn comment_hidden_in_single() {
|
|
||||||
assert!(has_trailing_comments("Hello /* World // */", "//", ("/*", "*/"), true))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn comment_start_in_line() {
|
|
||||||
assert!(has_trailing_comments("Hello /* World", "//", ("/*", "*/"), false));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn both_comments_in_line_nested() {
|
|
||||||
assert!(has_trailing_comments("Hello (* (* *) World", "--", ("(*", "*)"), true));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn comments_of_uneven_length() {
|
|
||||||
assert!(has_trailing_comments("Hello \\<open> \\<open> \\<close> World",
|
|
||||||
"",
|
|
||||||
("\\<open>", "\\<close>"),
|
|
||||||
true));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn comment_start_in_line_nested() {
|
|
||||||
assert!(has_trailing_comments("Hello (* World", "", ("(*", "*)"), true));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -2,6 +2,3 @@
|
||||||
mod macros;
|
mod macros;
|
||||||
pub mod fs;
|
pub mod fs;
|
||||||
pub mod multi_line;
|
pub mod multi_line;
|
||||||
|
|
||||||
pub use self::fs::*;
|
|
||||||
pub use self::multi_line::*;
|
|
||||||
|
|
|
@ -1,40 +1,20 @@
|
||||||
use std::cmp;
|
use language::Language;
|
||||||
|
|
||||||
/// This is used to catch lines like "let x = 5; /* Comment */"
|
/// This is used to catch lines like "let x = 5; /* Comment */"
|
||||||
pub fn has_trailing_comments(line: &str, language: &Language) -> Vec<&'static str> {
|
pub fn handle_multi_line(line: &str,
|
||||||
let line = slice_to_single(line, language);
|
language: &Language,
|
||||||
let mut is_in_comments = 0u64;
|
stack: &mut Vec<&'static str>,
|
||||||
let mut start = None;
|
quote: &mut Option<&'static str>) {
|
||||||
let mut stack = vec![];
|
let mut chars = line.chars();
|
||||||
|
|
||||||
for &(comment, comment_end) in &language.multi_line {
|
|
||||||
start = line.find(comment).and_then(|x| cmp::min(x, start.unwrap_or(x)));
|
|
||||||
|
|
||||||
// This should short circuit 99% of languages.
|
|
||||||
if start.is_none() && !language.nested && language.multi_line.len() == 1 {
|
|
||||||
if let Some(end) = line.rfind(comment_end) {
|
|
||||||
if let Some(end_check) = line.rfind(comment) {
|
|
||||||
if end_check > end {
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let start = match start {
|
|
||||||
Some(pos) => pos,
|
|
||||||
None => return stack,
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut chars = line[start..].chars();
|
|
||||||
let mut cont = false;
|
let mut cont = false;
|
||||||
loop {
|
let nested_is_empty = language.nested_comments.is_empty();
|
||||||
|
|
||||||
|
'window: loop {
|
||||||
let window = chars.as_str();
|
let window = chars.as_str();
|
||||||
|
if window.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
chars.next();
|
||||||
|
|
||||||
// Prevents counting overlaps like /*/*
|
// Prevents counting overlaps like /*/*
|
||||||
if cont {
|
if cont {
|
||||||
|
@ -42,90 +22,135 @@ pub fn has_trailing_comments(line: &str, language: &Language) -> Vec<&'static st
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut end = false;
|
||||||
|
|
||||||
|
if let &mut Some(quote_str) = quote {
|
||||||
|
if window.starts_with("\\") {
|
||||||
|
cont = true;
|
||||||
|
continue;
|
||||||
|
} else if window.starts_with(quote_str) {
|
||||||
|
end = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if end {
|
||||||
|
if let &mut Some(quote_str) = quote {
|
||||||
|
*quote = None;
|
||||||
|
|
||||||
|
if quote_str.chars().count() == 1 {
|
||||||
|
cont = true
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if quote.is_some() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut pop = false;
|
||||||
if let Some(last) = stack.last() {
|
if let Some(last) = stack.last() {
|
||||||
if window.starts_with(last) {
|
if window.starts_with(last) {
|
||||||
stack.pop();
|
pop = true;
|
||||||
cont = true;
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for &(comment, comment_end) in &language.multi_line {
|
if pop {
|
||||||
|
stack.pop();
|
||||||
|
cont = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if stack.is_empty() {
|
||||||
|
for &(start, end) in &language.quotes {
|
||||||
|
if window.starts_with(start) {
|
||||||
|
*quote = Some(end);
|
||||||
|
cont = true;
|
||||||
|
continue 'window;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
for comment in &language.line_comment {
|
||||||
if window.starts_with(comment) {
|
if window.starts_with(comment) {
|
||||||
if nested {
|
break 'window;
|
||||||
stack.push(comment_end);
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for &(start, end) in &language.nested_comments {
|
||||||
|
if window.starts_with(start) {
|
||||||
|
stack.push(end);
|
||||||
|
cont = true;
|
||||||
|
continue 'window;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for &(start, end) in &language.multi_line {
|
||||||
|
if window.starts_with(start) {
|
||||||
|
if language.nested && nested_is_empty {
|
||||||
|
stack.push(end);
|
||||||
} else if stack.len() == 0 {
|
} else if stack.len() == 0 {
|
||||||
stack.push(comment_end);
|
stack.push(end);
|
||||||
}
|
}
|
||||||
cont = true;
|
cont = true;
|
||||||
continue;
|
continue 'window;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if chars.next().is_none() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stack
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline]
|
|
||||||
fn slice_to_single(line: &str, language: &language) -> &str {
|
|
||||||
if !language.single.is_empty() {
|
|
||||||
let found = None;
|
|
||||||
for single in &language.line_comment {
|
|
||||||
if let Some(pos) = line.find(single) {
|
|
||||||
found = Some(pos);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(pos) = found {
|
|
||||||
&line[0..pos]
|
|
||||||
} else {
|
|
||||||
line
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
line
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use language::Language;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn both_comments_in_line() {
|
fn both_comments_in_line() {
|
||||||
assert!(!has_trailing_comments("Hello /* /* */ World", "//", ("/*", "*/"), false));
|
let mut stack = vec![];
|
||||||
|
let mut quote = None;
|
||||||
|
let language = Language::new_c();
|
||||||
|
handle_multi_line("Hello /* /* */ World", &language, &mut stack, &mut quote);
|
||||||
|
assert_eq!(stack.len(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn comment_hidden_in_single() {
|
fn comment_hidden_in_single() {
|
||||||
assert!(has_trailing_comments("Hello /* World // */", "//", ("/*", "*/"), true))
|
let mut stack = vec![];
|
||||||
|
let mut quote = None;
|
||||||
|
let language = Language::new_c();
|
||||||
|
handle_multi_line("Hello World // /*", &language, &mut stack, &mut quote);
|
||||||
|
assert_eq!(stack.len(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn comment_start_in_line() {
|
fn comment_start() {
|
||||||
assert!(has_trailing_comments("Hello /* World", "//", ("/*", "*/"), false));
|
let mut stack = vec![];
|
||||||
|
let mut quote = None;
|
||||||
|
let language = Language::new_c();
|
||||||
|
handle_multi_line("/*Hello World", &language, &mut stack, &mut quote);
|
||||||
|
assert_eq!(stack.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn both_comments_in_line_nested() {
|
fn both_comments_in_line_nested() {
|
||||||
assert!(has_trailing_comments("Hello (* (* *) World", "--", ("(*", "*)"), true));
|
let mut stack = vec![];
|
||||||
|
let mut quote = None;
|
||||||
|
let language = Language::new_func().nested();
|
||||||
|
handle_multi_line("Hello (* (* *) World", &language, &mut stack, &mut quote);
|
||||||
|
assert_eq!(stack.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn comments_of_uneven_length() {
|
fn comments_of_uneven_length() {
|
||||||
assert!(has_trailing_comments("Hello \\<open> \\<open> \\<close> World",
|
let mut stack = vec![];
|
||||||
"",
|
let mut quote = None;
|
||||||
("\\<open>", "\\<close>"),
|
let language = Language::new(vec![], vec![("\\<open>", "\\<close>")]).nested();
|
||||||
true));
|
handle_multi_line("Hello \\<open> \\<open> \\<close> World",
|
||||||
}
|
&language,
|
||||||
|
&mut stack,
|
||||||
#[test]
|
&mut quote);
|
||||||
fn comment_start_in_line_nested() {
|
assert_eq!(stack.len(), 1);
|
||||||
assert!(has_trailing_comments("Hello (* World", "", ("(*", "*)"), true));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue