mirror of
https://github.com/XAMPPRocky/tokei
synced 2024-08-28 03:39:37 +00:00
Merge branch 'tune-up'
This commit is contained in:
commit
2b75021a8d
17
.travis.yml
17
.travis.yml
|
@ -1,18 +1,19 @@
|
|||
os:
|
||||
- linux
|
||||
- osx
|
||||
- linux
|
||||
- osx
|
||||
env:
|
||||
- RUST_TEST_THREADS=1
|
||||
language: rust
|
||||
rust:
|
||||
- stable
|
||||
- beta
|
||||
- nightly
|
||||
- stable
|
||||
- beta
|
||||
- nightly
|
||||
script:
|
||||
- cargo build --verbose --release
|
||||
- cargo build --verbose --release --features all
|
||||
- cargo test --verbose --release
|
||||
- cargo test --verbose --release --features all
|
||||
matrix:
|
||||
allow_failure:
|
||||
- rust: nightly
|
||||
allow_failure:
|
||||
- rust: nightly
|
||||
sudo: false
|
||||
|
||||
|
|
|
@ -8,7 +8,7 @@ license = "MIT/Apache-2.0"
|
|||
name = "tokei"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/Aaronepower/tokei.git"
|
||||
version = "4.1.1"
|
||||
version = "4.2.0"
|
||||
|
||||
[[bin]]
|
||||
doc = false
|
||||
|
|
98
cli.yml
98
cli.yml
|
@ -1,49 +1,49 @@
|
|||
# Copyright (c) 2015 Aaron Power
|
||||
# Use of this source code is governed by the APACHE2.0/MIT license that can be
|
||||
# found in the LICENCE-{APACHE/MIT} file.
|
||||
about: Count Code, Quickly.
|
||||
author: Aaron P. <theaaronepower@gmail.com>
|
||||
bin_name: Tokei
|
||||
name: Tokei
|
||||
version: 4.1.1
|
||||
args:
|
||||
- exclude:
|
||||
help: Ignore all files & directories containing the word.
|
||||
long: exclude
|
||||
short: e
|
||||
takes_value: true
|
||||
- files:
|
||||
help: Will print out statistics on individual files.
|
||||
long: files
|
||||
short: f
|
||||
takes_value: false
|
||||
- file_input:
|
||||
help: "Gives statistics from a previous tokei run. Can be given a file path, or \"stdin\" to read from stdin."
|
||||
long: input
|
||||
short: i
|
||||
takes_value: true
|
||||
- input:
|
||||
conflicts_with:
|
||||
- languages
|
||||
help: The input file(s)/directory(ies)
|
||||
index: 1
|
||||
multiple: true
|
||||
required: true
|
||||
- languages:
|
||||
conflicts_with:
|
||||
- input
|
||||
help: Prints out supported languages and their extensions.
|
||||
long: languages
|
||||
short: l
|
||||
- output:
|
||||
help: Outputs Tokei in a specific format.
|
||||
long: output
|
||||
possible_values: [cbor, json, toml, yaml]
|
||||
short: o
|
||||
takes_value: true
|
||||
- sort:
|
||||
help: Will sort based on column
|
||||
long: sort
|
||||
possible_values: [files, lines, blanks, code, comments]
|
||||
short: s
|
||||
takes_value: true
|
||||
# Copyright (c) 2015 Aaron Power
|
||||
# Use of this source code is governed by the APACHE2.0/MIT license that can be
|
||||
# found in the LICENCE-{APACHE/MIT} file.
|
||||
about: Count Code, Quickly.
|
||||
author: Aaron P. <theaaronepower@gmail.com>
|
||||
bin_name: Tokei
|
||||
name: Tokei
|
||||
version: 4.2.0
|
||||
args:
|
||||
- exclude:
|
||||
help: Ignore all files & directories containing the word.
|
||||
long: exclude
|
||||
short: e
|
||||
takes_value: true
|
||||
- files:
|
||||
help: Will print out statistics on individual files.
|
||||
long: files
|
||||
short: f
|
||||
takes_value: false
|
||||
- file_input:
|
||||
help: "Gives statistics from a previous tokei run. Can be given a file path, or \"stdin\" to read from stdin."
|
||||
long: input
|
||||
short: i
|
||||
takes_value: true
|
||||
- input:
|
||||
conflicts_with:
|
||||
- languages
|
||||
help: The input file(s)/directory(ies)
|
||||
index: 1
|
||||
multiple: true
|
||||
required: true
|
||||
- languages:
|
||||
conflicts_with:
|
||||
- input
|
||||
help: Prints out supported languages and their extensions.
|
||||
long: languages
|
||||
short: l
|
||||
- output:
|
||||
help: Outputs Tokei in a specific format.
|
||||
long: output
|
||||
possible_values: [cbor, json, toml, yaml]
|
||||
short: o
|
||||
takes_value: true
|
||||
- sort:
|
||||
help: Will sort based on column
|
||||
long: sort
|
||||
possible_values: [files, lines, blanks, code, comments]
|
||||
short: s
|
||||
takes_value: true
|
||||
|
|
|
@ -31,6 +31,12 @@ pub struct Language {
|
|||
/// Whether the language supports nested multi line comments or not.
|
||||
#[serde(skip_deserializing, skip_serializing)]
|
||||
pub nested: bool,
|
||||
/// A list of specific nested comments if this is empty all `multi_line` comments count.
|
||||
#[serde(skip_deserializing, skip_serializing)]
|
||||
pub nested_comments: Vec<(&'static str, &'static str)>,
|
||||
/// A list of quotes by default it is `""`.
|
||||
#[serde(skip_deserializing, skip_serializing)]
|
||||
pub quotes: Vec<(&'static str, &'static str)>,
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "io"))]
|
||||
|
@ -54,6 +60,10 @@ pub struct Language {
|
|||
pub multi_line: Vec<(&'static str, &'static str)>,
|
||||
/// Whether the language supports nested multi line comments or not.
|
||||
pub nested: bool,
|
||||
/// A list of specific nested comments if this is empty all `multi_line` comments count.
|
||||
pub nested_comments: Vec<(&'static str, &'static str)>,
|
||||
/// A list of quotes by default it is `""`.
|
||||
pub quotes: Vec<(&'static str, &'static str)>,
|
||||
}
|
||||
|
||||
|
||||
|
@ -101,6 +111,7 @@ impl Language {
|
|||
Language {
|
||||
line_comment: vec!["//"],
|
||||
multi_line: vec![("/*", "*/")],
|
||||
quotes: vec![("\"", "\"")],
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
@ -116,7 +127,11 @@ impl Language {
|
|||
/// assert_eq!(ocaml.multi_line, coq.multi_line);
|
||||
/// ```
|
||||
pub fn new_func() -> Self {
|
||||
Language { multi_line: vec![("(*", "*)")], ..Self::default() }
|
||||
Language {
|
||||
multi_line: vec![("(*", "*)")],
|
||||
quotes: vec![("\"", "\"")],
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convience constructor for creating a language that has the same commenting syntax as HTML like languages.
|
||||
|
@ -130,7 +145,11 @@ impl Language {
|
|||
/// assert_eq!(xml.multi_line, html.multi_line);
|
||||
/// ```
|
||||
pub fn new_html() -> Self {
|
||||
Language { multi_line: vec![("<!--", "-->")], ..Self::default() }
|
||||
Language {
|
||||
multi_line: vec![("<!--", "-->")],
|
||||
quotes: vec![("\"", "\"")],
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convience constructor for creating a language that has the same commenting syntax as Bash.
|
||||
|
@ -154,7 +173,11 @@ impl Language {
|
|||
/// let mustache = Language::new_multi(vec![("{{!", "}}")]);
|
||||
/// ```
|
||||
pub fn new_multi(multi_line: Vec<(&'static str, &'static str)>) -> Self {
|
||||
Language { multi_line: multi_line, ..Self::default() }
|
||||
Language {
|
||||
multi_line: multi_line,
|
||||
quotes: vec![("\"", "\"")],
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convience constructor for creating a language that has the same commenting syntax as Prolog.
|
||||
|
@ -171,6 +194,7 @@ impl Language {
|
|||
Language {
|
||||
line_comment: vec!["%"],
|
||||
multi_line: vec![("/*", "*/")],
|
||||
quotes: vec![("\"", "\"")],
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
@ -182,7 +206,11 @@ impl Language {
|
|||
/// let haskell = Language::new_single(vec!["--"]);
|
||||
/// ```
|
||||
pub fn new_single(line_comment: Vec<&'static str>) -> Self {
|
||||
Language { line_comment: line_comment, ..Self::default() }
|
||||
Language {
|
||||
line_comment: line_comment,
|
||||
quotes: vec![("\"", "\"")],
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the language is empty. Empty meaning it doesn't have any statistics.
|
||||
|
@ -221,7 +249,40 @@ impl Language {
|
|||
self
|
||||
}
|
||||
|
||||
/// Sorts each of the `Stats` structs contained in the language based on what category is provided
|
||||
/// Specify if the the language supports nested multi line comments.
|
||||
/// And which are nested. If this is specified there is no need to
|
||||
/// call the `nested` function.
|
||||
///
|
||||
/// ```
|
||||
/// # use tokei::*;
|
||||
/// let mut d = Language::new(vec!["//"], vec![("/*", "*/")])
|
||||
/// .nested_comments(vec![("/+", "+/")]);
|
||||
/// assert!(d.nested);
|
||||
/// assert_eq!(d.nested_comments, vec![("/+", "+/")]);
|
||||
/// ```
|
||||
pub fn nested_comments(mut self, nested_comments: Vec<(&'static str, &'static str)>) -> Self {
|
||||
self.nested = true;
|
||||
self.nested_comments = nested_comments;
|
||||
self
|
||||
}
|
||||
|
||||
/// Specifies if the language has a quotes to define a string where
|
||||
/// the commenting syntax would be ignored. By default it is only
|
||||
/// `""` quotes that are ignored.
|
||||
///
|
||||
/// ```
|
||||
/// # use tokei::*;
|
||||
/// let mut javascript = Language::new(vec!["//"], vec![("/*", "*/")])
|
||||
/// .set_quotes(vec![("\"", "\""), ("'", "'")]);
|
||||
/// assert!(!javascript.quotes.is_empty());
|
||||
/// ```
|
||||
pub fn set_quotes(mut self, quotes: Vec<(&'static str, &'static str)>) -> Self {
|
||||
self.quotes = quotes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sorts each of the `Stats` structs contained in the language based
|
||||
/// on what category is provided
|
||||
/// panic!'s if given the wrong category.
|
||||
///
|
||||
/// ```
|
||||
|
|
|
@ -7,7 +7,7 @@ use std::borrow::Cow;
|
|||
use std::fmt;
|
||||
use std::path::Path;
|
||||
|
||||
use utils::*;
|
||||
use utils::fs;
|
||||
use self::LanguageType::*;
|
||||
|
||||
#[cfg_attr(feature = "io", derive(Deserialize, Serialize))]
|
||||
|
@ -269,7 +269,7 @@ impl LanguageType {
|
|||
/// assert_eq!(rust, Some(LanguageType::Rust));
|
||||
/// ```
|
||||
pub fn from_extension<P: AsRef<Path>>(entry: P) -> Option<Self> {
|
||||
if let Some(extension) = get_extension(entry) {
|
||||
if let Some(extension) = fs::get_extension(entry) {
|
||||
match &*extension {
|
||||
"as" => Some(ActionScript),
|
||||
"ada" | "adb" | "ads" => Some(Ada),
|
||||
|
@ -289,7 +289,8 @@ impl LanguageType {
|
|||
"dts" | "dtsi" => Some(DeviceTree),
|
||||
"el" | "lisp" | "lsp" => Some(Lisp),
|
||||
"erl" | "hrl" => Some(Erlang),
|
||||
"4th" | "forth" | "fr" | "frt" | "fth" | "f83" | "fb" | "fpm" | "e4" | "rx" | "ft" => Some(Forth),
|
||||
"4th" | "forth" | "fr" | "frt" | "fth" | "f83" | "fb" | "fpm" | "e4" | "rx" |
|
||||
"ft" => Some(Forth),
|
||||
"f" | "for" | "ftn" | "f77" | "pfo" => Some(FortranLegacy),
|
||||
"f03" | "f08" | "f90" | "f95" => Some(FortranModern),
|
||||
"go" => Some(Go),
|
||||
|
@ -358,6 +359,12 @@ impl LanguageType {
|
|||
|
||||
impl From<String> for LanguageType {
|
||||
fn from(from: String) -> Self {
|
||||
LanguageType::from(&*from)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<&'a str> for LanguageType {
|
||||
fn from(from: &str) -> Self {
|
||||
match &*from {
|
||||
"ActionScript" => ActionScript,
|
||||
"Ada" => Ada,
|
||||
|
|
|
@ -19,7 +19,7 @@ use serde_yaml;
|
|||
use toml;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use utils::*;
|
||||
use utils::{fs, multi_line};
|
||||
use super::{Language, LanguageType};
|
||||
use super::LanguageType::*;
|
||||
use stats::Stats;
|
||||
|
@ -31,6 +31,87 @@ const TOML_ERROR: &'static str = "Tokei was not compiled with the `toml-io` flag
|
|||
#[cfg(not(feature = "yaml"))]
|
||||
const YAML_ERROR: &'static str = "Tokei was not compiled with the `yaml` flag.";
|
||||
|
||||
fn count_files(language_tuple: &mut (&LanguageType, &mut Language)) {
|
||||
|
||||
let &mut (name, ref mut language) = language_tuple;
|
||||
|
||||
if language.files.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let is_fortran = name == &FortranModern || name == &FortranLegacy;
|
||||
|
||||
let files: Vec<_> = language.files.drain(..).collect();
|
||||
let mut contents = String::new();
|
||||
let mut stack = vec![];
|
||||
let mut quote = None;
|
||||
|
||||
for file in files {
|
||||
let mut stats = Stats::new(opt_or_cont!(file.to_str()));
|
||||
stack.clear();
|
||||
contents.clear();
|
||||
|
||||
rs_or_cont!(rs_or_cont!(File::open(file)).read_to_string(&mut contents));
|
||||
|
||||
let lines = contents.lines();
|
||||
|
||||
if language.is_blank() {
|
||||
stats.code += lines.count();
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
'line: for line in lines {
|
||||
stats.lines += 1;
|
||||
// FORTRAN has a rule where it only counts as a comment if it's the first
|
||||
// character in the column, so removing starting whitespace could cause a
|
||||
// miscount.
|
||||
let line = if is_fortran {
|
||||
line
|
||||
} else {
|
||||
line.trim()
|
||||
};
|
||||
|
||||
if line.trim().is_empty() {
|
||||
stats.blanks += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
for single in &language.line_comment {
|
||||
if line.starts_with(single) {
|
||||
stats.comments += 1;
|
||||
continue 'line;
|
||||
}
|
||||
}
|
||||
|
||||
let mut started_with = false;
|
||||
if quote.is_none() {
|
||||
let chain = language.multi_line.iter().chain(language.nested_comments.iter());
|
||||
|
||||
for &(start, _) in chain {
|
||||
if line.starts_with(start) {
|
||||
started_with = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
multi_line::handle_multi_line(line, &language, &mut stack, &mut quote);
|
||||
|
||||
if !stack.is_empty() {
|
||||
if started_with {
|
||||
stats.code += 1;
|
||||
} else {
|
||||
stats.comments += 1;
|
||||
}
|
||||
} else {
|
||||
stats.code += 1;
|
||||
}
|
||||
}
|
||||
|
||||
**language += stats;
|
||||
}
|
||||
}
|
||||
|
||||
/// A collection of existing languages([_List of Languages_](https://github.com/Aaronepower/tokei#supported-languages))
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Languages {
|
||||
|
@ -168,100 +249,11 @@ impl Languages {
|
|||
where I: Into<Cow<'a, [&'a str]>>
|
||||
{
|
||||
|
||||
get_all_files(paths.into(), ignored.into(), &mut self.inner);
|
||||
fs::get_all_files(paths.into(), ignored.into(), &mut self.inner);
|
||||
|
||||
let mut language_iter: Vec<_> = self.inner.iter_mut().collect();
|
||||
|
||||
language_iter.par_iter_mut().for_each(|&mut (name, ref mut language)| {
|
||||
if language.files.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let is_fortran = name == &FortranModern || name == &FortranLegacy;
|
||||
|
||||
let files: Vec<_> = language.files.drain(..).collect();
|
||||
for file in files {
|
||||
let mut is_in_comments = false;
|
||||
let mut previous_comment_start = "";
|
||||
let mut comment_depth: usize = 0;
|
||||
let mut stats = Stats::new(opt_or_cont!(file.to_str()));
|
||||
|
||||
let contents = {
|
||||
let mut contents = String::new();
|
||||
let _ = rs_or_cont!(rs_or_cont!(File::open(file))
|
||||
.read_to_string(&mut contents));
|
||||
contents
|
||||
};
|
||||
|
||||
let lines = contents.lines();
|
||||
|
||||
if language.is_blank() {
|
||||
stats.code += lines.count();
|
||||
continue;
|
||||
}
|
||||
|
||||
'line: for line in lines {
|
||||
stats.lines += 1;
|
||||
|
||||
// FORTRAN has a rule where it only counts as a comment if it's the first
|
||||
// character in the column, so removing starting whitespace could cause a
|
||||
// miscount.
|
||||
let line = if is_fortran {
|
||||
line
|
||||
} else {
|
||||
line.trim()
|
||||
};
|
||||
|
||||
if line.trim().is_empty() {
|
||||
stats.blanks += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
for &(multi_line, multi_line_end) in &language.multi_line {
|
||||
if line.starts_with(multi_line) ||
|
||||
has_trailing_comments(line,
|
||||
multi_line,
|
||||
multi_line_end,
|
||||
language.nested) {
|
||||
previous_comment_start = multi_line;
|
||||
is_in_comments = true;
|
||||
if language.nested {
|
||||
comment_depth += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if is_in_comments {
|
||||
for &(multi_line, multi_line_end) in &language.multi_line {
|
||||
if multi_line == previous_comment_start &&
|
||||
line.contains(multi_line_end) {
|
||||
if language.nested {
|
||||
comment_depth -= 1;
|
||||
if comment_depth == 0 {
|
||||
is_in_comments = false;
|
||||
}
|
||||
} else {
|
||||
is_in_comments = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
stats.comments += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
for single in &language.line_comment {
|
||||
if line.starts_with(single) {
|
||||
stats.comments += 1;
|
||||
continue 'line;
|
||||
}
|
||||
}
|
||||
stats.code += 1;
|
||||
}
|
||||
|
||||
**language += stats;
|
||||
}
|
||||
});
|
||||
language_iter.par_iter_mut().for_each(count_files);
|
||||
}
|
||||
|
||||
/// Constructs a new, blank `Languages`.
|
||||
|
@ -282,7 +274,8 @@ impl Languages {
|
|||
C => Language::new_c(),
|
||||
CHeader => Language::new_c(),
|
||||
Clojure => Language::new_single(vec![";","#"]),
|
||||
CoffeeScript => Language::new(vec!["#"], vec![("###", "###")]),
|
||||
CoffeeScript => Language::new(vec!["#"], vec![("###", "###")])
|
||||
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||
ColdFusion => Language::new_multi(vec![("<!---", "--->")]),
|
||||
ColdFusionScript => Language::new_c(),
|
||||
Coq => Language::new_func(),
|
||||
|
@ -290,8 +283,9 @@ impl Languages {
|
|||
CppHeader => Language::new_c(),
|
||||
CSharp => Language::new_c(),
|
||||
CShell => Language::new_hash(),
|
||||
Css => Language::new_c(),
|
||||
D => Language::new_c(),
|
||||
Css => Language::new_c()
|
||||
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||
D => Language::new(vec!["//"], vec![("/*", "*/")]).nested_comments(vec![("/+", "+/")]),
|
||||
Dart => Language::new_c(),
|
||||
DeviceTree => Language::new_c(),
|
||||
Erlang => Language::new_single(vec!["%"]),
|
||||
|
@ -299,9 +293,11 @@ impl Languages {
|
|||
FortranLegacy => Language::new_single(vec!["c","C","!","*"]),
|
||||
FortranModern => Language::new_single(vec!["!"]),
|
||||
Go => Language::new_c(),
|
||||
Handlebars => Language::new_multi(vec![("<!--", "-->"), ("{{!", "}}")]),
|
||||
Handlebars => Language::new_multi(vec![("<!--", "-->"), ("{{!", "}}")])
|
||||
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||
Haskell => Language::new_single(vec!["--"]),
|
||||
Html => Language::new_html(),
|
||||
Html => Language::new_html()
|
||||
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||
Idris => Language::new(vec!["--"], vec![("{-", "-}")]),
|
||||
Isabelle => Language::new(
|
||||
vec!["--"],
|
||||
|
@ -313,9 +309,11 @@ impl Languages {
|
|||
),
|
||||
Jai => Language::new_c(),
|
||||
Java => Language::new_c(),
|
||||
JavaScript => Language::new_c(),
|
||||
JavaScript => Language::new_c()
|
||||
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||
Json => Language::new_blank(),
|
||||
Jsx => Language::new_c(),
|
||||
Jsx => Language::new_c()
|
||||
.set_quotes(vec![("\"", "\""), ("'", "'")]),
|
||||
Julia => Language::new(vec!["#"], vec![("#=", "=#")]),
|
||||
Kotlin => Language::new_c(),
|
||||
Less => Language::new_c(),
|
||||
|
@ -577,3 +575,108 @@ impl DerefMut for Languages {
|
|||
&mut self.inner
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod accuracy_tests {
|
||||
use super::*;
|
||||
use std::io;
|
||||
use language::LanguageType;
|
||||
|
||||
fn write(contents: &'static str, file_name: &str) -> io::Result<()> {
|
||||
use std::io::prelude::*;
|
||||
use std::fs::{File, create_dir};
|
||||
|
||||
let _ = create_dir("./_temp/");
|
||||
let mut f = try!(File::create(file_name));
|
||||
|
||||
try!(f.write_all(&contents.as_bytes()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cleanup() -> io::Result<()> {
|
||||
try!(::std::fs::remove_dir_all("./_temp/"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn test_accuracy(file_name: &'static str, expected: u32, contents: &'static str) {
|
||||
let file_name = format!("./_temp/{}", file_name);
|
||||
write(contents, &*file_name).unwrap();
|
||||
let mut l = Languages::new();
|
||||
let l_type = LanguageType::from_extension(file_name).expect("Can't find language type");
|
||||
|
||||
l.get_statistics(vec!["./_temp/"], vec![]);
|
||||
|
||||
let _ = cleanup();
|
||||
let language = l.get_mut(&l_type).expect("Couldn't find language");
|
||||
|
||||
assert_eq!(expected as usize, language.code);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inside_quotes() {
|
||||
test_accuracy("inside_quotes.rs",
|
||||
8,
|
||||
r#"fn main() {
|
||||
let start = "/*";
|
||||
loop {
|
||||
if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */
|
||||
break;
|
||||
}
|
||||
}
|
||||
}"#)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shouldnt_panic() {
|
||||
test_accuracy("shouldnt_panic.rs",
|
||||
9,
|
||||
r#"fn foo() {
|
||||
let this_ends = "a \"test/*.";
|
||||
call1();
|
||||
call2();
|
||||
let this_does_not = /* a /* nested */ comment " */
|
||||
"*/another /*test
|
||||
call3();
|
||||
*/";
|
||||
}"#)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_quotes_no_comment() {
|
||||
test_accuracy("all_quotes_no_comment.rs",
|
||||
10,
|
||||
r#"fn foobar() {
|
||||
let does_not_start = // "
|
||||
"until here,
|
||||
test/*
|
||||
test"; // a quote: "
|
||||
let also_doesnt_start = /* " */
|
||||
"until here,
|
||||
test,*/
|
||||
test"; // another quote: "
|
||||
}"#)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commenting_on_comments() {
|
||||
test_accuracy("commenting_on_comments.rs",
|
||||
5,
|
||||
r#"fn foo() {
|
||||
let a = 4; // /*
|
||||
let b = 5;
|
||||
let c = 6; // */
|
||||
}"#)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nesting_with_nesting_comments() {
|
||||
test_accuracy("nesting_with_nesting_comments.d",
|
||||
5,
|
||||
r#"void main() {
|
||||
auto x = 5; /+ a /+ nested +/ comment /* +/
|
||||
writefln("hello");
|
||||
auto y = 4; // */
|
||||
}"#)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,46 +14,6 @@ use walkdir::{WalkDir, WalkDirIterator};
|
|||
use language::{Language, LanguageType};
|
||||
use language::LanguageType::*;
|
||||
|
||||
/// This is used to catch lines like "let x = 5; /* Comment */"
|
||||
pub fn has_trailing_comments(line: &str,
|
||||
comment: &'static str,
|
||||
comment_end: &'static str,
|
||||
nested: bool)
|
||||
-> bool {
|
||||
let mut is_in_comments = 0u64;
|
||||
|
||||
let start = match line.find(comment) {
|
||||
Some(start) => start,
|
||||
None => return false,
|
||||
};
|
||||
|
||||
let end = match line.rfind(comment_end) {
|
||||
Some(end) => end,
|
||||
None => return true,
|
||||
};
|
||||
|
||||
let mut chars = line[start..end + comment_end.len()].chars();
|
||||
loop {
|
||||
let window = chars.as_str();
|
||||
|
||||
if window.starts_with(comment) {
|
||||
if nested {
|
||||
is_in_comments += 1;
|
||||
} else {
|
||||
is_in_comments = 1;
|
||||
}
|
||||
} else if window.starts_with(comment_end) {
|
||||
is_in_comments = is_in_comments.saturating_sub(1);
|
||||
}
|
||||
|
||||
if chars.next().is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
is_in_comments != 0
|
||||
}
|
||||
|
||||
pub fn get_all_files<'a>(paths: Cow<'a, [&'a str]>,
|
||||
ignored_directories: Cow<'a, [&'a str]>,
|
||||
languages: &mut BTreeMap<LanguageType, Language>) {
|
||||
|
@ -154,28 +114,3 @@ pub fn get_filetype_from_shebang<P: AsRef<Path>>(file: P) -> Option<&'static str
|
|||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn both_comments_in_line() {
|
||||
assert!(!has_trailing_comments("Hello /* /* */ World", "/*", "*/", false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_start_in_line() {
|
||||
assert!(has_trailing_comments("Hello /* World", "/*", "*/", false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_comments_in_line_nested() {
|
||||
assert!(has_trailing_comments("Hello (* (* *) World", "(*", "*)", true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_start_in_line_nested() {
|
||||
assert!(has_trailing_comments("Hello (* World", "(*", "*)", true));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,26 +2,26 @@
|
|||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
macro_rules! opt_or_cont {
|
||||
($option:expr) => {
|
||||
match $option {
|
||||
Some(result) => result,
|
||||
None => continue,
|
||||
}
|
||||
}
|
||||
macro_rules! opt_or_cont {
|
||||
($option:expr) => {
|
||||
match $option {
|
||||
Some(result) => result,
|
||||
None => continue,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! rs_or_cont {
|
||||
($result:expr) => {
|
||||
match $result {
|
||||
Ok(result) => result,
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
macro_rules! rs_or_cont {
|
||||
($result:expr) => {
|
||||
match $result {
|
||||
Ok(result) => result,
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
macro_rules! debug {
|
||||
($fmt:expr) => (if cfg!(debug_assertions) {println!($fmt)});
|
||||
($fmt:expr, $($arg:tt)*) => (if cfg!(debug_assertions) {println!($fmt, $($arg)*)});
|
||||
macro_rules! debug {
|
||||
($fmt:expr) => (if cfg!(debug_assertions) {println!($fmt)});
|
||||
($fmt:expr, $($arg:tt)*) => (if cfg!(debug_assertions) {println!($fmt, $($arg)*)});
|
||||
}
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#[macro_use]
|
||||
mod macros;
|
||||
pub mod fs;
|
||||
|
||||
pub use self::fs::*;
|
||||
pub mod multi_line;
|
||||
|
|
156
src/lib/utils/multi_line.rs
Normal file
156
src/lib/utils/multi_line.rs
Normal file
|
@ -0,0 +1,156 @@
|
|||
use language::Language;
|
||||
|
||||
/// This is used to catch lines like "let x = 5; /* Comment */"
|
||||
pub fn handle_multi_line(line: &str,
|
||||
language: &Language,
|
||||
stack: &mut Vec<&'static str>,
|
||||
quote: &mut Option<&'static str>) {
|
||||
let mut chars = line.chars();
|
||||
let mut cont = false;
|
||||
let nested_is_empty = language.nested_comments.is_empty();
|
||||
|
||||
'window: loop {
|
||||
let window = chars.as_str();
|
||||
if window.is_empty() {
|
||||
break;
|
||||
}
|
||||
chars.next();
|
||||
|
||||
// Prevents counting overlaps like /*/*
|
||||
if cont {
|
||||
cont = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut end = false;
|
||||
|
||||
if let &mut Some(quote_str) = quote {
|
||||
if window.starts_with("\\") {
|
||||
cont = true;
|
||||
continue;
|
||||
} else if window.starts_with(quote_str) {
|
||||
end = true;
|
||||
}
|
||||
}
|
||||
|
||||
if end {
|
||||
if let &mut Some(quote_str) = quote {
|
||||
*quote = None;
|
||||
|
||||
if quote_str.chars().count() == 1 {
|
||||
cont = true
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if quote.is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut pop = false;
|
||||
if let Some(last) = stack.last() {
|
||||
if window.starts_with(last) {
|
||||
pop = true;
|
||||
}
|
||||
}
|
||||
|
||||
if pop {
|
||||
stack.pop();
|
||||
cont = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
if stack.is_empty() {
|
||||
for &(start, end) in &language.quotes {
|
||||
if window.starts_with(start) {
|
||||
*quote = Some(end);
|
||||
cont = true;
|
||||
continue 'window;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for comment in &language.line_comment {
|
||||
if window.starts_with(comment) {
|
||||
break 'window;
|
||||
}
|
||||
}
|
||||
|
||||
for &(start, end) in &language.nested_comments {
|
||||
if window.starts_with(start) {
|
||||
stack.push(end);
|
||||
cont = true;
|
||||
continue 'window;
|
||||
}
|
||||
}
|
||||
|
||||
for &(start, end) in &language.multi_line {
|
||||
if window.starts_with(start) {
|
||||
if language.nested && nested_is_empty {
|
||||
stack.push(end);
|
||||
} else if stack.len() == 0 {
|
||||
stack.push(end);
|
||||
}
|
||||
cont = true;
|
||||
continue 'window;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use language::Language;
|
||||
|
||||
#[test]
|
||||
fn both_comments_in_line() {
|
||||
let mut stack = vec![];
|
||||
let mut quote = None;
|
||||
let language = Language::new_c();
|
||||
handle_multi_line("Hello /* /* */ World", &language, &mut stack, &mut quote);
|
||||
assert_eq!(stack.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_hidden_in_single() {
|
||||
let mut stack = vec![];
|
||||
let mut quote = None;
|
||||
let language = Language::new_c();
|
||||
handle_multi_line("Hello World // /*", &language, &mut stack, &mut quote);
|
||||
assert_eq!(stack.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_start() {
|
||||
let mut stack = vec![];
|
||||
let mut quote = None;
|
||||
let language = Language::new_c();
|
||||
handle_multi_line("/*Hello World", &language, &mut stack, &mut quote);
|
||||
assert_eq!(stack.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_comments_in_line_nested() {
|
||||
let mut stack = vec![];
|
||||
let mut quote = None;
|
||||
let language = Language::new_func().nested();
|
||||
handle_multi_line("Hello (* (* *) World", &language, &mut stack, &mut quote);
|
||||
assert_eq!(stack.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_of_uneven_length() {
|
||||
let mut stack = vec![];
|
||||
let mut quote = None;
|
||||
let language = Language::new(vec![], vec![("\\<open>", "\\<close>")]).nested();
|
||||
handle_multi_line("Hello \\<open> \\<open> \\<close> World",
|
||||
&language,
|
||||
&mut stack,
|
||||
&mut quote);
|
||||
assert_eq!(stack.len(), 1);
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue