forgot files

This commit is contained in:
Aaronepower 2016-06-07 12:09:38 +01:00
parent 1e6a4c1015
commit 4f6cf406e7
14 changed files with 2743 additions and 0 deletions

17
src/lib/build.rs Normal file
View file

@ -0,0 +1,17 @@
extern crate syntex;
extern crate serde_codegen;
use std::env;
use std::path::Path;
pub fn main() {
let out_dir = env::var_os("OUT_DIR").unwrap();
let src = Path::new("src/lib/lib.rs.in");
let dst = Path::new(&out_dir).join("lib.rs");
let mut registry = syntex::Registry::new();
serde_codegen::register(&mut registry);
registry.expand("", &src, &dst).unwrap();
}

View file

@ -0,0 +1,262 @@
use std::path::PathBuf;
use std::ops::AddAssign;
use utils::*;
use stats::Stats;
/// Struct representing a single Language.
#[derive(Clone, Debug, Default, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
pub struct Language {
/// Number of blank lines.
pub blanks: usize,
/// Number of lines of code.
pub code: usize,
/// Number of comments(both single, and multi-line)
pub comments: usize,
/// A collection of files to be analysed.
#[serde(skip_deserializing, skip_serializing)]
pub files: Vec<PathBuf>,
/// A collection of statistics based on the files provide from `files`
pub stats: Vec<Stats>,
/// Number of total lines.
pub lines: usize,
/// A collection of single line comments in the language. ie. `//` in Rust.
#[serde(skip_deserializing, skip_serializing)]
pub line_comment: Vec<&'static str>,
/// A collection of tuples representing the start and end of multi line comments. ie. `/* comment */` in Rust.
#[serde(skip_deserializing, skip_serializing)]
pub multi_line: Vec<(&'static str, &'static str)>,
/// Whether the language supports nested multi line comments or not.
#[serde(skip_deserializing, skip_serializing)]
pub nested: bool,
/// The total number of files from `stats`.
pub total_files: usize,
}
impl Language {
/// Constructs a new empty Language with the comments provided.
///
/// ```
/// let mut rust = Language::new(vec!["//"], vec![("/*", "*/")]);
/// ```
pub fn new(line_comment: Vec<&'static str>,
multi_line: Vec<(&'static str, &'static str)>)
-> Self {
Language {
line_comment: line_comment,
multi_line: multi_line,
..Self::default()
}
}
/// Convience constructor for creating a language that has no commenting syntax.
///
/// ```
/// let json = Language::new_blank();
///
/// assert_eq!(json.line_comment, vec![]);
/// ```
pub fn new_blank() -> Self {
Self::default()
}
/// Convience constructor for creating a language that has the same commenting syntax as C like languages.
///
/// ```
/// let rust = Language::new(vec!["//"], vec![("/*", "*/")]);
/// let c = Language::new_c();
///
/// assert_eq!(rust.line_comment, c.line_comment);
/// assert_eq!(rust.multi_line, c.multi_line);
/// ```
pub fn new_c() -> Self {
Language {
line_comment: vec!["//"],
multi_line: vec![("/*", "*/")],
..Self::default()
}
}
/// Convience constructor for creating a language that has the same commenting syntax as ML like languages.
///
/// ```
/// let ocaml = Language::new_multi(vec![("(*", "*)")]);
/// let coq = Language::new_func();
///
/// assert_eq!(ocaml.line_comment, coq.line_comment);
/// assert_eq!(ocaml.multi_line, coq.multi_line);
/// ```
pub fn new_func() -> Self {
Language { multi_line: vec![("(*", "*)")], ..Self::default() }
}
/// Convience constructor for creating a language that has the same commenting syntax as HTML like languages.
///
/// ```
/// let xml = Language::new_multi(vec![("<!--", "-->")]);
/// let html = Language::new_html();
///
/// assert_eq!(xml.line_comment, html.line_comment);
/// assert_eq!(xml.multi_line, html.multi_line);
/// ```
pub fn new_html() -> Self {
Language { multi_line: vec![("<!--", "-->")], ..Self::default() }
}
/// Convience constructor for creating a language that has the same commenting syntax as Bash.
///
/// ```
/// let bash = Language::new_single(vec!["#"]);
/// let yaml = Language::new_hash();
///
/// assert_eq!(bash.line_comment, yaml.line_comment);
/// assert_eq!(bash.multi_line, yaml.multi_line);
/// ```
pub fn new_hash() -> Self {
Self::new_single(vec!["#"])
}
/// Convience constructor for creating a language that only has multi line comments.
///
/// ```
/// let mustache = Language::new_multi(vec![("{{!", "}}")]);
/// ```
pub fn new_multi(multi_line: Vec<(&'static str, &'static str)>) -> Self {
Language { multi_line: multi_line, ..Self::default() }
}
/// Convience constructor for creating a language that has the same commenting syntax as Prolog.
///
/// ```
/// let prolog = Language::new(vec!["%"], vec![("/*", "*/")]);
/// let oz = Language::new_pro();
///
/// assert_eq!(prolog.line_comment, oz.line_comment);
/// assert_eq!(prolog.multi_line, oz.multi_line);
/// ```
pub fn new_pro() -> Self {
Language {
line_comment: vec!["%"],
multi_line: vec![("/*", "*/")],
..Self::default()
}
}
/// Convience constructor for creating a language that only has single line comments.
///
/// ```
/// let haskell = Language::new_single(vec!["--"]);
/// ```
pub fn new_single(line_comment: Vec<&'static str>) -> Self {
Language { line_comment: line_comment, ..Self::default() }
}
/// Checks if the language is empty. Empty meaning it doesn't have any statistics.
///
/// ```
/// let rust = Language::new_c();
///
/// assert!(rust.is_empty());
/// ```
pub fn is_empty(&self) -> bool {
self.code == 0 && self.comments == 0 && self.blanks == 0 && self.lines == 0
}
/// Checks if the language doesn't contain any comments.
///
/// ```
/// let json = Language::new_blank();
///
/// assert!(json.is_blank());
/// ```
pub fn is_blank(&self) -> bool {
self.line_comment.is_empty() && self.multi_line.is_empty()
}
/// Specify if the the language supports nested multi line comments.
///
/// ```
/// let mut rust = Language::new(vec!["//"], vec![("/*", "*/")]).nested();
/// assert!(rust.nested);
/// ```
pub fn nested(mut self) -> Self {
self.nested = true;
self
}
/// Sorts each of the `Stats` structs contained in the language based on what category is provided
/// panic!'s if given the wrong category.
///
/// ```
/// let mut rust = Language::new_c();
/// let foo_stats = Stats::new();
/// let bar_stats = Stats::new();
///
/// foo_stats.code += 20;
/// bar_stats.code += 10;
///
/// rust.stats.push(bar_stats.clone());
/// rust.stats.push(foo_stats.clone());
///
/// assert_eq!(rust.stats, vec![bar_stats.clone(), foo_stats.clone()]);
///
/// rust.sort_by(CODE);
///
/// assert_eq!(rust.stats, vec![foo_stats, bar_stats]);
///
/// ```
pub fn sort_by(&mut self, category: &str) {
match category {
BLANKS => self.stats.sort_by(|a, b| b.blanks.cmp(&a.blanks)),
COMMENTS => self.stats.sort_by(|a, b| b.comments.cmp(&a.comments)),
CODE => self.stats.sort_by(|a, b| b.code.cmp(&a.code)),
FILES => self.stats.sort_by(|a, b| a.name.cmp(&b.name)),
LINES => self.stats.sort_by(|a, b| b.lines.cmp(&a.lines)),
_ => unreachable!(),
}
}
}
impl AddAssign for Language {
fn add_assign(&mut self, rhs: Self) {
self.total_files += rhs.total_files;
self.lines += rhs.lines;
self.comments += rhs.comments;
self.blanks += rhs.blanks;
self.code += rhs.code;
self.stats.extend_from_slice(&*rhs.stats);
}
}
impl<'a> AddAssign<&'a Language> for Language {
fn add_assign(&mut self, rhs: &'a Self) {
self.total_files += rhs.total_files;
self.lines += rhs.lines;
self.comments += rhs.comments;
self.blanks += rhs.blanks;
self.code += rhs.code;
self.stats.extend_from_slice(&*rhs.stats);
}
}
impl<'a> AddAssign<&'a mut Language> for Language {
fn add_assign(&mut self, rhs: &mut Self) {
self.total_files += rhs.total_files;
self.lines += rhs.lines;
self.comments += rhs.comments;
self.blanks += rhs.blanks;
self.code += rhs.code;
self.stats.extend_from_slice(&*rhs.stats);
}
}
impl AddAssign<Stats> for Language {
fn add_assign(&mut self, rhs: Stats) {
self.lines += rhs.lines;
self.code += rhs.code;
self.comments += rhs.comments;
self.blanks += rhs.blanks;
self.stats.push(rhs);
}
}

View file

@ -0,0 +1,521 @@
// Copyright (c) 2015 Aaron Power
// Use of this source code is governed by the APACHE2.0/MIT licence that can be
// found in the LICENCE-{APACHE/MIT} file.
use std::borrow::Cow;
use std::fmt;
use std::path::Path;
use utils::*;
use self::LanguageType::*;
serializable_enum! {
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub enum LanguageType {
/// ActionScript
ActionScript,
/// Assembly
Assembly,
/// Bash
Bash,
/// Batch
Batch,
/// C
C,
/// CHeader
CHeader,
/// Clojure
Clojure,
/// CoffeeScript
CoffeeScript,
/// ColdFusion
ColdFusion,
/// ColdFusionScript
ColdFusionScript,
/// Coq
Coq,
/// Cpp
Cpp,
/// CppHeader
CppHeader,
/// CSharp
CSharp,
/// CShell
CShell,
/// Css
Css,
/// D
D,
/// Dart
Dart,
/// DeviceTree
DeviceTree,
/// Erlang
Erlang,
/// FortranLegacy
FortranLegacy,
/// FortranModern
FortranModern,
/// Go
Go,
/// Haskell
Haskell,
/// Html
Html,
/// Idris
Idris,
/// Isabelle
Isabelle,
/// Jai
Jai,
/// Java
Java,
/// JavaScript
JavaScript,
/// Julia
Julia,
/// Json
Json,
/// Jsx
Jsx,
/// Kotlin
Kotlin,
/// Less
Less,
/// LinkerScript
LinkerScript,
/// Lisp
Lisp,
/// Lua
Lua,
/// Makefile
Makefile,
/// Markdown
Markdown,
/// Mustache
Mustache,
/// Nim
Nim,
/// ObjectiveC
ObjectiveC,
/// ObjectiveCpp
ObjectiveCpp,
/// OCaml
OCaml,
/// Oz
Oz,
/// Pascal
Pascal,
/// Perl
Perl,
/// Polly
Polly,
/// Php
Php,
/// Protobuf
Protobuf,
/// Prolog
Prolog,
/// Python
Python,
/// Qcl
Qcl,
/// R
R,
/// Ruby
Ruby,
/// RubyHtml
RubyHtml,
/// Rust
Rust,
/// Sass
Sass,
/// Scala
Scala,
/// Sml
Sml,
/// Sql
Sql,
/// Swift
Swift,
/// Tex
Tex,
/// Text
Text,
/// Toml
Toml,
/// TypeScript
TypeScript,
/// VimScript
VimScript,
/// UnrealScript
UnrealScript,
/// Wolfram
Wolfram,
/// Xml
Xml,
/// Yaml
Yaml,
/// Zsh
Zsh,
/// __Total
__Total,
}
LanguageTypeVisitor
}
#[derive(Debug)]
pub enum Error {
Parse(String),
}
impl ::std::fmt::Display for Error {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
write!(f, "{:?}", self)
}
}
impl_as_ref_from_str! {
LanguageType {
ActionScript => "ActionScript",
Assembly => "Assembly",
Bash => "BASH",
Batch => "Batch",
C => "C",
CHeader => "C Header",
Clojure => "Clojure",
CoffeeScript => "CoffeeScript",
ColdFusion => "ColdFusion",
ColdFusionScript => "ColdFusion CFScript",
Coq => "Coq",
Cpp => "C++",
CppHeader => "C++ Header",
CSharp => "C#",
CShell => "C Shell",
Css => "CSS",
D => "D",
Dart => "Dart",
DeviceTree => "Device Tree",
Erlang => "Erlang",
FortranLegacy => "FORTRAN Legacy",
FortranModern => "FORTRAN Modern",
Go => "Go",
Haskell => "Haskell",
Html => "HTML",
Idris => "Idris",
Isabelle => "Isabelle",
Jai => "JAI",
Java => "Java",
JavaScript => "JavaScript",
Json => "JSON",
Jsx => "JSX",
Julia => "Julia",
Kotlin => "Kotlin",
Less => "LESS",
LinkerScript => "LD Script",
Lisp => "LISP",
Lua => "Lua",
Makefile => "Makefile",
Markdown => "Markdown",
Mustache => "Mustache",
Nim => "Nim",
ObjectiveC => "Objective C",
ObjectiveCpp => "Objective C++",
OCaml => "OCaml",
Oz => "Oz",
Pascal => "Pascal",
Perl => "Perl",
Polly => "Polly",
Php => "PHP",
Protobuf => "Protocol Buffers",
Prolog => "Prolog",
Python => "Python",
Qcl => "QCL",
R => "R",
Ruby => "Ruby",
RubyHtml => "Ruby HTML",
Rust => "Rust",
Sass => "Sass",
Scala => "Scala",
Sml => "Standard ML",
Sql => "SQL",
Swift => "Swift",
Tex => "TeX",
Text => "Plain Text",
Toml => "TOML",
TypeScript => "TypeScript",
UnrealScript => "Unreal Script",
VimScript => "Vim Script",
Wolfram => "Wolfram",
Xml => "XML",
Yaml => "YAML",
Zsh => "Zsh",
__Total => "Total",
}
Error::Parse
}
impl LanguageType {
pub fn name(&self) -> &'static str {
match *self {
ActionScript => "ActionScript",
Assembly => "Assembly",
Bash => "BASH",
Batch => "Batch",
C => "C",
CHeader => "C Header",
Clojure => "Clojure",
CoffeeScript => "CoffeeScript",
ColdFusion => "ColdFusion",
ColdFusionScript => "ColdFusion CFScript",
Coq => "Coq",
Cpp => "C++",
CppHeader => "C++ Header",
CSharp => "C#",
CShell => "C Shell",
Css => "CSS",
D => "D",
Dart => "Dart",
DeviceTree => "Device Tree",
Erlang => "Erlang",
FortranLegacy => "FORTRAN Legacy",
FortranModern => "FORTRAN Modern",
Go => "Go",
Haskell => "Haskell",
Html => "HTML",
Idris => "Idris",
Isabelle => "Isabelle",
Jai => "JAI",
Java => "Java",
JavaScript => "JavaScript",
Json => "JSON",
Jsx => "JSX",
Julia => "Julia",
Kotlin => "Kotlin",
Less => "LESS",
LinkerScript => "LD Script",
Lisp => "LISP",
Lua => "Lua",
Makefile => "Makefile",
Markdown => "Markdown",
Mustache => "Mustache",
Nim => "Nim",
ObjectiveC => "Objective C",
ObjectiveCpp => "Objective C++",
OCaml => "OCaml",
Oz => "Oz",
Pascal => "Pascal",
Perl => "Perl",
Polly => "Polly",
Php => "PHP",
Protobuf => "Protocol Buffers",
Prolog => "Prolog",
Python => "Python",
Qcl => "QCL",
R => "R",
Ruby => "Ruby",
RubyHtml => "Ruby HTML",
Rust => "Rust",
Sass => "Sass",
Scala => "Scala",
Sml => "Standard ML",
Sql => "SQL",
Swift => "Swift",
Tex => "TeX",
Text => "Plain Text",
Toml => "TOML",
TypeScript => "TypeScript",
UnrealScript => "Unreal Script",
VimScript => "Vim Script",
Wolfram => "Wolfram",
Xml => "XML",
Yaml => "YAML",
Zsh => "Zsh",
__Total => "Total",
}
}
pub fn from_extension<P: AsRef<Path>>(entry: P) -> Option<Self> {
if let Some(extension) = get_extension(entry) {
match &*extension {
"as" => Some(ActionScript),
"bash" | "sh" => Some(Bash),
"bat" | "btm" | "cmd" => Some(Batch),
"c" | "ec" | "pgc" => Some(C),
"cc" | "cpp" | "cxx" | "c++" | "pcc" => Some(Cpp),
"cfc" => Some(ColdFusionScript),
"cfm" => Some(ColdFusion),
"clj" => Some(Clojure),
"coffee" => Some(CoffeeScript),
"cs" => Some(CSharp),
"csh" => Some(CShell),
"css" => Some(Css),
"d" => Some(D),
"dart" => Some(Dart),
"dts" | "dtsi" => Some(DeviceTree),
"el" | "lisp" | "lsp" => Some(Lisp),
"erl" | "hrl" => Some(Erlang),
"f" | "for" | "ftn" | "f77" | "pfo" => Some(FortranLegacy),
"f03" | "f08" | "f90" | "f95" => Some(FortranModern),
"go" => Some(Go),
"h" => Some(CHeader),
"hh" | "hpp" | "hxx" => Some(CppHeader),
"hs" => Some(Haskell),
"html" => Some(Html),
"idr" | "lidr" => Some(Idris),
"jai" => Some(Jai),
"java" => Some(Java),
"jl" => Some(Julia),
"js" => Some(JavaScript),
"json" => Some(Json),
"jsx" => Some(Jsx),
"kt" | "kts" => Some(Kotlin),
"lds" => Some(LinkerScript),
"less" => Some(Less),
"lua" => Some(Lua),
"m" => Some(ObjectiveC),
"markdown" | "md" => Some(Markdown),
"ml" | "mli" => Some(OCaml),
"mm" => Some(ObjectiveCpp),
"makefile" => Some(Makefile),
"mustache" => Some(Mustache),
"nim" => Some(Nim),
"nb" | "wl" => Some(Wolfram),
"oz" => Some(Oz),
"p" | "pro" => Some(Prolog),
"pas" => Some(Pascal),
"php" => Some(Php),
"pl" => Some(Perl),
"qcl" => Some(Qcl),
"text" | "txt" => Some(Text),
"polly" => Some(Polly),
"proto" => Some(Protobuf),
"py" => Some(Python),
"r" => Some(R),
"rake" | "rb" => Some(Ruby),
"rhtml" => Some(RubyHtml),
"rs" | "in" => Some(Rust),
"s" => Some(Assembly),
"sass" | "scss" => Some(Sass),
"sc" | "scala" => Some(Scala),
"sml" => Some(Sml),
"sql" => Some(Sql),
"swift" => Some(Swift),
"tex" | "sty" => Some(Tex),
"toml" => Some(Toml),
"ts" => Some(TypeScript),
"thy" => Some(Isabelle),
"uc" | "uci" | "upkg" => Some(UnrealScript),
"v" => Some(Coq),
"vim" => Some(VimScript),
"xml" => Some(Xml),
"yaml" | "yml" => Some(Yaml),
"zsh" => Some(Zsh),
_ => None,
}
} else {
None
}
}
}
impl From<String> for LanguageType {
fn from(from: String) -> Self {
match &*from {
"ActionScript" => ActionScript,
"Assembly" => Assembly,
"Bash" => Bash,
"Batch" => Batch,
"C" => C,
"CHeader" => CHeader,
"Clojure" => Clojure,
"CoffeeScript" => CoffeeScript,
"ColdFusion" => ColdFusion,
"ColdFusionScript" => ColdFusionScript,
"Coq" => Coq,
"Cpp" => Cpp,
"CppHeader" => CppHeader,
"CSharp" => CSharp,
"CShell" => CShell,
"Css" => Css,
"D" => D,
"Dart" => Dart,
"DeviceTree" => DeviceTree,
"Erlang" => Erlang,
"FortranLegacy" => FortranLegacy,
"FortranModern" => FortranModern,
"Go" => Go,
"Haskell" => Haskell,
"Html" => Html,
"Idris" => Idris,
"Jai" => Jai,
"Java" => Java,
"JavaScript" => JavaScript,
"Julia" => Julia,
"Json" => Json,
"Jsx" => Jsx,
"Kotlin" => Kotlin,
"Less" => Less,
"LinkerScript" => LinkerScript,
"Lisp" => Lisp,
"Lua" => Lua,
"Makefile" => Makefile,
"Markdown" => Markdown,
"Mustache" => Mustache,
"Nim" => Nim,
"ObjectiveC" => ObjectiveC,
"ObjectiveCpp" => ObjectiveCpp,
"OCaml" => OCaml,
"Oz" => Oz,
"Pascal" => Pascal,
"Perl" => Perl,
"Polly" => Polly,
"Php" => Php,
"Protobuf" => Protobuf,
"Prolog" => Prolog,
"Python" => Python,
"Qcl" => Qcl,
"R" => R,
"Ruby" => Ruby,
"RubyHtml" => RubyHtml,
"Rust" => Rust,
"Sass" => Sass,
"Scala" => Scala,
"Sml" => Sml,
"Sql" => Sql,
"Swift" => Swift,
"Tex" => Tex,
"Text" => Text,
"Toml" => Toml,
"TypeScript" => TypeScript,
"VimScript" => VimScript,
"UnrealScript" => UnrealScript,
"Wolfram" => Wolfram,
"Xml" => Xml,
"Yaml" => Yaml,
"Zsh" => Zsh,
_ => unreachable!(),
}
}
}
impl fmt::Display for LanguageType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self.name())
}
}
impl<'a> From<LanguageType> for Cow<'a, LanguageType> {
fn from(from: LanguageType) -> Self {
Cow::Owned(from)
}
}
impl<'a> From<&'a LanguageType> for Cow<'a, LanguageType> {
fn from(from: &'a LanguageType) -> Self {
Cow::Borrowed(from)
}
}

View file

@ -0,0 +1,320 @@
// Copyright (c) 2015 Aaron Power
// Use of this source code is governed by the APACHE2.0/MIT licence that can be
// found in the LICENCE-{APACHE/MIT} file.
use std::borrow::Cow;
use std::collections::{btree_map, BTreeMap};
use std::fs::File;
use std::io::Read;
use std::iter::IntoIterator;
use std::ops::{AddAssign, Deref, DerefMut};
use serde_cbor;
use serde_json;
use serde_yaml;
use rayon::prelude::*;
use utils::*;
use super::{Language, LanguageType};
use super::LanguageType::*;
use stats::Stats;
#[derive(Debug, Clone)]
pub struct Languages {
inner: BTreeMap<LanguageType, Language>,
}
impl Languages {
pub fn from_cbor<'a, I: Into<&'a [u8]>>(cbor: I) -> serde_cbor::Result<Self> {
let map = try!(serde_cbor::from_slice(cbor.into()));
Ok(Self::from_previous(map))
}
pub fn from_json<'a, I: Into<&'a [u8]>>(json: I) -> serde_json::Result<Self> {
let map = try!(serde_json::from_slice(json.into()));
Ok(Self::from_previous(map))
}
pub fn from_yaml<'a, I: Into<&'a [u8]>>(yaml: I) -> serde_yaml::Result<Self> {
let map = try!(serde_yaml::from_slice(yaml.into()));
Ok(Self::from_previous(map))
}
fn from_previous(map: BTreeMap<LanguageType, Language>) -> Self {
let mut _self = Self::new();
for (name, input_language) in map {
if let Some(language) = _self.get_mut(&LanguageType::from(name)) {
*language += input_language;
}
}
_self
}
pub fn get_statistics<'a, I>(&mut self, paths: I, ignored: I)
where I: Into<Cow<'a, [&'a str]>>
{
get_all_files(paths.into(), ignored.into(), &mut self.inner);
let mut language_iter: Vec<_> = self.inner.iter_mut().collect();
language_iter.par_iter_mut().for_each(|&mut (name, ref mut language)| {
if language.files.is_empty() {
return;
}
language.total_files = language.files.len();
let is_fortran = name == &FortranModern || name == &FortranLegacy;
let files: Vec<_> = language.files.drain(..).collect();
for file in files {
let mut is_in_comments = false;
let mut previous_comment_start = "";
let mut comment_depth: usize = 0;
let mut stats = Stats::new(opt_or_cont!(file.to_str()));
let contents = {
let mut contents = String::new();
let _ = rs_or_cont!(rs_or_cont!(File::open(file))
.read_to_string(&mut contents));
contents
};
let lines = contents.lines();
if language.is_blank() {
stats.code += lines.count();
continue;
}
'line: for line in lines {
stats.lines += 1;
// FORTRAN has a rule where it only counts as a comment if it's the first
// character in the column, so removing starting whitespace could cause a
// miscount.
let line = if is_fortran {
line
} else {
line.trim()
};
if line.trim().is_empty() {
stats.blanks += 1;
continue;
}
for &(multi_line, multi_line_end) in &language.multi_line {
if line.starts_with(multi_line) ||
has_trailing_comments(line,
multi_line,
multi_line_end,
language.nested) {
previous_comment_start = multi_line;
is_in_comments = true;
if language.nested {
comment_depth += 1;
}
}
}
if is_in_comments {
for &(multi_line, multi_line_end) in &language.multi_line {
if multi_line == previous_comment_start &&
line.contains(multi_line_end) {
if language.nested {
comment_depth -= 1;
if comment_depth == 0 {
is_in_comments = false;
}
} else {
is_in_comments = false;
}
}
}
stats.comments += 1;
continue;
}
for single in &language.line_comment {
if line.starts_with(single) {
stats.comments += 1;
continue 'line;
}
}
stats.code += 1;
}
**language += stats;
}
});
}
pub fn new() -> Self {
use super::LanguageType::*;
let map = btreemap! {
ActionScript => Language::new_c(),
Assembly => Language::new_single(vec![";"]),
Bash => Language::new_hash(),
Batch => Language::new_single(vec!["REM"]),
C => Language::new_c(),
CHeader => Language::new_c(),
Clojure => Language::new_single(vec![";","#"]),
CoffeeScript => Language::new(vec!["#"], vec![("###", "###")]),
ColdFusion => Language::new_multi(vec![("<!---", "--->")]),
ColdFusionScript => Language::new_c(),
Coq => Language::new_func(),
Cpp => Language::new_c(),
CppHeader => Language::new_c(),
CSharp => Language::new_c(),
CShell => Language::new_hash(),
Css => Language::new_c(),
D => Language::new_c(),
Dart => Language::new_c(),
DeviceTree => Language::new_c(),
Erlang => Language::new_single(vec!["%"]),
FortranLegacy => Language::new_single(vec!["c","C","!","*"]),
FortranModern => Language::new_single(vec!["!"]),
Go => Language::new_c(),
Haskell => Language::new_single(vec!["--"]),
Html => Language::new_html(),
Idris => Language::new(vec!["--"], vec![("{-", "-}")]),
Isabelle => Language::new(
vec!["--"],
vec![ ("{*","*}"),
("(*","*)"),
("",""),
("\\<open>", "\\<close>"),
]
),
Jai => Language::new_c(),
Java => Language::new_c(),
JavaScript => Language::new_c(),
Json => Language::new_blank(),
Jsx => Language::new_c(),
Julia => Language::new(vec!["#"], vec![("#=", "=#")]),
Kotlin => Language::new_c(),
Less => Language::new_c(),
LinkerScript => Language::new_c(),
Lisp => Language::new(vec![";"], vec![("#|", "|#")]),
Lua => Language::new(vec!["--"], vec![("--[[", "]]")]),
Makefile => Language::new_hash(),
Markdown => Language::new_blank(),
Mustache => Language::new_multi(vec![("{{!", "}}")]),
Nim => Language::new_hash(),
ObjectiveC => Language::new_c(),
ObjectiveCpp => Language::new_c(),
OCaml => Language::new_func(),
Oz => Language::new_pro(),
Pascal => Language::new(vec!["//","(*"], vec![("{", "}")]),
Perl => Language::new(vec!["#"], vec![("=", "=cut")]),
Php => Language::new(vec!["#","//"], vec![("/*", "*/")]),
Polly => Language::new_html(),
Prolog => Language::new_pro(),
Protobuf => Language::new_single(vec!["//"]),
Python => Language::new(vec!["#"], vec![("'''", "'''")]),
Qcl => Language::new_c(),
R => Language::new_hash(),
Ruby => Language::new(vec!["#"], vec![("=begin", "=end")]),
RubyHtml => Language::new_html(),
Rust => Language::new_c().nested(),
Sass => Language::new_c(),
Scala => Language::new_c(),
Sml => Language::new_func(),
Sql => Language::new(vec!["--"], vec![("/*", "*/")]),
Swift => Language::new_c(),
Tex => Language::new_single(vec!["%"]),
Text => Language::new_blank(),
Toml => Language::new_hash(),
TypeScript => Language::new_c(),
UnrealScript => Language::new_c(),
VimScript => Language::new_single(vec!["\""]),
Wolfram => Language::new_func(),
Xml => Language::new_html(),
Yaml => Language::new_hash(),
Zsh => Language::new_hash(),
};
Languages { inner: map }
}
fn remove_empty(&self) -> BTreeMap<LanguageType, Language> {
let mut map: BTreeMap<LanguageType, Language> = BTreeMap::new();
for (name, language) in &self.inner {
if !language.is_empty() {
map.insert(name.clone(), language.clone());
}
}
map
}
pub fn to_cbor(&self) -> Result<Vec<u8>, serde_cbor::Error> {
serde_cbor::to_vec(&self.remove_empty())
}
pub fn to_json(&self) -> Result<String, serde_json::Error> {
serde_json::to_string(&self.remove_empty())
}
pub fn to_yaml(&self) -> Result<String, serde_yaml::Error> {
serde_yaml::to_string(&self.remove_empty())
}
}
impl IntoIterator for Languages {
type Item = <BTreeMap<LanguageType, Language> as IntoIterator>::Item;
type IntoIter = <BTreeMap<LanguageType, Language> as IntoIterator>::IntoIter;
fn into_iter(self) -> Self::IntoIter {
self.inner.into_iter()
}
}
impl<'a> IntoIterator for &'a Languages {
type Item = (&'a LanguageType, &'a Language);
type IntoIter = btree_map::Iter<'a, LanguageType, Language>;
fn into_iter(self) -> Self::IntoIter {
self.inner.iter()
}
}
impl<'a> IntoIterator for &'a mut Languages {
type Item = (&'a LanguageType, &'a mut Language);
type IntoIter = btree_map::IterMut<'a, LanguageType, Language>;
fn into_iter(self) -> Self::IntoIter {
self.inner.iter_mut()
}
}
impl AddAssign<BTreeMap<LanguageType, Language>> for Languages {
fn add_assign(&mut self, rhs: BTreeMap<LanguageType, Language>) {
for (name, language) in rhs {
if let Some(result) = self.inner.get_mut(&name) {
*result += language;
}
}
}
}
impl Deref for Languages {
type Target = BTreeMap<LanguageType, Language>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl DerefMut for Languages {
fn deref_mut(&mut self) -> &mut BTreeMap<LanguageType, Language> {
&mut self.inner
}
}

11
src/lib/language/mod.rs Normal file
View file

@ -0,0 +1,11 @@
// Copyright (c) 2015 Aaron Power
// Use of this source code is governed by the MIT/APACHE2.0 license that can be
// found in the LICENCE-{APACHE - MIT} file.
pub mod language;
pub mod languages;
pub mod language_type;
pub use self::languages::Languages;
pub use self::language::Language;
pub use self::language_type::*;

25
src/lib/lib.rs Normal file
View file

@ -0,0 +1,25 @@
#![deny(missing_docs,
missing_debug_implementations, missing_copy_implementations,
trivial_casts, trivial_numeric_casts,
unsafe_code,
unstable_features,
unused_import_braces)]
//! # Tokei: Code Analysis Library(For the [binary](https://github.com/Aaronepower/tokei/))
//!
//! Tokei is the library powering the application of the same name.
#[macro_use]
extern crate maplit;
#[macro_use]
extern crate serializable_enum;
extern crate glob;
extern crate rayon;
extern crate serde;
extern crate serde_cbor;
extern crate serde_json;
extern crate serde_yaml;
extern crate toml;
extern crate walkdir;
include!(concat!(env!("OUT_DIR"), "/lib.rs"));

9
src/lib/lib.rs.in Normal file
View file

@ -0,0 +1,9 @@
#[macro_use]
mod utils;
mod language;
mod stats;
pub use language::{LanguageType, Languages, Language};
pub use stats::Stats;
pub use utils::consts;

41
src/lib/stats.rs Normal file
View file

@ -0,0 +1,41 @@
use std::fmt;
#[derive(Clone, Default, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
pub struct Stats {
pub name: String,
pub code: usize,
pub blanks: usize,
pub lines: usize,
pub comments: usize,
}
impl Stats {
pub fn new<S: Into<String>>(name: S) -> Self {
Stats { name: name.into(), ..Self::default() }
}
}
impl fmt::Display for Stats {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let name_length = self.name.len();
let name = if name_length == 25 {
self.name.clone()
} else if self.name.len() > 24 {
let mut name = String::from("|");
name.push_str(&self.name[self.name.len() - 24..]);
name
} else {
self.name.clone()
};
write!(f,
" {: <25} {:>12} {:>12} {:>12} {:>12}",
name,
self.lines,
self.code,
self.comments,
self.blanks)
}
}

6
src/lib/utils/consts.rs Normal file
View file

@ -0,0 +1,6 @@
pub const BLANKS: &'static str = "blanks";
pub const COMMENTS: &'static str = "comments";
pub const CODE: &'static str = "code";
pub const FILES: &'static str = "files";
pub const LINES: &'static str = "lines";

174
src/lib/utils/fs.rs Normal file
View file

@ -0,0 +1,174 @@
// Copyright (c) 2015 Aaron Power
// Use of this source code is governed by the APACHE2.0/MIT licence that can be
// found in the LICENCE-{APACHE/MIT} file.
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::io::{BufRead, BufReader};
use std::fs::File;
use std::path::Path;
use glob::glob;
use walkdir::{WalkDir, WalkDirIterator};
use language::{Language, LanguageType};
use language::LanguageType::*;
/// This is used to catch lines like "let x = 5; /* Comment */"
pub fn has_trailing_comments(line: &str,
comment: &'static str,
comment_end: &'static str,
nested: bool)
-> bool {
let mut in_comments: usize = 0;
for chars in line.chars().collect::<Vec<char>>().windows(comment.len()) {
let window = {
let mut window = String::new();
for ch in chars {
window.push(*ch);
}
window
};
if window == comment {
if nested {
in_comments += 1;
} else {
in_comments = 1;
}
continue;
} else if window == comment_end {
if nested && in_comments != 0 {
in_comments -= 1;
} else {
in_comments = 0;
}
continue;
}
}
in_comments != 0
}
pub fn get_all_files<'a>(paths: Cow<'a, [&'a str]>,
ignored_directories: Cow<'a, [&'a str]>,
languages: &mut BTreeMap<LanguageType, Language>) {
for path in &*paths {
// A small metadata check to check if the file actually exists, this is used over calling
// File::open because we're going to be passing the path to either glob() or WalkDir::new()
if let Err(_) = Path::new(path).metadata() {
if let Ok(paths) = glob(path) {
'path: for path in paths {
let path = rs_or_cont!(path);
for ig in &*ignored_directories {
if opt_or_cont!(path.to_str()).contains(ig) {
continue 'path;
}
}
let mut language = if opt_or_cont!(path.to_str()).contains("Makefile") {
languages.get_mut(&Makefile).unwrap()
} else {
opt_or_cont!(
languages.get_mut(&opt_or_cont!(LanguageType::from_extension(&path))))
};
language.files.push(path.to_owned());
}
}
} else {
let walker = WalkDir::new(path).into_iter().filter_entry(|entry| {
for ig in &*ignored_directories {
if entry.path().to_str().unwrap().contains(&*ig) {
return false;
}
}
true
});
for entry in walker {
let entry = rs_or_cont!(entry);
let mut language = if opt_or_cont!(entry.path().to_str()).contains("Makefile") {
languages.get_mut(&Makefile).unwrap()
} else {
opt_or_cont!(
languages.get_mut(&opt_or_cont!(LanguageType::from_extension(entry.path())))
)
};
language.files.push(entry.path().to_owned());
}
}
}
}
pub fn get_extension<P: AsRef<Path>>(path: P) -> Option<String> {
let path = path.as_ref();
match path.extension() {
Some(extension_os) => {
match extension_os.to_str() {
Some(extension) => Some(extension.to_lowercase()),
None => None,
}
}
None => {
match get_filetype_from_shebang(path) {
// Using String::from here because all file extensions from
// get_filetype_from_shebang are guaranteed to be lowercase.
Some(extension) => Some(String::from(extension)),
None => None,
}
}
}
}
/// This is for getting the file type from the first line of a file
pub fn get_filetype_from_shebang<P: AsRef<Path>>(file: P) -> Option<&'static str> {
let file = match File::open(file) {
Ok(file) => file,
_ => return None,
};
let mut buf = BufReader::new(file);
let mut line = String::new();
let _ = buf.read_line(&mut line);
let mut words = line.split_whitespace();
match words.next() {
Some("#!/bin/sh") => Some("sh"),
Some("#!/bin/csh") => Some("csh"),
Some("#!/usr/bin/perl") => Some("pl"),
Some("#!/usr/bin/env") => {
match words.next() {
Some("python") | Some("python2") | Some("python3") => Some("py"),
Some("sh") => Some("sh"),
_ => None,
}
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn both_comments_in_line() {
assert!(!has_trailing_comments("Hello /* /* */ World", "/*", "*/", false));
}
#[test]
fn comment_start_in_line() {
assert!(has_trailing_comments("Hello /* World", "/*", "*/", false));
}
#[test]
fn both_comments_in_line_nested() {
assert!(has_trailing_comments("Hello (* (* *) World", "(*", "*)", true));
}
#[test]
fn comment_start_in_line_nested() {
assert!(has_trailing_comments("Hello (* World", "(*", "*)", true));
}
}

27
src/lib/utils/macros.rs Normal file
View file

@ -0,0 +1,27 @@
// Copyright (c) 2015 Aaron Power
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.
macro_rules! opt_or_cont {
($option:expr) => {
match $option {
Some(result) => result,
None => continue,
}
}
}
macro_rules! rs_or_cont {
($result:expr) => {
match $result {
Ok(result) => result,
Err(_) => continue,
}
}
}
macro_rules! debug {
($fmt:expr) => (if cfg!(debug_assertions) {println!($fmt)});
($fmt:expr, $($arg:tt)*) => (if cfg!(debug_assertions) {println!($fmt, $($arg)*)});
}

7
src/lib/utils/mod.rs Normal file
View file

@ -0,0 +1,7 @@
#[macro_use]
mod macros;
pub mod consts;
pub mod fs;
pub use self::consts::*;
pub use self::fs::*;

14
tokei.sublime-project Normal file
View file

@ -0,0 +1,14 @@
{
"folders":
[
{
"path": "C:\\Users\\Aaron\\Documents\\GitHub"
}
],
"settings":
{
"sublimegdb_workingdir": "C:\\Users\\Aaron\\Documents\\GitHub\\tokei\\",
"sublimegdb_commandline": "gdb target\\debug\\tokei.exe"
}
}

1309
tokei.sublime-workspace Normal file

File diff suppressed because it is too large Load diff