development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-27 21:09:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Search provider - implement FileIndexProvider

Browse source
This commit is contained in:
Rob Lourens 2018-07-25 20:48:18 -07:00
parent 9999dac541
commit d36a3d2395
19 changed files with 899 additions and 2069 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

234
extensions/search-rg/src/cachedSearchProvider.ts
View file

@ -1,234 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as path from 'path';
import * as vscode from 'vscode';
import * as arrays from './common/arrays';
import { compareItemsByScore, IItemAccessor, prepareQuery, ScorerCache } from './common/fileSearchScorer';
import * as strings from './common/strings';
import { joinPath } from './utils';
interface IProviderArgs {
query: vscode.FileSearchQuery;
options: vscode.FileSearchOptions;
progress: vscode.Progress<vscode.Uri>;
token: vscode.CancellationToken;
}
export interface IInternalFileSearchProvider {
provideFileSearchResults(options: vscode.FileSearchOptions, progress: vscode.Progress<string>, token: vscode.CancellationToken): Thenable<void>;
}
export class CachedSearchProvider {
private static readonly BATCH_SIZE = 512;
private caches: { [cacheKey: string]: Cache; } = Object.create(null);
provideFileSearchResults(provider: IInternalFileSearchProvider, query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<vscode.Uri>, token: vscode.CancellationToken): Thenable<void> {
const onResult = (result: IInternalFileMatch) => {
progress.report(joinPath(options.folder, result.relativePath));
};
const providerArgs: IProviderArgs = {
query, options, progress, token
};
let sortedSearch = this.trySortedSearchFromCache(providerArgs, onResult);
if (!sortedSearch) {
const engineOpts = options.maxResults ?
{
...options,
...{ maxResults: 1e9 }
} :
options;
providerArgs.options = engineOpts;
sortedSearch = this.doSortedSearch(providerArgs, provider);
}
return sortedSearch.then(rawMatches => {
rawMatches.forEach(onResult);
});
}
private doSortedSearch(args: IProviderArgs, provider: IInternalFileSearchProvider): Promise<IInternalFileMatch[]> {
const allResultsPromise = new Promise<IInternalFileMatch[]>((c, e) => {
const results: IInternalFileMatch[] = [];
const onResult = (progress: IInternalFileMatch[]) => results.push(...progress);
// TODO@roblou set maxResult = null
this.doSearch(args, provider, onResult, CachedSearchProvider.BATCH_SIZE)
.then(() => c(results), e);
});
let cache: Cache;
if (args.query.cacheKey) {
cache = this.getOrCreateCache(args.query.cacheKey); // TODO include folder in cache key
cache.resultsToSearchCache[args.query.pattern] = { finished: allResultsPromise };
allResultsPromise.then(null, err => {
delete cache.resultsToSearchCache[args.query.pattern];
});
}
return allResultsPromise.then(results => {
// TODO@roblou quickopen results are not scored until the first keypress
if (args.query.pattern) {
const scorerCache: ScorerCache = cache ? cache.scorerCache : Object.create(null);
return this.sortResults(args, results, scorerCache);
} else {
return results;
}
});
}
private getOrCreateCache(cacheKey: string): Cache {
const existing = this.caches[cacheKey];
if (existing) {
return existing;
}
return this.caches[cacheKey] = new Cache();
}
private trySortedSearchFromCache(args: IProviderArgs, onResult: (result: IInternalFileMatch) => void): Promise<IInternalFileMatch[]> {
const cache = args.query.cacheKey && this.caches[args.query.cacheKey];
if (!cache) {
return undefined;
}
const cached = this.getResultsFromCache(cache, args.query.pattern, onResult);
if (cached) {
return cached.then((results) => this.sortResults(args, results, cache.scorerCache));
}
return undefined;
}
private sortResults(args: IProviderArgs, results: IInternalFileMatch[], scorerCache: ScorerCache): Promise<IInternalFileMatch[]> {
// we use the same compare function that is used later when showing the results using fuzzy scoring
// this is very important because we are also limiting the number of results by config.maxResults
// and as such we want the top items to be included in this result set if the number of items
// exceeds config.maxResults.
const preparedQuery = prepareQuery(args.query.pattern);
const compare = (matchA: IInternalFileMatch, matchB: IInternalFileMatch) => compareItemsByScore(matchA, matchB, preparedQuery, true, FileMatchItemAccessor, scorerCache);
return arrays.topAsync(results, compare, args.options.maxResults || 0, 10000);
}
private getResultsFromCache(cache: Cache, searchValue: string, onResult: (results: IInternalFileMatch) => void): Promise<IInternalFileMatch[]> {
// Find cache entries by prefix of search value
const hasPathSep = searchValue.indexOf(path.sep) >= 0;
let cached: CacheEntry<IInternalFileMatch>;
let wasResolved: boolean;
for (let previousSearch in cache.resultsToSearchCache) {
// If we narrow down, we might be able to reuse the cached results
if (searchValue.startsWith(previousSearch)) {
if (hasPathSep && previousSearch.indexOf(path.sep) < 0) {
continue; // since a path character widens the search for potential more matches, require it in previous search too
}
const c = cache.resultsToSearchCache[previousSearch];
c.finished.then(() => { wasResolved = false; });
cached = c;
wasResolved = true;
break;
}
}
if (!cached) {
return null;
}
return new Promise((c, e) => {
cached.finished.then(cachedEntries => {
const cacheFilterStartTime = Date.now();
// Pattern match on results
let results: IInternalFileMatch[] = [];
const normalizedSearchValueLowercase = strings.stripWildcards(searchValue).toLowerCase();
for (let i = 0; i < cachedEntries.length; i++) {
let entry = cachedEntries[i];
// Check if this entry is a match for the search value
if (!strings.fuzzyContains(entry.relativePath, normalizedSearchValueLowercase)) {
continue;
}
results.push(entry);
}
c(results);
}, e);
});
}
private doSearch(args: IProviderArgs, provider: IInternalFileSearchProvider, onResult: (result: IInternalFileMatch[]) => void, batchSize: number): Promise<void> {
return new Promise<void>((c, e) => {
let batch: IInternalFileMatch[] = [];
const onProviderResult = (match: string) => {
if (match) {
const internalMatch: IInternalFileMatch = {
relativePath: match,
basename: path.basename(match)
};
batch.push(internalMatch);
if (batchSize > 0 && batch.length >= batchSize) {
onResult(batch);
batch = [];
}
}
};
provider.provideFileSearchResults(args.options, { report: onProviderResult }, args.token).then(() => {
if (batch.length) {
onResult(batch);
}
c();
}, error => {
if (batch.length) {
onResult(batch);
}
e(error);
});
});
}
public clearCache(cacheKey: string): Promise<void> {
delete this.caches[cacheKey];
return Promise.resolve(undefined);
}
}
interface IInternalFileMatch {
relativePath?: string; // Not present for extraFiles or absolute path matches
basename: string;
}
interface CacheEntry<T> {
finished: Promise<T[]>;
}
class Cache {
public resultsToSearchCache: { [searchValue: string]: CacheEntry<IInternalFileMatch> } = Object.create(null);
public scorerCache: ScorerCache = Object.create(null);
}
const FileMatchItemAccessor = new class implements IItemAccessor<IInternalFileMatch> {
public getItemLabel(match: IInternalFileMatch): string {
return match.basename; // e.g. myFile.txt
}
public getItemDescription(match: IInternalFileMatch): string {
return match.relativePath.substr(0, match.relativePath.length - match.basename.length - 1); // e.g. some/path/to/file
}
public getItemPath(match: IInternalFileMatch): string {
return match.relativePath; // e.g. some/path/to/file/myFile.txt
}
};

75
extensions/search-rg/src/common/arrays.ts
View file

@ -1,75 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
/**
* Asynchronous variant of `top()` allowing for splitting up work in batches between which the event loop can run.
*
* Returns the top N elements from the array.
*
* Faster than sorting the entire array when the array is a lot larger than N.
*
* @param array The unsorted array.
* @param compare A sort function for the elements.
* @param n The number of elements to return.
* @param batch The number of elements to examine before yielding to the event loop.
* @return The first n elemnts from array when sorted with compare.
*/
export function topAsync<T>(array: T[], compare: (a: T, b: T) => number, n: number, batch: number): Promise<T[]> {
// TODO@roblou cancellation
if (n === 0) {
return Promise.resolve([]);
}
let canceled = false;
return new Promise((resolve, reject) => {
(async () => {
const o = array.length;
const result = array.slice(0, n).sort(compare);
for (let i = n, m = Math.min(n + batch, o); i < o; i = m, m = Math.min(m + batch, o)) {
if (i > n) {
await new Promise(resolve => setTimeout(resolve, 0)); // nextTick() would starve I/O.
}
if (canceled) {
throw new Error('canceled');
}
topStep(array, compare, result, i, m);
}
return result;
})()
.then(resolve, reject);
});
}
function topStep<T>(array: T[], compare: (a: T, b: T) => number, result: T[], i: number, m: number): void {
for (const n = result.length; i < m; i++) {
const element = array[i];
if (compare(element, result[n - 1]) < 0) {
result.pop();
const j = findFirstInSorted(result, e => compare(element, e) < 0);
result.splice(j, 0, element);
}
}
}
/**
* Takes a sorted array and a function p. The array is sorted in such a way that all elements where p(x) is false
* are located before all elements where p(x) is true.
* @returns the least x for which p(x) is true or array.length if no element fullfills the given function.
*/
export function findFirstInSorted<T>(array: T[], p: (x: T) => boolean): number {
let low = 0, high = array.length;
if (high === 0) {
return 0; // no children
}
while (low < high) {
let mid = Math.floor((low + high) / 2);
if (p(array[mid])) {
high = mid;
} else {
low = mid + 1;
}
}
return low;
}

422
extensions/search-rg/src/common/charCode.ts
View file

@ -1,422 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
// Names from https://blog.codinghorror.com/ascii-pronunciation-rules-for-programmers/
/**
* An inlined enum containing useful character codes (to be used with String.charCodeAt).
* Please leave the const keyword such that it gets inlined when compiled to JavaScript!
*/
export const enum CharCode {
Null = 0,
/**
* The `\t` character.
*/
Tab = 9,
/**
* The `\n` character.
*/
LineFeed = 10,
/**
* The `\r` character.
*/
CarriageReturn = 13,
Space = 32,
/**
* The `!` character.
*/
ExclamationMark = 33,
/**
* The `"` character.
*/
DoubleQuote = 34,
/**
* The `#` character.
*/
Hash = 35,
/**
* The `$` character.
*/
DollarSign = 36,
/**
* The `%` character.
*/
PercentSign = 37,
/**
* The `&` character.
*/
Ampersand = 38,
/**
* The `'` character.
*/
SingleQuote = 39,
/**
* The `(` character.
*/
OpenParen = 40,
/**
* The `)` character.
*/
CloseParen = 41,
/**
* The `*` character.
*/
Asterisk = 42,
/**
* The `+` character.
*/
Plus = 43,
/**
* The `,` character.
*/
Comma = 44,
/**
* The `-` character.
*/
Dash = 45,
/**
* The `.` character.
*/
Period = 46,
/**
* The `/` character.
*/
Slash = 47,
Digit0 = 48,
Digit1 = 49,
Digit2 = 50,
Digit3 = 51,
Digit4 = 52,
Digit5 = 53,
Digit6 = 54,
Digit7 = 55,
Digit8 = 56,
Digit9 = 57,
/**
* The `:` character.
*/
Colon = 58,
/**
* The `;` character.
*/
Semicolon = 59,
/**
* The `<` character.
*/
LessThan = 60,
/**
* The `=` character.
*/
Equals = 61,
/**
* The `>` character.
*/
GreaterThan = 62,
/**
* The `?` character.
*/
QuestionMark = 63,
/**
* The `@` character.
*/
AtSign = 64,
A = 65,
B = 66,
C = 67,
D = 68,
E = 69,
F = 70,
G = 71,
H = 72,
I = 73,
J = 74,
K = 75,
L = 76,
M = 77,
N = 78,
O = 79,
P = 80,
Q = 81,
R = 82,
S = 83,
T = 84,
U = 85,
V = 86,
W = 87,
X = 88,
Y = 89,
Z = 90,
/**
* The `[` character.
*/
OpenSquareBracket = 91,
/**
* The `\` character.
*/
Backslash = 92,
/**
* The `]` character.
*/
CloseSquareBracket = 93,
/**
* The `^` character.
*/
Caret = 94,
/**
* The `_` character.
*/
Underline = 95,
/**
* The ``(`)`` character.
*/
BackTick = 96,
a = 97,
b = 98,
c = 99,
d = 100,
e = 101,
f = 102,
g = 103,
h = 104,
i = 105,
j = 106,
k = 107,
l = 108,
m = 109,
n = 110,
o = 111,
p = 112,
q = 113,
r = 114,
s = 115,
t = 116,
u = 117,
v = 118,
w = 119,
x = 120,
y = 121,
z = 122,
/**
* The `{` character.
*/
OpenCurlyBrace = 123,
/**
* The `|` character.
*/
Pipe = 124,
/**
* The `}` character.
*/
CloseCurlyBrace = 125,
/**
* The `~` character.
*/
Tilde = 126,
U_Combining_Grave_Accent = 0x0300, // U+0300 Combining Grave Accent
U_Combining_Acute_Accent = 0x0301, // U+0301 Combining Acute Accent
U_Combining_Circumflex_Accent = 0x0302, // U+0302 Combining Circumflex Accent
U_Combining_Tilde = 0x0303, // U+0303 Combining Tilde
U_Combining_Macron = 0x0304, // U+0304 Combining Macron
U_Combining_Overline = 0x0305, // U+0305 Combining Overline
U_Combining_Breve = 0x0306, // U+0306 Combining Breve
U_Combining_Dot_Above = 0x0307, // U+0307 Combining Dot Above
U_Combining_Diaeresis = 0x0308, // U+0308 Combining Diaeresis
U_Combining_Hook_Above = 0x0309, // U+0309 Combining Hook Above
U_Combining_Ring_Above = 0x030A, // U+030A Combining Ring Above
U_Combining_Double_Acute_Accent = 0x030B, // U+030B Combining Double Acute Accent
U_Combining_Caron = 0x030C, // U+030C Combining Caron
U_Combining_Vertical_Line_Above = 0x030D, // U+030D Combining Vertical Line Above
U_Combining_Double_Vertical_Line_Above = 0x030E, // U+030E Combining Double Vertical Line Above
U_Combining_Double_Grave_Accent = 0x030F, // U+030F Combining Double Grave Accent
U_Combining_Candrabindu = 0x0310, // U+0310 Combining Candrabindu
U_Combining_Inverted_Breve = 0x0311, // U+0311 Combining Inverted Breve
U_Combining_Turned_Comma_Above = 0x0312, // U+0312 Combining Turned Comma Above
U_Combining_Comma_Above = 0x0313, // U+0313 Combining Comma Above
U_Combining_Reversed_Comma_Above = 0x0314, // U+0314 Combining Reversed Comma Above
U_Combining_Comma_Above_Right = 0x0315, // U+0315 Combining Comma Above Right
U_Combining_Grave_Accent_Below = 0x0316, // U+0316 Combining Grave Accent Below
U_Combining_Acute_Accent_Below = 0x0317, // U+0317 Combining Acute Accent Below
U_Combining_Left_Tack_Below = 0x0318, // U+0318 Combining Left Tack Below
U_Combining_Right_Tack_Below = 0x0319, // U+0319 Combining Right Tack Below
U_Combining_Left_Angle_Above = 0x031A, // U+031A Combining Left Angle Above
U_Combining_Horn = 0x031B, // U+031B Combining Horn
U_Combining_Left_Half_Ring_Below = 0x031C, // U+031C Combining Left Half Ring Below
U_Combining_Up_Tack_Below = 0x031D, // U+031D Combining Up Tack Below
U_Combining_Down_Tack_Below = 0x031E, // U+031E Combining Down Tack Below
U_Combining_Plus_Sign_Below = 0x031F, // U+031F Combining Plus Sign Below
U_Combining_Minus_Sign_Below = 0x0320, // U+0320 Combining Minus Sign Below
U_Combining_Palatalized_Hook_Below = 0x0321, // U+0321 Combining Palatalized Hook Below
U_Combining_Retroflex_Hook_Below = 0x0322, // U+0322 Combining Retroflex Hook Below
U_Combining_Dot_Below = 0x0323, // U+0323 Combining Dot Below
U_Combining_Diaeresis_Below = 0x0324, // U+0324 Combining Diaeresis Below
U_Combining_Ring_Below = 0x0325, // U+0325 Combining Ring Below
U_Combining_Comma_Below = 0x0326, // U+0326 Combining Comma Below
U_Combining_Cedilla = 0x0327, // U+0327 Combining Cedilla
U_Combining_Ogonek = 0x0328, // U+0328 Combining Ogonek
U_Combining_Vertical_Line_Below = 0x0329, // U+0329 Combining Vertical Line Below
U_Combining_Bridge_Below = 0x032A, // U+032A Combining Bridge Below
U_Combining_Inverted_Double_Arch_Below = 0x032B, // U+032B Combining Inverted Double Arch Below
U_Combining_Caron_Below = 0x032C, // U+032C Combining Caron Below
U_Combining_Circumflex_Accent_Below = 0x032D, // U+032D Combining Circumflex Accent Below
U_Combining_Breve_Below = 0x032E, // U+032E Combining Breve Below
U_Combining_Inverted_Breve_Below = 0x032F, // U+032F Combining Inverted Breve Below
U_Combining_Tilde_Below = 0x0330, // U+0330 Combining Tilde Below
U_Combining_Macron_Below = 0x0331, // U+0331 Combining Macron Below
U_Combining_Low_Line = 0x0332, // U+0332 Combining Low Line
U_Combining_Double_Low_Line = 0x0333, // U+0333 Combining Double Low Line
U_Combining_Tilde_Overlay = 0x0334, // U+0334 Combining Tilde Overlay
U_Combining_Short_Stroke_Overlay = 0x0335, // U+0335 Combining Short Stroke Overlay
U_Combining_Long_Stroke_Overlay = 0x0336, // U+0336 Combining Long Stroke Overlay
U_Combining_Short_Solidus_Overlay = 0x0337, // U+0337 Combining Short Solidus Overlay
U_Combining_Long_Solidus_Overlay = 0x0338, // U+0338 Combining Long Solidus Overlay
U_Combining_Right_Half_Ring_Below = 0x0339, // U+0339 Combining Right Half Ring Below
U_Combining_Inverted_Bridge_Below = 0x033A, // U+033A Combining Inverted Bridge Below
U_Combining_Square_Below = 0x033B, // U+033B Combining Square Below
U_Combining_Seagull_Below = 0x033C, // U+033C Combining Seagull Below
U_Combining_X_Above = 0x033D, // U+033D Combining X Above
U_Combining_Vertical_Tilde = 0x033E, // U+033E Combining Vertical Tilde
U_Combining_Double_Overline = 0x033F, // U+033F Combining Double Overline
U_Combining_Grave_Tone_Mark = 0x0340, // U+0340 Combining Grave Tone Mark
U_Combining_Acute_Tone_Mark = 0x0341, // U+0341 Combining Acute Tone Mark
U_Combining_Greek_Perispomeni = 0x0342, // U+0342 Combining Greek Perispomeni
U_Combining_Greek_Koronis = 0x0343, // U+0343 Combining Greek Koronis
U_Combining_Greek_Dialytika_Tonos = 0x0344, // U+0344 Combining Greek Dialytika Tonos
U_Combining_Greek_Ypogegrammeni = 0x0345, // U+0345 Combining Greek Ypogegrammeni
U_Combining_Bridge_Above = 0x0346, // U+0346 Combining Bridge Above
U_Combining_Equals_Sign_Below = 0x0347, // U+0347 Combining Equals Sign Below
U_Combining_Double_Vertical_Line_Below = 0x0348, // U+0348 Combining Double Vertical Line Below
U_Combining_Left_Angle_Below = 0x0349, // U+0349 Combining Left Angle Below
U_Combining_Not_Tilde_Above = 0x034A, // U+034A Combining Not Tilde Above
U_Combining_Homothetic_Above = 0x034B, // U+034B Combining Homothetic Above
U_Combining_Almost_Equal_To_Above = 0x034C, // U+034C Combining Almost Equal To Above
U_Combining_Left_Right_Arrow_Below = 0x034D, // U+034D Combining Left Right Arrow Below
U_Combining_Upwards_Arrow_Below = 0x034E, // U+034E Combining Upwards Arrow Below
U_Combining_Grapheme_Joiner = 0x034F, // U+034F Combining Grapheme Joiner
U_Combining_Right_Arrowhead_Above = 0x0350, // U+0350 Combining Right Arrowhead Above
U_Combining_Left_Half_Ring_Above = 0x0351, // U+0351 Combining Left Half Ring Above
U_Combining_Fermata = 0x0352, // U+0352 Combining Fermata
U_Combining_X_Below = 0x0353, // U+0353 Combining X Below
U_Combining_Left_Arrowhead_Below = 0x0354, // U+0354 Combining Left Arrowhead Below
U_Combining_Right_Arrowhead_Below = 0x0355, // U+0355 Combining Right Arrowhead Below
U_Combining_Right_Arrowhead_And_Up_Arrowhead_Below = 0x0356, // U+0356 Combining Right Arrowhead And Up Arrowhead Below
U_Combining_Right_Half_Ring_Above = 0x0357, // U+0357 Combining Right Half Ring Above
U_Combining_Dot_Above_Right = 0x0358, // U+0358 Combining Dot Above Right
U_Combining_Asterisk_Below = 0x0359, // U+0359 Combining Asterisk Below
U_Combining_Double_Ring_Below = 0x035A, // U+035A Combining Double Ring Below
U_Combining_Zigzag_Above = 0x035B, // U+035B Combining Zigzag Above
U_Combining_Double_Breve_Below = 0x035C, // U+035C Combining Double Breve Below
U_Combining_Double_Breve = 0x035D, // U+035D Combining Double Breve
U_Combining_Double_Macron = 0x035E, // U+035E Combining Double Macron
U_Combining_Double_Macron_Below = 0x035F, // U+035F Combining Double Macron Below
U_Combining_Double_Tilde = 0x0360, // U+0360 Combining Double Tilde
U_Combining_Double_Inverted_Breve = 0x0361, // U+0361 Combining Double Inverted Breve
U_Combining_Double_Rightwards_Arrow_Below = 0x0362, // U+0362 Combining Double Rightwards Arrow Below
U_Combining_Latin_Small_Letter_A = 0x0363, // U+0363 Combining Latin Small Letter A
U_Combining_Latin_Small_Letter_E = 0x0364, // U+0364 Combining Latin Small Letter E
U_Combining_Latin_Small_Letter_I = 0x0365, // U+0365 Combining Latin Small Letter I
U_Combining_Latin_Small_Letter_O = 0x0366, // U+0366 Combining Latin Small Letter O
U_Combining_Latin_Small_Letter_U = 0x0367, // U+0367 Combining Latin Small Letter U
U_Combining_Latin_Small_Letter_C = 0x0368, // U+0368 Combining Latin Small Letter C
U_Combining_Latin_Small_Letter_D = 0x0369, // U+0369 Combining Latin Small Letter D
U_Combining_Latin_Small_Letter_H = 0x036A, // U+036A Combining Latin Small Letter H
U_Combining_Latin_Small_Letter_M = 0x036B, // U+036B Combining Latin Small Letter M
U_Combining_Latin_Small_Letter_R = 0x036C, // U+036C Combining Latin Small Letter R
U_Combining_Latin_Small_Letter_T = 0x036D, // U+036D Combining Latin Small Letter T
U_Combining_Latin_Small_Letter_V = 0x036E, // U+036E Combining Latin Small Letter V
U_Combining_Latin_Small_Letter_X = 0x036F, // U+036F Combining Latin Small Letter X
/**
* Unicode Character 'LINE SEPARATOR' (U+2028)
* http://www.fileformat.info/info/unicode/char/2028/index.htm
*/
LINE_SEPARATOR_2028 = 8232,
// http://www.fileformat.info/info/unicode/category/Sk/list.htm
U_CIRCUMFLEX = 0x005E, // U+005E CIRCUMFLEX
U_GRAVE_ACCENT = 0x0060, // U+0060 GRAVE ACCENT
U_DIAERESIS = 0x00A8, // U+00A8 DIAERESIS
U_MACRON = 0x00AF, // U+00AF MACRON
U_ACUTE_ACCENT = 0x00B4, // U+00B4 ACUTE ACCENT
U_CEDILLA = 0x00B8, // U+00B8 CEDILLA
U_MODIFIER_LETTER_LEFT_ARROWHEAD = 0x02C2, // U+02C2 MODIFIER LETTER LEFT ARROWHEAD
U_MODIFIER_LETTER_RIGHT_ARROWHEAD = 0x02C3, // U+02C3 MODIFIER LETTER RIGHT ARROWHEAD
U_MODIFIER_LETTER_UP_ARROWHEAD = 0x02C4, // U+02C4 MODIFIER LETTER UP ARROWHEAD
U_MODIFIER_LETTER_DOWN_ARROWHEAD = 0x02C5, // U+02C5 MODIFIER LETTER DOWN ARROWHEAD
U_MODIFIER_LETTER_CENTRED_RIGHT_HALF_RING = 0x02D2, // U+02D2 MODIFIER LETTER CENTRED RIGHT HALF RING
U_MODIFIER_LETTER_CENTRED_LEFT_HALF_RING = 0x02D3, // U+02D3 MODIFIER LETTER CENTRED LEFT HALF RING
U_MODIFIER_LETTER_UP_TACK = 0x02D4, // U+02D4 MODIFIER LETTER UP TACK
U_MODIFIER_LETTER_DOWN_TACK = 0x02D5, // U+02D5 MODIFIER LETTER DOWN TACK
U_MODIFIER_LETTER_PLUS_SIGN = 0x02D6, // U+02D6 MODIFIER LETTER PLUS SIGN
U_MODIFIER_LETTER_MINUS_SIGN = 0x02D7, // U+02D7 MODIFIER LETTER MINUS SIGN
U_BREVE = 0x02D8, // U+02D8 BREVE
U_DOT_ABOVE = 0x02D9, // U+02D9 DOT ABOVE
U_RING_ABOVE = 0x02DA, // U+02DA RING ABOVE
U_OGONEK = 0x02DB, // U+02DB OGONEK
U_SMALL_TILDE = 0x02DC, // U+02DC SMALL TILDE
U_DOUBLE_ACUTE_ACCENT = 0x02DD, // U+02DD DOUBLE ACUTE ACCENT
U_MODIFIER_LETTER_RHOTIC_HOOK = 0x02DE, // U+02DE MODIFIER LETTER RHOTIC HOOK
U_MODIFIER_LETTER_CROSS_ACCENT = 0x02DF, // U+02DF MODIFIER LETTER CROSS ACCENT
U_MODIFIER_LETTER_EXTRA_HIGH_TONE_BAR = 0x02E5, // U+02E5 MODIFIER LETTER EXTRA-HIGH TONE BAR
U_MODIFIER_LETTER_HIGH_TONE_BAR = 0x02E6, // U+02E6 MODIFIER LETTER HIGH TONE BAR
U_MODIFIER_LETTER_MID_TONE_BAR = 0x02E7, // U+02E7 MODIFIER LETTER MID TONE BAR
U_MODIFIER_LETTER_LOW_TONE_BAR = 0x02E8, // U+02E8 MODIFIER LETTER LOW TONE BAR
U_MODIFIER_LETTER_EXTRA_LOW_TONE_BAR = 0x02E9, // U+02E9 MODIFIER LETTER EXTRA-LOW TONE BAR
U_MODIFIER_LETTER_YIN_DEPARTING_TONE_MARK = 0x02EA, // U+02EA MODIFIER LETTER YIN DEPARTING TONE MARK
U_MODIFIER_LETTER_YANG_DEPARTING_TONE_MARK = 0x02EB, // U+02EB MODIFIER LETTER YANG DEPARTING TONE MARK
U_MODIFIER_LETTER_UNASPIRATED = 0x02ED, // U+02ED MODIFIER LETTER UNASPIRATED
U_MODIFIER_LETTER_LOW_DOWN_ARROWHEAD = 0x02EF, // U+02EF MODIFIER LETTER LOW DOWN ARROWHEAD
U_MODIFIER_LETTER_LOW_UP_ARROWHEAD = 0x02F0, // U+02F0 MODIFIER LETTER LOW UP ARROWHEAD
U_MODIFIER_LETTER_LOW_LEFT_ARROWHEAD = 0x02F1, // U+02F1 MODIFIER LETTER LOW LEFT ARROWHEAD
U_MODIFIER_LETTER_LOW_RIGHT_ARROWHEAD = 0x02F2, // U+02F2 MODIFIER LETTER LOW RIGHT ARROWHEAD
U_MODIFIER_LETTER_LOW_RING = 0x02F3, // U+02F3 MODIFIER LETTER LOW RING
U_MODIFIER_LETTER_MIDDLE_GRAVE_ACCENT = 0x02F4, // U+02F4 MODIFIER LETTER MIDDLE GRAVE ACCENT
U_MODIFIER_LETTER_MIDDLE_DOUBLE_GRAVE_ACCENT = 0x02F5, // U+02F5 MODIFIER LETTER MIDDLE DOUBLE GRAVE ACCENT
U_MODIFIER_LETTER_MIDDLE_DOUBLE_ACUTE_ACCENT = 0x02F6, // U+02F6 MODIFIER LETTER MIDDLE DOUBLE ACUTE ACCENT
U_MODIFIER_LETTER_LOW_TILDE = 0x02F7, // U+02F7 MODIFIER LETTER LOW TILDE
U_MODIFIER_LETTER_RAISED_COLON = 0x02F8, // U+02F8 MODIFIER LETTER RAISED COLON
U_MODIFIER_LETTER_BEGIN_HIGH_TONE = 0x02F9, // U+02F9 MODIFIER LETTER BEGIN HIGH TONE
U_MODIFIER_LETTER_END_HIGH_TONE = 0x02FA, // U+02FA MODIFIER LETTER END HIGH TONE
U_MODIFIER_LETTER_BEGIN_LOW_TONE = 0x02FB, // U+02FB MODIFIER LETTER BEGIN LOW TONE
U_MODIFIER_LETTER_END_LOW_TONE = 0x02FC, // U+02FC MODIFIER LETTER END LOW TONE
U_MODIFIER_LETTER_SHELF = 0x02FD, // U+02FD MODIFIER LETTER SHELF
U_MODIFIER_LETTER_OPEN_SHELF = 0x02FE, // U+02FE MODIFIER LETTER OPEN SHELF
U_MODIFIER_LETTER_LOW_LEFT_ARROW = 0x02FF, // U+02FF MODIFIER LETTER LOW LEFT ARROW
U_GREEK_LOWER_NUMERAL_SIGN = 0x0375, // U+0375 GREEK LOWER NUMERAL SIGN
U_GREEK_TONOS = 0x0384, // U+0384 GREEK TONOS
U_GREEK_DIALYTIKA_TONOS = 0x0385, // U+0385 GREEK DIALYTIKA TONOS
U_GREEK_KORONIS = 0x1FBD, // U+1FBD GREEK KORONIS
U_GREEK_PSILI = 0x1FBF, // U+1FBF GREEK PSILI
U_GREEK_PERISPOMENI = 0x1FC0, // U+1FC0 GREEK PERISPOMENI
U_GREEK_DIALYTIKA_AND_PERISPOMENI = 0x1FC1, // U+1FC1 GREEK DIALYTIKA AND PERISPOMENI
U_GREEK_PSILI_AND_VARIA = 0x1FCD, // U+1FCD GREEK PSILI AND VARIA
U_GREEK_PSILI_AND_OXIA = 0x1FCE, // U+1FCE GREEK PSILI AND OXIA
U_GREEK_PSILI_AND_PERISPOMENI = 0x1FCF, // U+1FCF GREEK PSILI AND PERISPOMENI
U_GREEK_DASIA_AND_VARIA = 0x1FDD, // U+1FDD GREEK DASIA AND VARIA
U_GREEK_DASIA_AND_OXIA = 0x1FDE, // U+1FDE GREEK DASIA AND OXIA
U_GREEK_DASIA_AND_PERISPOMENI = 0x1FDF, // U+1FDF GREEK DASIA AND PERISPOMENI
U_GREEK_DIALYTIKA_AND_VARIA = 0x1FED, // U+1FED GREEK DIALYTIKA AND VARIA
U_GREEK_DIALYTIKA_AND_OXIA = 0x1FEE, // U+1FEE GREEK DIALYTIKA AND OXIA
U_GREEK_VARIA = 0x1FEF, // U+1FEF GREEK VARIA
U_GREEK_OXIA = 0x1FFD, // U+1FFD GREEK OXIA
U_GREEK_DASIA = 0x1FFE, // U+1FFE GREEK DASIA
U_OVERLINE = 0x203E, // Unicode Character 'OVERLINE'
/**
* UTF-8 BOM
* Unicode Character 'ZERO WIDTH NO-BREAK SPACE' (U+FEFF)
* http://www.fileformat.info/info/unicode/char/feff/index.htm
*/
UTF8_BOM = 65279
}

115
extensions/search-rg/src/common/comparers.ts
View file

@ -1,115 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as strings from './strings';
let intlFileNameCollator: Intl.Collator;
let intlFileNameCollatorIsNumeric: boolean;
setFileNameComparer(new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }));
export function setFileNameComparer(collator: Intl.Collator): void {
intlFileNameCollator = collator;
intlFileNameCollatorIsNumeric = collator.resolvedOptions().numeric;
}
export function compareFileNames(one: string, other: string, caseSensitive = false): number {
if (intlFileNameCollator) {
const a = one || '';
const b = other || '';
const result = intlFileNameCollator.compare(a, b);
// Using the numeric option in the collator will
// make compare(`foo1`, `foo01`) === 0. We must disambiguate.
if (intlFileNameCollatorIsNumeric && result === 0 && a !== b) {
return a < b ? -1 : 1;
}
return result;
}
return noIntlCompareFileNames(one, other, caseSensitive);
}
const FileNameMatch = /^(.*?)(\.([^.]*))?$/;
export function noIntlCompareFileNames(one: string, other: string, caseSensitive = false): number {
if (!caseSensitive) {
one = one && one.toLowerCase();
other = other && other.toLowerCase();
}
const [oneName, oneExtension] = extractNameAndExtension(one);
const [otherName, otherExtension] = extractNameAndExtension(other);
if (oneName !== otherName) {
return oneName < otherName ? -1 : 1;
}
if (oneExtension === otherExtension) {
return 0;
}
return oneExtension < otherExtension ? -1 : 1;
}
function extractNameAndExtension(str?: string): [string, string] {
const match = str ? FileNameMatch.exec(str) : [] as RegExpExecArray;
return [(match && match[1]) || '', (match && match[3]) || ''];
}
export function compareAnything(one: string, other: string, lookFor: string): number {
let elementAName = one.toLowerCase();
let elementBName = other.toLowerCase();
// Sort prefix matches over non prefix matches
const prefixCompare = compareByPrefix(one, other, lookFor);
if (prefixCompare) {
return prefixCompare;
}
// Sort suffix matches over non suffix matches
let elementASuffixMatch = strings.endsWith(elementAName, lookFor);
let elementBSuffixMatch = strings.endsWith(elementBName, lookFor);
if (elementASuffixMatch !== elementBSuffixMatch) {
return elementASuffixMatch ? -1 : 1;
}
// Understand file names
let r = compareFileNames(elementAName, elementBName);
if (r !== 0) {
return r;
}
// Compare by name
return elementAName.localeCompare(elementBName);
}
export function compareByPrefix(one: string, other: string, lookFor: string): number {
let elementAName = one.toLowerCase();
let elementBName = other.toLowerCase();
// Sort prefix matches over non prefix matches
let elementAPrefixMatch = strings.startsWith(elementAName, lookFor);
let elementBPrefixMatch = strings.startsWith(elementBName, lookFor);
if (elementAPrefixMatch !== elementBPrefixMatch) {
return elementAPrefixMatch ? -1 : 1;
}
// Same prefix: Sort shorter matches to the top to have those on top that match more precisely
else if (elementAPrefixMatch && elementBPrefixMatch) {
if (elementAName.length < elementBName.length) {
return -1;
}
if (elementAName.length > elementBName.length) {
return 1;
}
}
return 0;
}

619
extensions/search-rg/src/common/fileSearchScorer.ts
View file

@ -1,619 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { stripWildcards, equalsIgnoreCase } from './strings';
import { matchesPrefix, matchesCamelCase, createMatches, IMatch, isUpper } from './filters';
import { compareAnything } from './comparers';
import { CharCode } from './charCode';
const isWindows = process.platform === 'win32';
const isMacintosh = (process.platform === 'darwin');
const isLinux = (process.platform === 'linux');
const nativeSep = isWindows ? '\\' : '/';
export type Score = [number /* score */, number[] /* match positions */];
export type ScorerCache = { [key: string]: IItemScore };
const NO_MATCH = 0;
const NO_SCORE: Score = [NO_MATCH, []];
// const DEBUG = false;
// const DEBUG_MATRIX = false;
export function score(target: string, query: string, queryLower: string, fuzzy: boolean): Score {
if (!target || !query) {
return NO_SCORE; // return early if target or query are undefined
}
const targetLength = target.length;
const queryLength = query.length;
if (targetLength < queryLength) {
return NO_SCORE; // impossible for query to be contained in target
}
// if (DEBUG) {
// console.group(`Target: ${target}, Query: ${query}`);
// }
const targetLower = target.toLowerCase();
// When not searching fuzzy, we require the query to be contained fully
// in the target string contiguously.
if (!fuzzy) {
const indexOfQueryInTarget = targetLower.indexOf(queryLower);
if (indexOfQueryInTarget === -1) {
// if (DEBUG) {
// console.log(`Characters not matching consecutively ${queryLower} within ${targetLower}`);
// }
return NO_SCORE;
}
}
const res = doScore(query, queryLower, queryLength, target, targetLower, targetLength);
// if (DEBUG) {
// console.log(`%cFinal Score: ${res[0]}`, 'font-weight: bold');
// console.groupEnd();
// }
return res;
}
function doScore(query: string, queryLower: string, queryLength: number, target: string, targetLower: string, targetLength: number): [number, number[]] {
const scores = [];
const matches = [];
//
// Build Scorer Matrix:
//
// The matrix is composed of query q and target t. For each index we score
// q[i] with t[i] and compare that with the previous score. If the score is
// equal or larger, we keep the match. In addition to the score, we also keep
// the length of the consecutive matches to use as boost for the score.
//
// t a r g e t
// q
// u
// e
// r
// y
//
for (let queryIndex = 0; queryIndex < queryLength; queryIndex++) {
for (let targetIndex = 0; targetIndex < targetLength; targetIndex++) {
const currentIndex = queryIndex * targetLength + targetIndex;
const leftIndex = currentIndex - 1;
const diagIndex = (queryIndex - 1) * targetLength + targetIndex - 1;
const leftScore: number = targetIndex > 0 ? scores[leftIndex] : 0;
const diagScore: number = queryIndex > 0 && targetIndex > 0 ? scores[diagIndex] : 0;
const matchesSequenceLength: number = queryIndex > 0 && targetIndex > 0 ? matches[diagIndex] : 0;
// If we are not matching on the first query character any more, we only produce a
// score if we had a score previously for the last query index (by looking at the diagScore).
// This makes sure that the query always matches in sequence on the target. For example
// given a target of "ede" and a query of "de", we would otherwise produce a wrong high score
// for query[1] ("e") matching on target[0] ("e") because of the "beginning of word" boost.
let score: number;
if (!diagScore && queryIndex > 0) {
score = 0;
} else {
score = computeCharScore(query, queryLower, queryIndex, target, targetLower, targetIndex, matchesSequenceLength);
}
// We have a score and its equal or larger than the left score
// Match: sequence continues growing from previous diag value
// Score: increases by diag score value
if (score && diagScore + score >= leftScore) {
matches[currentIndex] = matchesSequenceLength + 1;
scores[currentIndex] = diagScore + score;
}
// We either have no score or the score is lower than the left score
// Match: reset to 0
// Score: pick up from left hand side
else {
matches[currentIndex] = NO_MATCH;
scores[currentIndex] = leftScore;
}
}
}
// Restore Positions (starting from bottom right of matrix)
const positions = [];
let queryIndex = queryLength - 1;
let targetIndex = targetLength - 1;
while (queryIndex >= 0 && targetIndex >= 0) {
const currentIndex = queryIndex * targetLength + targetIndex;
const match = matches[currentIndex];
if (match === NO_MATCH) {
targetIndex--; // go left
} else {
positions.push(targetIndex);
// go up and left
queryIndex--;
targetIndex--;
}
}
// Print matrix
// if (DEBUG_MATRIX) {
// printMatrix(query, target, matches, scores);
// }
return [scores[queryLength * targetLength - 1], positions.reverse()];
}
function computeCharScore(query: string, queryLower: string, queryIndex: number, target: string, targetLower: string, targetIndex: number, matchesSequenceLength: number): number {
let score = 0;
if (queryLower[queryIndex] !== targetLower[targetIndex]) {
return score; // no match of characters
}
// Character match bonus
score += 1;
// if (DEBUG) {
// console.groupCollapsed(`%cCharacter match bonus: +1 (char: ${queryLower[queryIndex]} at index ${targetIndex}, total score: ${score})`, 'font-weight: normal');
// }
// Consecutive match bonus
if (matchesSequenceLength > 0) {
score += (matchesSequenceLength * 5);
// if (DEBUG) {
// console.log('Consecutive match bonus: ' + (matchesSequenceLength * 5));
// }
}
// Same case bonus
if (query[queryIndex] === target[targetIndex]) {
score += 1;
// if (DEBUG) {
// console.log('Same case bonus: +1');
// }
}
// Start of word bonus
if (targetIndex === 0) {
score += 8;
// if (DEBUG) {
// console.log('Start of word bonus: +8');
// }
}
else {
// After separator bonus
const separatorBonus = scoreSeparatorAtPos(target.charCodeAt(targetIndex - 1));
if (separatorBonus) {
score += separatorBonus;
// if (DEBUG) {
// console.log('After separtor bonus: +4');
// }
}
// Inside word upper case bonus (camel case)
else if (isUpper(target.charCodeAt(targetIndex))) {
score += 1;
// if (DEBUG) {
// console.log('Inside word upper case bonus: +1');
// }
}
}
// if (DEBUG) {
// console.groupEnd();
// }
return score;
}
function scoreSeparatorAtPos(charCode: number): number {
switch (charCode) {
case CharCode.Slash:
case CharCode.Backslash:
return 5; // prefer path separators...
case CharCode.Underline:
case CharCode.Dash:
case CharCode.Period:
case CharCode.Space:
case CharCode.SingleQuote:
case CharCode.DoubleQuote:
case CharCode.Colon:
return 4; // ...over other separators
default:
return 0;
}
}
// function printMatrix(query: string, target: string, matches: number[], scores: number[]): void {
// console.log('\t' + target.split('').join('\t'));
// for (let queryIndex = 0; queryIndex < query.length; queryIndex++) {
// let line = query[queryIndex] + '\t';
// for (let targetIndex = 0; targetIndex < target.length; targetIndex++) {
// const currentIndex = queryIndex * target.length + targetIndex;
// line = line + 'M' + matches[currentIndex] + '/' + 'S' + scores[currentIndex] + '\t';
// }
// console.log(line);
// }
// }
/**
* Scoring on structural items that have a label and optional description.
*/
export interface IItemScore {
/**
* Overall score.
*/
score: number;
/**
* Matches within the label.
*/
labelMatch?: IMatch[];
/**
* Matches within the description.
*/
descriptionMatch?: IMatch[];
}
const NO_ITEM_SCORE: IItemScore = Object.freeze({ score: 0 });
export interface IItemAccessor<T> {
/**
* Just the label of the item to score on.
*/
getItemLabel(item: T): string;
/**
* The optional description of the item to score on. Can be null.
*/
getItemDescription(item: T): string;
/**
* If the item is a file, the path of the file to score on. Can be null.
*/
getItemPath(file: T): string;
}
const PATH_IDENTITY_SCORE = 1 << 18;
const LABEL_PREFIX_SCORE = 1 << 17;
const LABEL_CAMELCASE_SCORE = 1 << 16;
const LABEL_SCORE_THRESHOLD = 1 << 15;
export interface IPreparedQuery {
original: string;
value: string;
lowercase: string;
containsPathSeparator: boolean;
}
/**
* Helper function to prepare a search value for scoring in quick open by removing unwanted characters.
*/
export function prepareQuery(original: string): IPreparedQuery {
let lowercase: string;
let containsPathSeparator: boolean;
let value: string;
if (original) {
value = stripWildcards(original).replace(/\s/g, ''); // get rid of all wildcards and whitespace
if (isWindows) {
value = value.replace(/\//g, nativeSep); // Help Windows users to search for paths when using slash
}
lowercase = value.toLowerCase();
containsPathSeparator = value.indexOf(nativeSep) >= 0;
}
return { original, value, lowercase, containsPathSeparator };
}
export function scoreItem<T>(item: T, query: IPreparedQuery, fuzzy: boolean, accessor: IItemAccessor<T>, cache: ScorerCache): IItemScore {
if (!item || !query.value) {
return NO_ITEM_SCORE; // we need an item and query to score on at least
}
const label = accessor.getItemLabel(item);
if (!label) {
return NO_ITEM_SCORE; // we need a label at least
}
const description = accessor.getItemDescription(item);
let cacheHash: string;
if (description) {
cacheHash = `${label}${description}${query.value}${fuzzy}`;
} else {
cacheHash = `${label}${query.value}${fuzzy}`;
}
const cached = cache[cacheHash];
if (cached) {
return cached;
}
const itemScore = doScoreItem(label, description, accessor.getItemPath(item), query, fuzzy);
cache[cacheHash] = itemScore;
return itemScore;
}
function doScoreItem(label: string, description: string, path: string, query: IPreparedQuery, fuzzy: boolean): IItemScore {
// 1.) treat identity matches on full path highest
if (path && isLinux ? query.original === path : equalsIgnoreCase(query.original, path)) {
return { score: PATH_IDENTITY_SCORE, labelMatch: [{ start: 0, end: label.length }], descriptionMatch: description ? [{ start: 0, end: description.length }] : void 0 };
}
// We only consider label matches if the query is not including file path separators
const preferLabelMatches = !path || !query.containsPathSeparator;
if (preferLabelMatches) {
// 2.) treat prefix matches on the label second highest
const prefixLabelMatch = matchesPrefix(query.value, label);
if (prefixLabelMatch) {
return { score: LABEL_PREFIX_SCORE, labelMatch: prefixLabelMatch };
}
// 3.) treat camelcase matches on the label third highest
const camelcaseLabelMatch = matchesCamelCase(query.value, label);
if (camelcaseLabelMatch) {
return { score: LABEL_CAMELCASE_SCORE, labelMatch: camelcaseLabelMatch };
}
// 4.) prefer scores on the label if any
const [labelScore, labelPositions] = score(label, query.value, query.lowercase, fuzzy);
if (labelScore) {
return { score: labelScore + LABEL_SCORE_THRESHOLD, labelMatch: createMatches(labelPositions) };
}
}
// 5.) finally compute description + label scores if we have a description
if (description) {
let descriptionPrefix = description;
if (!!path) {
descriptionPrefix = `${description}${nativeSep}`; // assume this is a file path
}
const descriptionPrefixLength = descriptionPrefix.length;
const descriptionAndLabel = `${descriptionPrefix}${label}`;
const [labelDescriptionScore, labelDescriptionPositions] = score(descriptionAndLabel, query.value, query.lowercase, fuzzy);
if (labelDescriptionScore) {
const labelDescriptionMatches = createMatches(labelDescriptionPositions);
const labelMatch: IMatch[] = [];
const descriptionMatch: IMatch[] = [];
// We have to split the matches back onto the label and description portions
labelDescriptionMatches.forEach(h => {
// Match overlaps label and description part, we need to split it up
if (h.start < descriptionPrefixLength && h.end > descriptionPrefixLength) {
labelMatch.push({ start: 0, end: h.end - descriptionPrefixLength });
descriptionMatch.push({ start: h.start, end: descriptionPrefixLength });
}
// Match on label part
else if (h.start >= descriptionPrefixLength) {
labelMatch.push({ start: h.start - descriptionPrefixLength, end: h.end - descriptionPrefixLength });
}
// Match on description part
else {
descriptionMatch.push(h);
}
});
return { score: labelDescriptionScore, labelMatch, descriptionMatch };
}
}
return NO_ITEM_SCORE;
}
export function compareItemsByScore<T>(itemA: T, itemB: T, query: IPreparedQuery, fuzzy: boolean, accessor: IItemAccessor<T>, cache: ScorerCache, fallbackComparer = fallbackCompare): number {
const itemScoreA = scoreItem(itemA, query, fuzzy, accessor, cache);
const itemScoreB = scoreItem(itemB, query, fuzzy, accessor, cache);
const scoreA = itemScoreA.score;
const scoreB = itemScoreB.score;
// 1.) prefer identity matches
if (scoreA === PATH_IDENTITY_SCORE || scoreB === PATH_IDENTITY_SCORE) {
if (scoreA !== scoreB) {
return scoreA === PATH_IDENTITY_SCORE ? -1 : 1;
}
}
// 2.) prefer label prefix matches
if (scoreA === LABEL_PREFIX_SCORE || scoreB === LABEL_PREFIX_SCORE) {
if (scoreA !== scoreB) {
return scoreA === LABEL_PREFIX_SCORE ? -1 : 1;
}
const labelA = accessor.getItemLabel(itemA);
const labelB = accessor.getItemLabel(itemB);
// prefer shorter names when both match on label prefix
if (labelA.length !== labelB.length) {
return labelA.length - labelB.length;
}
}
// 3.) prefer camelcase matches
if (scoreA === LABEL_CAMELCASE_SCORE || scoreB === LABEL_CAMELCASE_SCORE) {
if (scoreA !== scoreB) {
return scoreA === LABEL_CAMELCASE_SCORE ? -1 : 1;
}
const labelA = accessor.getItemLabel(itemA);
const labelB = accessor.getItemLabel(itemB);
// prefer more compact camel case matches over longer
const comparedByMatchLength = compareByMatchLength(itemScoreA.labelMatch, itemScoreB.labelMatch);
if (comparedByMatchLength !== 0) {
return comparedByMatchLength;
}
// prefer shorter names when both match on label camelcase
if (labelA.length !== labelB.length) {
return labelA.length - labelB.length;
}
}
// 4.) prefer label scores
if (scoreA > LABEL_SCORE_THRESHOLD || scoreB > LABEL_SCORE_THRESHOLD) {
if (scoreB < LABEL_SCORE_THRESHOLD) {
return -1;
}
if (scoreA < LABEL_SCORE_THRESHOLD) {
return 1;
}
}
// 5.) compare by score
if (scoreA !== scoreB) {
return scoreA > scoreB ? -1 : 1;
}
// 6.) scores are identical, prefer more compact matches (label and description)
const itemAMatchDistance = computeLabelAndDescriptionMatchDistance(itemA, itemScoreA, accessor);
const itemBMatchDistance = computeLabelAndDescriptionMatchDistance(itemB, itemScoreB, accessor);
if (itemAMatchDistance && itemBMatchDistance && itemAMatchDistance !== itemBMatchDistance) {
return itemBMatchDistance > itemAMatchDistance ? -1 : 1;
}
// 7.) at this point, scores are identical and match compactness as well
// for both items so we start to use the fallback compare
return fallbackComparer(itemA, itemB, query, accessor);
}
function computeLabelAndDescriptionMatchDistance<T>(item: T, score: IItemScore, accessor: IItemAccessor<T>): number {
const hasLabelMatches = (score.labelMatch && score.labelMatch.length);
const hasDescriptionMatches = (score.descriptionMatch && score.descriptionMatch.length);
let matchStart: number = -1;
let matchEnd: number = -1;
// If we have description matches, the start is first of description match
if (hasDescriptionMatches) {
matchStart = score.descriptionMatch[0].start;
}
// Otherwise, the start is the first label match
else if (hasLabelMatches) {
matchStart = score.labelMatch[0].start;
}
// If we have label match, the end is the last label match
// If we had a description match, we add the length of the description
// as offset to the end to indicate this.
if (hasLabelMatches) {
matchEnd = score.labelMatch[score.labelMatch.length - 1].end;
if (hasDescriptionMatches) {
const itemDescription = accessor.getItemDescription(item);
if (itemDescription) {
matchEnd += itemDescription.length;
}
}
}
// If we have just a description match, the end is the last description match
else if (hasDescriptionMatches) {
matchEnd = score.descriptionMatch[score.descriptionMatch.length - 1].end;
}
return matchEnd - matchStart;
}
function compareByMatchLength(matchesA?: IMatch[], matchesB?: IMatch[]): number {
if ((!matchesA && !matchesB) || (!matchesA.length && !matchesB.length)) {
return 0; // make sure to not cause bad comparing when matches are not provided
}
if (!matchesB || !matchesB.length) {
return -1;
}
if (!matchesA || !matchesA.length) {
return 1;
}
// Compute match length of A (first to last match)
const matchStartA = matchesA[0].start;
const matchEndA = matchesA[matchesA.length - 1].end;
const matchLengthA = matchEndA - matchStartA;
// Compute match length of B (first to last match)
const matchStartB = matchesB[0].start;
const matchEndB = matchesB[matchesB.length - 1].end;
const matchLengthB = matchEndB - matchStartB;
// Prefer shorter match length
return matchLengthA === matchLengthB ? 0 : matchLengthB < matchLengthA ? 1 : -1;
}
export function fallbackCompare<T>(itemA: T, itemB: T, query: IPreparedQuery, accessor: IItemAccessor<T>): number {
// check for label + description length and prefer shorter
const labelA = accessor.getItemLabel(itemA);
const labelB = accessor.getItemLabel(itemB);
const descriptionA = accessor.getItemDescription(itemA);
const descriptionB = accessor.getItemDescription(itemB);
const labelDescriptionALength = labelA.length + (descriptionA ? descriptionA.length : 0);
const labelDescriptionBLength = labelB.length + (descriptionB ? descriptionB.length : 0);
if (labelDescriptionALength !== labelDescriptionBLength) {
return labelDescriptionALength - labelDescriptionBLength;
}
// check for path length and prefer shorter
const pathA = accessor.getItemPath(itemA);
const pathB = accessor.getItemPath(itemB);
if (pathA && pathB && pathA.length !== pathB.length) {
return pathA.length - pathB.length;
}
// 7.) finally we have equal scores and equal length, we fallback to comparer
// compare by label
if (labelA !== labelB) {
return compareAnything(labelA, labelB, query.value);
}
// compare by description
if (descriptionA && descriptionB && descriptionA !== descriptionB) {
return compareAnything(descriptionA, descriptionB, query.value);
}
// compare by path
if (pathA && pathB && pathA !== pathB) {
return compareAnything(pathA, pathB, query.value);
}
// equal
return 0;
}

224
extensions/search-rg/src/common/filters.ts
View file

@ -1,224 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as strings from './strings';
import { CharCode } from './charCode';
export interface IFilter {
// Returns null if word doesn't match.
(word: string, wordToMatchAgainst: string): IMatch[];
}
export interface IMatch {
start: number;
end: number;
}
// Prefix
export const matchesPrefix: IFilter = _matchesPrefix.bind(undefined, true);
function _matchesPrefix(ignoreCase: boolean, word: string, wordToMatchAgainst: string): IMatch[] {
if (!wordToMatchAgainst || wordToMatchAgainst.length < word.length) {
return null;
}
let matches: boolean;
if (ignoreCase) {
matches = strings.startsWithIgnoreCase(wordToMatchAgainst, word);
} else {
matches = wordToMatchAgainst.indexOf(word) === 0;
}
if (!matches) {
return null;
}
return word.length > 0 ? [{ start: 0, end: word.length }] : [];
}
// CamelCase
function isLower(code: number): boolean {
return CharCode.a <= code && code <= CharCode.z;
}
export function isUpper(code: number): boolean {
return CharCode.A <= code && code <= CharCode.Z;
}
function isNumber(code: number): boolean {
return CharCode.Digit0 <= code && code <= CharCode.Digit9;
}
function isWhitespace(code: number): boolean {
return (
code === CharCode.Space
|| code === CharCode.Tab
|| code === CharCode.LineFeed
|| code === CharCode.CarriageReturn
);
}
function isAlphanumeric(code: number): boolean {
return isLower(code) || isUpper(code) || isNumber(code);
}
function join(head: IMatch, tail: IMatch[]): IMatch[] {
if (tail.length === 0) {
tail = [head];
} else if (head.end === tail[0].start) {
tail[0].start = head.start;
} else {
tail.unshift(head);
}
return tail;
}
function nextAnchor(camelCaseWord: string, start: number): number {
for (let i = start; i < camelCaseWord.length; i++) {
let c = camelCaseWord.charCodeAt(i);
if (isUpper(c) || isNumber(c) || (i > 0 && !isAlphanumeric(camelCaseWord.charCodeAt(i - 1)))) {
return i;
}
}
return camelCaseWord.length;
}
function _matchesCamelCase(word: string, camelCaseWord: string, i: number, j: number): IMatch[] {
if (i === word.length) {
return [];
} else if (j === camelCaseWord.length) {
return null;
} else if (word[i] !== camelCaseWord[j].toLowerCase()) {
return null;
} else {
let result: IMatch[] = null;
let nextUpperIndex = j + 1;
result = _matchesCamelCase(word, camelCaseWord, i + 1, j + 1);
while (!result && (nextUpperIndex = nextAnchor(camelCaseWord, nextUpperIndex)) < camelCaseWord.length) {
result = _matchesCamelCase(word, camelCaseWord, i + 1, nextUpperIndex);
nextUpperIndex++;
}
return result === null ? null : join({ start: j, end: j + 1 }, result);
}
}
interface ICamelCaseAnalysis {
upperPercent: number;
lowerPercent: number;
alphaPercent: number;
numericPercent: number;
}
// Heuristic to avoid computing camel case matcher for words that don't
// look like camelCaseWords.
function analyzeCamelCaseWord(word: string): ICamelCaseAnalysis {
let upper = 0, lower = 0, alpha = 0, numeric = 0, code = 0;
for (let i = 0; i < word.length; i++) {
code = word.charCodeAt(i);
if (isUpper(code)) { upper++; }
if (isLower(code)) { lower++; }
if (isAlphanumeric(code)) { alpha++; }
if (isNumber(code)) { numeric++; }
}
let upperPercent = upper / word.length;
let lowerPercent = lower / word.length;
let alphaPercent = alpha / word.length;
let numericPercent = numeric / word.length;
return { upperPercent, lowerPercent, alphaPercent, numericPercent };
}
function isUpperCaseWord(analysis: ICamelCaseAnalysis): boolean {
const { upperPercent, lowerPercent } = analysis;
return lowerPercent === 0 && upperPercent > 0.6;
}
function isCamelCaseWord(analysis: ICamelCaseAnalysis): boolean {
const { upperPercent, lowerPercent, alphaPercent, numericPercent } = analysis;
return lowerPercent > 0.2 && upperPercent < 0.8 && alphaPercent > 0.6 && numericPercent < 0.2;
}
// Heuristic to avoid computing camel case matcher for words that don't
// look like camel case patterns.
function isCamelCasePattern(word: string): boolean {
let upper = 0, lower = 0, code = 0, whitespace = 0;
for (let i = 0; i < word.length; i++) {
code = word.charCodeAt(i);
if (isUpper(code)) { upper++; }
if (isLower(code)) { lower++; }
if (isWhitespace(code)) { whitespace++; }
}
if ((upper === 0 || lower === 0) && whitespace === 0) {
return word.length <= 30;
} else {
return upper <= 5;
}
}
export function matchesCamelCase(word: string, camelCaseWord: string): IMatch[] {
if (!camelCaseWord) {
return null;
}
camelCaseWord = camelCaseWord.trim();
if (camelCaseWord.length === 0) {
return null;
}
if (!isCamelCasePattern(word)) {
return null;
}
if (camelCaseWord.length > 60) {
return null;
}
const analysis = analyzeCamelCaseWord(camelCaseWord);
if (!isCamelCaseWord(analysis)) {
if (!isUpperCaseWord(analysis)) {
return null;
}
camelCaseWord = camelCaseWord.toLowerCase();
}
let result: IMatch[] = null;
let i = 0;
word = word.toLowerCase();
while (i < camelCaseWord.length && (result = _matchesCamelCase(word, camelCaseWord, 0, i)) === null) {
i = nextAnchor(camelCaseWord, i + 1);
}
return result;
}
export function createMatches(position: number[]): IMatch[] {
let ret: IMatch[] = [];
if (!position) {
return ret;
}
let last: IMatch;
for (const pos of position) {
if (last && last.end === pos) {
last.end += 1;
} else {
last = { start: pos, end: pos + 1 };
ret.push(last);
}
}
return ret;
}

143
extensions/search-rg/src/common/strings.ts
View file

@ -1,143 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { CharCode } from './charCode';
export function stripWildcards(pattern: string): string {
return pattern.replace(/\*/g, '');
}
/**
* Determines if haystack starts with needle.
*/
export function startsWith(haystack: string, needle: string): boolean {
if (haystack.length < needle.length) {
return false;
}
if (haystack === needle) {
return true;
}
for (let i = 0; i < needle.length; i++) {
if (haystack[i] !== needle[i]) {
return false;
}
}
return true;
}
export function startsWithIgnoreCase(str: string, candidate: string): boolean {
const candidateLength = candidate.length;
if (candidate.length > str.length) {
return false;
}
return doEqualsIgnoreCase(str, candidate, candidateLength);
}
/**
* Determines if haystack ends with needle.
*/
export function endsWith(haystack: string, needle: string): boolean {
let diff = haystack.length - needle.length;
if (diff > 0) {
return haystack.indexOf(needle, diff) === diff;
} else if (diff === 0) {
return haystack === needle;
} else {
return false;
}
}
function isLowerAsciiLetter(code: number): boolean {
return code >= CharCode.a && code <= CharCode.z;
}
function isUpperAsciiLetter(code: number): boolean {
return code >= CharCode.A && code <= CharCode.Z;
}
function isAsciiLetter(code: number): boolean {
return isLowerAsciiLetter(code) || isUpperAsciiLetter(code);
}
export function equalsIgnoreCase(a: string, b: string): boolean {
const len1 = a ? a.length : 0;
const len2 = b ? b.length : 0;
if (len1 !== len2) {
return false;
}
return doEqualsIgnoreCase(a, b);
}
function doEqualsIgnoreCase(a: string, b: string, stopAt = a.length): boolean {
if (typeof a !== 'string' || typeof b !== 'string') {
return false;
}
for (let i = 0; i < stopAt; i++) {
const codeA = a.charCodeAt(i);
const codeB = b.charCodeAt(i);
if (codeA === codeB) {
continue;
}
// a-z A-Z
if (isAsciiLetter(codeA) && isAsciiLetter(codeB)) {
let diff = Math.abs(codeA - codeB);
if (diff !== 0 && diff !== 32) {
return false;
}
}
// Any other charcode
else {
if (String.fromCharCode(codeA).toLowerCase() !== String.fromCharCode(codeB).toLowerCase()) {
return false;
}
}
}
return true;
}
/**
* Checks if the characters of the provided query string are included in the
* target string. The characters do not have to be contiguous within the string.
*/
export function fuzzyContains(target: string, query: string): boolean {
if (!target || !query) {
return false; // return early if target or query are undefined
}
if (target.length < query.length) {
return false; // impossible for query to be contained in target
}
const queryLen = query.length;
const targetLower = target.toLowerCase();
let index = 0;
let lastIndexOf = -1;
while (index < queryLen) {
let indexOf = targetLower.indexOf(query[index], lastIndexOf + 1);
if (indexOf < 0) {
return false;
}
lastIndexOf = indexOf;
index++;
}
return true;
}

24
extensions/search-rg/src/extension.ts
View file

@ -4,27 +4,26 @@
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { RipgrepTextSearchEngine } from './ripgrepTextSearch';
import { RipgrepFileSearchEngine } from './ripgrepFileSearch';
import { CachedSearchProvider } from './cachedSearchProvider';
import { RipgrepTextSearchEngine } from './ripgrepTextSearch';
import { joinPath } from './utils';
export function activate(): void {
if (vscode.workspace.getConfiguration('searchRipgrep').get('enable')) {
const outputChannel = vscode.window.createOutputChannel('search-rg');
const provider = new RipgrepSearchProvider(outputChannel);
vscode.workspace.registerSearchProvider('file', provider);
vscode.workspace.registerFileIndexProvider('file', provider);
vscode.workspace.registerTextSearchProvider('file', provider);
}
}
type SearchEngine = RipgrepFileSearchEngine | RipgrepTextSearchEngine;
class RipgrepSearchProvider implements vscode.SearchProvider, vscode.TextSearchProvider {
private cachedProvider: CachedSearchProvider;
class RipgrepSearchProvider implements vscode.FileIndexProvider, vscode.TextSearchProvider {
private inProgress: Set<SearchEngine> = new Set();
constructor(private outputChannel: vscode.OutputChannel) {
this.cachedProvider = new CachedSearchProvider();
process.once('exit', () => this.dispose());
}
@ -33,13 +32,16 @@ class RipgrepSearchProvider implements vscode.SearchProvider, vscode.TextSearchP
return this.withEngine(engine, () => engine.provideTextSearchResults(query, options, progress, token));
}
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.SearchOptions, progress: vscode.Progress<vscode.Uri>, token: vscode.CancellationToken): Thenable<void> {
provideFileIndex(options: vscode.FileSearchOptions, token: vscode.CancellationToken): Thenable<vscode.Uri[]> {
const engine = new RipgrepFileSearchEngine(this.outputChannel);
return this.withEngine(engine, () => this.cachedProvider.provideFileSearchResults(engine, query, options, progress, token));
}
clearCache(cacheKey: string): void {
this.cachedProvider.clearCache(cacheKey);
const results: vscode.Uri[] = [];
const onResult = relativePathMatch => {
results.push(joinPath(options.folder, relativePathMatch));
};
return this.withEngine(engine, () => engine.provideFileSearchResults(options, { report: onResult }, token))
.then(() => results);
}
private withEngine(engine: SearchEngine, fn: () => Thenable<void>): Thenable<void> {

0
extensions/search-rg/src/common/normalization.ts → extensions/search-rg/src/normalization.ts
View file

7
extensions/search-rg/src/ripgrepFileSearch.ts
View file

@ -7,18 +7,17 @@ import * as cp from 'child_process';
import { Readable } from 'stream';
import { NodeStringDecoder, StringDecoder } from 'string_decoder';
import * as vscode from 'vscode';
import { normalizeNFC, normalizeNFD } from './common/normalization';
import { normalizeNFC, normalizeNFD } from './normalization';
import { rgPath } from './ripgrep';
import { anchorGlob } from './utils';
import { rgErrorMsgForDisplay } from './ripgrepTextSearch';
import { IInternalFileSearchProvider } from './cachedSearchProvider';
import { anchorGlob } from './utils';
const isMac = process.platform === 'darwin';
// If vscode-ripgrep is in an .asar file, then the binary is unpacked.
const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
export class RipgrepFileSearchEngine implements IInternalFileSearchProvider {
export class RipgrepFileSearchEngine {
private rgProc: cp.ChildProcess;
private isDone: boolean;

11
src/vs/platform/search/common/search.ts
View file

@ -27,7 +27,7 @@ export interface ISearchService {
search(query: ISearchQuery, onProgress?: (result: ISearchProgressItem) => void): TPromise<ISearchComplete>;
extendQuery(query: ISearchQuery): void;
clearCache(cacheKey: string): TPromise<void>;
registerSearchResultProvider(scheme: string, provider: ISearchResultProvider): IDisposable;
registerSearchResultProvider(scheme: string, type: SearchProviderType, provider: ISearchResultProvider): IDisposable;
}
export interface ISearchHistoryValues {
@ -45,6 +45,15 @@ export interface ISearchHistoryService {
save(history: ISearchHistoryValues): void;
}
/**
* TODO@roblou - split text from file search entirely, or share code in a more natural way.
*/
export enum SearchProviderType {
file,
fileIndex,
text
}
export interface ISearchResultProvider {
search(query: ISearchQuery, onProgress?: (p: ISearchProgressItem) => void): TPromise<ISearchComplete>;
clearCache(cacheKey: string): TPromise<void>;

25
src/vs/vscode.proposed.d.ts vendored
View file

@ -160,15 +160,11 @@ declare module 'vscode' {
preview: TextSearchResultPreview;
}
// interface FileIndexProvider {
// provideFileIndex(options: FileSearchOptions, token: CancellationToken): Thenable<Uri[]>
// }
export interface FileIndexProvider {
provideFileIndex(options: FileSearchOptions, token: CancellationToken): Thenable<Uri[]>;
}
// interface FileSearchProvider {
// provideFileSearchResults(query: FileSear, options, token): Thenable<Uri[]>
// }
interface TextSearchProvider {
export interface TextSearchProvider {
/**
* Provide results that match the given text pattern.
* @param query The parameters for this query.
@ -251,7 +247,7 @@ declare module 'vscode' {
* @param provider The provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/
export function registerSearchProvider(scheme: string, provider: SearchProvider): Disposable;
export function registerFileSearchProvider(scheme: string, provider: SearchProvider): Disposable;
/**
* Register a text search provider.
@ -264,6 +260,17 @@ declare module 'vscode' {
*/
export function registerTextSearchProvider(scheme: string, provider: TextSearchProvider): Disposable;
/**
* Register a file index provider.
*
* Only one provider can be registered per scheme.
*
* @param scheme The provider will be invoked for workspace folders that have this file scheme.
* @param provider The provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/
export function registerFileIndexProvider(scheme: string, provider: FileIndexProvider): Disposable;
/**
* Search text in files across all [workspace folders](#workspace.workspaceFolders) in the workspace.

27
src/vs/workbench/api/electron-browser/mainThreadSearch.ts
View file

@ -9,7 +9,7 @@ import { dispose, IDisposable } from 'vs/base/common/lifecycle';
import { values } from 'vs/base/common/map';
import URI, { UriComponents } from 'vs/base/common/uri';
import { TPromise } from 'vs/base/common/winjs.base';
import { IFileMatch, IRawFileMatch2, ISearchComplete, ISearchCompleteStats, ISearchProgressItem, ISearchQuery, ISearchResultProvider, ISearchService, QueryType } from 'vs/platform/search/common/search';
import { IFileMatch, IRawFileMatch2, ISearchComplete, ISearchCompleteStats, ISearchProgressItem, ISearchQuery, ISearchResultProvider, ISearchService, QueryType, SearchProviderType } from 'vs/platform/search/common/search';
import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
import { extHostNamedCustomer } from 'vs/workbench/api/electron-browser/extHostCustomers';
import { ExtHostContext, ExtHostSearchShape, IExtHostContext, MainContext, MainThreadSearchShape } from '../node/extHost.protocol';
@ -33,8 +33,16 @@ export class MainThreadSearch implements MainThreadSearchShape {
this._searchProvider.clear();
}
$registerSearchProvider(handle: number, scheme: string): void {
this._searchProvider.set(handle, new RemoteSearchProvider(this._searchService, scheme, handle, this._proxy));
$registerTextSearchProvider(handle: number, scheme: string): void {
this._searchProvider.set(handle, new RemoteSearchProvider(this._searchService, SearchProviderType.text, scheme, handle, this._proxy));
}
$registerFileSearchProvider(handle: number, scheme: string): void {
this._searchProvider.set(handle, new RemoteSearchProvider(this._searchService, SearchProviderType.file, scheme, handle, this._proxy));
}
$registerFileIndexProvider(handle: number, scheme: string): void {
this._searchProvider.set(handle, new RemoteSearchProvider(this._searchService, SearchProviderType.fileIndex, scheme, handle, this._proxy));
}
$unregisterProvider(handle: number): void {
@ -86,11 +94,12 @@ class RemoteSearchProvider implements ISearchResultProvider, IDisposable {
constructor(
searchService: ISearchService,
type: SearchProviderType,
private readonly _scheme: string,
private readonly _handle: number,
private readonly _proxy: ExtHostSearchShape
) {
this._registrations = [searchService.registerSearchResultProvider(this._scheme, this)];
this._registrations = [searchService.registerSearchResultProvider(this._scheme, type, this)];
}
dispose(): void {
@ -103,16 +112,6 @@ class RemoteSearchProvider implements ISearchResultProvider, IDisposable {
return TPromise.as(undefined);
}
const folderQueriesForScheme = query.folderQueries.filter(fq => fq.folder.scheme === this._scheme);
if (!folderQueriesForScheme.length) {
return TPromise.wrap(null);
}
query = {
...query,
folderQueries: folderQueriesForScheme
};
let outer: TPromise;
return new TPromise((resolve, reject) => {

7
src/vs/workbench/api/node/extHost.api.impl.ts
View file

@ -582,12 +582,15 @@ export function createApiFactory(
registerFileSystemProvider(scheme, provider, options) {
return extHostFileSystem.registerFileSystemProvider(scheme, provider, options);
},
registerSearchProvider: proposedApiFunction(extension, (scheme, provider) => {
return extHostSearch.registerSearchProvider(scheme, provider);
registerFileSearchProvider: proposedApiFunction(extension, (scheme, provider) => {
return extHostSearch.registerFileSearchProvider(scheme, provider);
}),
registerTextSearchProvider: proposedApiFunction(extension, (scheme, provider) => {
return extHostSearch.registerTextSearchProvider(scheme, provider);
}),
registerFileIndexProvider: proposedApiFunction(extension, (scheme, provider) => {
return extHostSearch.registerFileIndexProvider(scheme, provider);
}),
registerDocumentCommentProvider: proposedApiFunction(extension, (provider: vscode.DocumentCommentProvider) => {
return exthostCommentProviders.registerDocumentCommentProvider(provider);
}),

4
src/vs/workbench/api/node/extHost.protocol.ts
View file

@ -486,7 +486,9 @@ export interface MainThreadFileSystemShape extends IDisposable {
}
export interface MainThreadSearchShape extends IDisposable {
$registerSearchProvider(handle: number, scheme: string): void;
$registerFileSearchProvider(handle: number, scheme: string): void;
$registerTextSearchProvider(handle: number, scheme: string): void;
$registerFileIndexProvider(handle: number, scheme: string): void;
$unregisterProvider(handle: number): void;
$handleFileMatch(handle: number, session: number, data: UriComponents[]): void;
$handleTextMatch(handle: number, session: number, data: IRawFileMatch2[]): void;

728
src/vs/workbench/api/node/extHostSearch.fileIndex.ts Normal file
View file

@ -0,0 +1,728 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as path from 'path';
import * as arrays from 'vs/base/common/arrays';
import { CancellationTokenSource } from 'vs/base/common/cancellation';
import { toErrorMessage } from 'vs/base/common/errorMessage';
import * as glob from 'vs/base/common/glob';
import * as resources from 'vs/base/common/resources';
import * as strings from 'vs/base/common/strings';
import URI from 'vs/base/common/uri';
import { PPromise, TPromise } from 'vs/base/common/winjs.base';
import { compareItemsByScore, IItemAccessor, prepareQuery, ScorerCache } from 'vs/base/parts/quickopen/common/quickOpenScorer';
import { ICachedSearchStats, IFileMatch, IFolderQuery, IRawSearchQuery, ISearchCompleteStats, ISearchQuery } from 'vs/platform/search/common/search';
import * as vscode from 'vscode';
export interface IInternalFileMatch {
base: URI;
relativePath?: string; // Not present for extraFiles or absolute path matches
basename: string;
size?: number;
}
/**
* Computes the patterns that the provider handles. Discards sibling clauses and 'false' patterns
*/
export function resolvePatternsForProvider(globalPattern: glob.IExpression, folderPattern: glob.IExpression): string[] {
const merged = {
...(globalPattern || {}),
...(folderPattern || {})
};
return Object.keys(merged)
.filter(key => {
const value = merged[key];
return typeof value === 'boolean' && value;
});
}
export class QueryGlobTester {
private _excludeExpression: glob.IExpression;
private _parsedExcludeExpression: glob.ParsedExpression;
private _parsedIncludeExpression: glob.ParsedExpression;
constructor(config: ISearchQuery, folderQuery: IFolderQuery) {
this._excludeExpression = {
...(config.excludePattern || {}),
...(folderQuery.excludePattern || {})
};
this._parsedExcludeExpression = glob.parse(this._excludeExpression);
// Empty includeExpression means include nothing, so no {} shortcuts
let includeExpression: glob.IExpression = config.includePattern;
if (folderQuery.includePattern) {
if (includeExpression) {
includeExpression = {
...includeExpression,
...folderQuery.includePattern
};
} else {
includeExpression = folderQuery.includePattern;
}
}
if (includeExpression) {
this._parsedIncludeExpression = glob.parse(includeExpression);
}
}
/**
* Guaranteed sync - siblingsFn should not return a promise.
*/
public includedInQuerySync(testPath: string, basename?: string, hasSibling?: (name: string) => boolean): boolean {
if (this._parsedExcludeExpression && this._parsedExcludeExpression(testPath, basename, hasSibling)) {
return false;
}
if (this._parsedIncludeExpression && !this._parsedIncludeExpression(testPath, basename, hasSibling)) {
return false;
}
return true;
}
/**
* Guaranteed async.
*/
public includedInQuery(testPath: string, basename?: string, hasSibling?: (name: string) => boolean | TPromise<boolean>): TPromise<boolean> {
const excludeP = this._parsedExcludeExpression ?
TPromise.as(this._parsedExcludeExpression(testPath, basename, hasSibling)).then(result => !!result) :
TPromise.wrap(false);
return excludeP.then(excluded => {
if (excluded) {
return false;
}
return this._parsedIncludeExpression ?
TPromise.as(this._parsedIncludeExpression(testPath, basename, hasSibling)).then(result => !!result) :
TPromise.wrap(true);
}).then(included => {
return included;
});
}
public hasSiblingExcludeClauses(): boolean {
return hasSiblingClauses(this._excludeExpression);
}
}
function hasSiblingClauses(pattern: glob.IExpression): boolean {
for (let key in pattern) {
if (typeof pattern[key] !== 'boolean') {
return true;
}
}
return false;
}
export interface IDirectoryEntry {
base: URI;
relativePath: string;
basename: string;
}
export interface IDirectoryTree {
rootEntries: IDirectoryEntry[];
pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}
export class FileIndexSearchEngine {
private filePattern: string;
private normalizedFilePatternLowercase: string;
private includePattern: glob.ParsedExpression;
private maxResults: number;
private exists: boolean;
// private maxFilesize: number;
private isLimitHit: boolean;
private resultCount: number;
private isCanceled: boolean;
private activeCancellationTokens: Set<CancellationTokenSource>;
// private filesWalked: number;
// private directoriesWalked: number;
private globalExcludePattern: glob.ParsedExpression;
constructor(private config: ISearchQuery, private provider: vscode.FileIndexProvider) {
this.filePattern = config.filePattern;
this.includePattern = config.includePattern && glob.parse(config.includePattern);
this.maxResults = config.maxResults || null;
this.exists = config.exists;
// this.maxFilesize = config.maxFileSize || null;
this.resultCount = 0;
this.isLimitHit = false;
this.activeCancellationTokens = new Set<CancellationTokenSource>();
// this.filesWalked = 0;
// this.directoriesWalked = 0;
if (this.filePattern) {
this.normalizedFilePatternLowercase = strings.stripWildcards(this.filePattern).toLowerCase();
}
this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);
}
public cancel(): void {
this.isCanceled = true;
this.activeCancellationTokens.forEach(t => t.cancel());
this.activeCancellationTokens = new Set();
}
public search(): PPromise<{ isLimitHit: boolean }, IInternalFileMatch> {
const folderQueries = this.config.folderQueries;
return new PPromise<{ isLimitHit: boolean }, IInternalFileMatch>((resolve, reject, _onResult) => {
const onResult = (match: IInternalFileMatch) => {
this.resultCount++;
_onResult(match);
};
if (this.isCanceled) {
return resolve({ isLimitHit: this.isLimitHit });
}
// For each extra file
if (this.config.extraFileResources) {
this.config.extraFileResources
.forEach(extraFile => {
const extraFileStr = extraFile.toString(); // ?
const basename = path.basename(extraFileStr);
if (this.globalExcludePattern && this.globalExcludePattern(extraFileStr, basename)) {
return; // excluded
}
// File: Check for match on file pattern and include pattern
this.matchFile(onResult, { base: extraFile, basename });
});
}
// For each root folder
PPromise.join(folderQueries.map(fq => {
return this.searchInFolder(fq).then(null, null, onResult);
})).then(() => {
resolve({ isLimitHit: this.isLimitHit });
}, (errs: Error[]) => {
const errMsg = errs
.map(err => toErrorMessage(err))
.filter(msg => !!msg)[0];
reject(new Error(errMsg));
});
});
}
private searchInFolder(fq: IFolderQuery<URI>): PPromise<void, IInternalFileMatch> {
let cancellation = new CancellationTokenSource();
return new PPromise((resolve, reject, onResult) => {
const options = this.getSearchOptionsForFolder(fq);
const tree = this.initDirectoryTree();
const queryTester = new QueryGlobTester(this.config, fq);
const noSiblingsClauses = !queryTester.hasSiblingExcludeClauses();
const onProviderResult = (uri: URI) => {
if (this.isCanceled) {
return;
}
// TODO@rob - ???
const relativePath = path.relative(fq.folder.path, uri.path);
if (noSiblingsClauses) {
const basename = path.basename(uri.path);
this.matchFile(onResult, { base: fq.folder, relativePath, basename });
return;
}
// TODO: Optimize siblings clauses with ripgrep here.
this.addDirectoryEntries(tree, fq.folder, relativePath, onResult);
};
new TPromise(resolve => process.nextTick(resolve))
.then(() => {
this.activeCancellationTokens.add(cancellation);
return this.provider.provideFileIndex(options, cancellation.token);
})
.then(results => {
this.activeCancellationTokens.delete(cancellation);
if (this.isCanceled) {
return null;
}
results.forEach(onProviderResult);
this.matchDirectoryTree(tree, queryTester, onResult);
return null;
}).then(
() => {
cancellation.dispose();
resolve(undefined);
},
err => {
cancellation.dispose();
reject(err);
});
});
}
private getSearchOptionsForFolder(fq: IFolderQuery<URI>): vscode.FileSearchOptions {
const includes = resolvePatternsForProvider(this.config.includePattern, fq.includePattern);
const excludes = resolvePatternsForProvider(this.config.excludePattern, fq.excludePattern);
return {
folder: fq.folder,
excludes,
includes,
useIgnoreFiles: !this.config.disregardIgnoreFiles,
followSymlinks: !this.config.ignoreSymlinks
};
}
private initDirectoryTree(): IDirectoryTree {
const tree: IDirectoryTree = {
rootEntries: [],
pathToEntries: Object.create(null)
};
tree.pathToEntries['.'] = tree.rootEntries;
return tree;
}
private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: URI, relativeFile: string, onResult: (result: IInternalFileMatch) => void) {
// Support relative paths to files from a root resource (ignores excludes)
if (relativeFile === this.filePattern) {
const basename = path.basename(this.filePattern);
this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
}
function add(relativePath: string) {
const basename = path.basename(relativePath);
const dirname = path.dirname(relativePath);
let entries = pathToEntries[dirname];
if (!entries) {
entries = pathToEntries[dirname] = [];
add(dirname);
}
entries.push({
base,
relativePath,
basename
});
}
add(relativeFile);
}
private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, queryTester: QueryGlobTester, onResult: (result: IInternalFileMatch) => void) {
const self = this;
const filePattern = this.filePattern;
function matchDirectory(entries: IDirectoryEntry[]) {
// self.directoriesWalked++;
for (let i = 0, n = entries.length; i < n; i++) {
const entry = entries[i];
const { relativePath, basename } = entry;
// Check exclude pattern
// If the user searches for the exact file name, we adjust the glob matching
// to ignore filtering by siblings because the user seems to know what she
// is searching for and we want to include the result in that case anyway
const hasSibling = glob.hasSiblingFn(() => entries.map(entry => entry.basename));
if (!queryTester.includedInQuerySync(relativePath, basename, filePattern !== basename ? hasSibling : undefined)) {
continue;
}
const sub = pathToEntries[relativePath];
if (sub) {
matchDirectory(sub);
} else {
// self.filesWalked++;
if (relativePath === filePattern) {
continue; // ignore file if its path matches with the file pattern because that is already matched above
}
self.matchFile(onResult, entry);
}
if (self.isLimitHit) {
break;
}
}
}
matchDirectory(rootEntries);
}
public getStats(): any {
return null;
// return {
// fromCache: false,
// traversal: Traversal[this.traversal],
// errors: this.errors,
// fileWalkStartTime: this.fileWalkStartTime,
// fileWalkResultTime: Date.now(),
// directoriesWalked: this.directoriesWalked,
// filesWalked: this.filesWalked,
// resultCount: this.resultCount,
// cmdForkResultTime: this.cmdForkResultTime,
// cmdResultCount: this.cmdResultCount
// };
}
private matchFile(onResult: (result: IInternalFileMatch) => void, candidate: IInternalFileMatch): void {
if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
if (this.exists || (this.maxResults && this.resultCount >= this.maxResults)) {
this.isLimitHit = true;
this.cancel();
}
if (!this.isLimitHit) {
onResult(candidate);
}
}
}
private isFilePatternMatch(path: string): boolean {
// Check for search pattern
if (this.filePattern) {
if (this.filePattern === '*') {
return true; // support the all-matching wildcard
}
return strings.fuzzyContains(path, this.normalizedFilePatternLowercase);
}
// No patterns means we match all
return true;
}
}
export class FileIndexSearchManager {
private static readonly BATCH_SIZE = 512;
private caches: { [cacheKey: string]: Cache; } = Object.create(null);
public fileSearch(config: ISearchQuery, provider: vscode.FileIndexProvider, onResult: (matches: IFileMatch[]) => void): TPromise<ISearchCompleteStats> {
if (config.sortByScore) {
let sortedSearch = this.trySortedSearchFromCache(config);
if (!sortedSearch) {
const engineConfig = config.maxResults ?
{
...config,
...{ maxResults: null }
} :
config;
const engine = new FileIndexSearchEngine(engineConfig, provider);
sortedSearch = this.doSortedSearch(engine, provider, config);
}
return new TPromise<ISearchCompleteStats>((c, e) => {
process.nextTick(() => { // allow caller to register progress callback first
sortedSearch.then(([result, rawMatches]) => {
const serializedMatches = rawMatches.map(rawMatch => this.rawMatchToSearchItem(rawMatch));
this.sendProgress(serializedMatches, onResult, FileIndexSearchManager.BATCH_SIZE);
c(result);
}, e, onResult);
});
}, () => {
sortedSearch.cancel();
});
}
let searchPromise: TPromise<void, IInternalFileMatch[]>;
return new TPromise<ISearchCompleteStats>((c, e) => {
const engine = new FileIndexSearchEngine(config, provider);
searchPromise = this.doSearch(engine, provider, FileIndexSearchManager.BATCH_SIZE)
.then(c, e, progress => {
if (Array.isArray(progress)) {
onResult(progress.map(m => this.rawMatchToSearchItem(m)));
} else if ((<IInternalFileMatch>progress).relativePath) {
onResult([this.rawMatchToSearchItem(<IInternalFileMatch>progress)]);
}
});
}, () => {
searchPromise.cancel();
});
}
private rawMatchToSearchItem(match: IInternalFileMatch): IFileMatch {
return {
resource: resources.joinPath(match.base, match.relativePath)
};
}
private doSortedSearch(engine: FileIndexSearchEngine, provider: vscode.FileIndexProvider, config: IRawSearchQuery): PPromise<[ISearchCompleteStats, IInternalFileMatch[]]> {
let searchPromise: PPromise<void, IInternalFileMatch[]>;
let allResultsPromise = new PPromise<[ISearchCompleteStats, IInternalFileMatch[]], IInternalFileMatch[]>((c, e, p) => {
let results: IInternalFileMatch[] = [];
searchPromise = this.doSearch(engine, provider, -1)
.then(result => {
c([result, results]);
}, e, progress => {
if (Array.isArray(progress)) {
results = progress;
} else {
p(progress);
}
});
}, () => {
searchPromise.cancel();
});
let cache: Cache;
if (config.cacheKey) {
cache = this.getOrCreateCache(config.cacheKey);
cache.resultsToSearchCache[config.filePattern] = allResultsPromise;
allResultsPromise.then(null, err => {
delete cache.resultsToSearchCache[config.filePattern];
});
allResultsPromise = this.preventCancellation(allResultsPromise);
}
let chained: TPromise<void>;
return new PPromise<[ISearchCompleteStats, IInternalFileMatch[]]>((c, e, p) => {
chained = allResultsPromise.then(([result, results]) => {
const scorerCache: ScorerCache = cache ? cache.scorerCache : Object.create(null);
const unsortedResultTime = Date.now();
return this.sortResults(config, results, scorerCache)
.then(sortedResults => {
const sortedResultTime = Date.now();
c([{
stats: {
...result.stats,
...{ unsortedResultTime, sortedResultTime }
},
limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults
}, sortedResults]);
});
}, e, p);
}, () => {
chained.cancel();
});
}
private getOrCreateCache(cacheKey: string): Cache {
const existing = this.caches[cacheKey];
if (existing) {
return existing;
}
return this.caches[cacheKey] = new Cache();
}
private trySortedSearchFromCache(config: IRawSearchQuery): TPromise<[ISearchCompleteStats, IInternalFileMatch[]]> {
const cache = config.cacheKey && this.caches[config.cacheKey];
if (!cache) {
return undefined;
}
const cacheLookupStartTime = Date.now();
const cached = this.getResultsFromCache(cache, config.filePattern);
if (cached) {
let chained: TPromise<void>;
return new TPromise<[ISearchCompleteStats, IInternalFileMatch[]]>((c, e) => {
chained = cached.then(([result, results, cacheStats]) => {
const cacheLookupResultTime = Date.now();
return this.sortResults(config, results, cache.scorerCache)
.then(sortedResults => {
const sortedResultTime = Date.now();
const stats: ICachedSearchStats = {
fromCache: true,
cacheLookupStartTime: cacheLookupStartTime,
cacheFilterStartTime: cacheStats.cacheFilterStartTime,
cacheLookupResultTime: cacheLookupResultTime,
cacheEntryCount: cacheStats.cacheFilterResultCount,
resultCount: results.length
};
if (config.sortByScore) {
stats.unsortedResultTime = cacheLookupResultTime;
stats.sortedResultTime = sortedResultTime;
}
if (!cacheStats.cacheWasResolved) {
stats.joined = result.stats;
}
c([
{
limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults,
stats: stats
},
sortedResults
]);
});
}, e);
}, () => {
chained.cancel();
});
}
return undefined;
}
private sortResults(config: IRawSearchQuery, results: IInternalFileMatch[], scorerCache: ScorerCache): TPromise<IInternalFileMatch[]> {
// we use the same compare function that is used later when showing the results using fuzzy scoring
// this is very important because we are also limiting the number of results by config.maxResults
// and as such we want the top items to be included in this result set if the number of items
// exceeds config.maxResults.
const query = prepareQuery(config.filePattern);
const compare = (matchA: IInternalFileMatch, matchB: IInternalFileMatch) => compareItemsByScore(matchA, matchB, query, true, FileMatchItemAccessor, scorerCache);
return arrays.topAsync(results, compare, config.maxResults, 10000);
}
private sendProgress(results: IFileMatch[], progressCb: (batch: IFileMatch[]) => void, batchSize: number) {
if (batchSize && batchSize > 0) {
for (let i = 0; i < results.length; i += batchSize) {
progressCb(results.slice(i, i + batchSize));
}
} else {
progressCb(results);
}
}
private getResultsFromCache(cache: Cache, searchValue: string): PPromise<[ISearchCompleteStats, IInternalFileMatch[], CacheStats]> {
if (path.isAbsolute(searchValue)) {
return null; // bypass cache if user looks up an absolute path where matching goes directly on disk
}
// Find cache entries by prefix of search value
const hasPathSep = searchValue.indexOf(path.sep) >= 0;
let cached: PPromise<[ISearchCompleteStats, IInternalFileMatch[]], IInternalFileMatch[]>;
let wasResolved: boolean;
for (let previousSearch in cache.resultsToSearchCache) {
// If we narrow down, we might be able to reuse the cached results
if (strings.startsWith(searchValue, previousSearch)) {
if (hasPathSep && previousSearch.indexOf(path.sep) < 0) {
continue; // since a path character widens the search for potential more matches, require it in previous search too
}
const c = cache.resultsToSearchCache[previousSearch];
c.then(() => { wasResolved = false; });
wasResolved = true;
cached = this.preventCancellation(c);
break;
}
}
if (!cached) {
return null;
}
return new PPromise<[ISearchCompleteStats, IInternalFileMatch[], CacheStats]>((c, e, p) => {
cached.then(([complete, cachedEntries]) => {
const cacheFilterStartTime = Date.now();
// Pattern match on results
let results: IInternalFileMatch[] = [];
const normalizedSearchValueLowercase = strings.stripWildcards(searchValue).toLowerCase();
for (let i = 0; i < cachedEntries.length; i++) {
let entry = cachedEntries[i];
// Check if this entry is a match for the search value
if (!strings.fuzzyContains(entry.relativePath, normalizedSearchValueLowercase)) {
continue;
}
results.push(entry);
}
c([complete, results, {
cacheWasResolved: wasResolved,
cacheFilterStartTime: cacheFilterStartTime,
cacheFilterResultCount: cachedEntries.length
}]);
}, e, p);
}, () => {
cached.cancel();
});
}
private doSearch(engine: FileIndexSearchEngine, provider: vscode.FileIndexProvider, batchSize?: number): PPromise<ISearchCompleteStats, IInternalFileMatch[]> {
return new PPromise<ISearchCompleteStats, IInternalFileMatch[]>((c, e, p) => {
let batch: IInternalFileMatch[] = [];
engine.search().then(result => {
if (batch.length) {
p(batch);
}
c({
limitHit: result.isLimitHit,
stats: engine.getStats() // TODO@roblou
});
}, error => {
if (batch.length) {
p(batch);
}
e(error);
}, match => {
if (match) {
if (batchSize) {
batch.push(match);
if (batchSize > 0 && batch.length >= batchSize) {
p(batch);
batch = [];
}
} else {
p([match]);
}
}
});
}, () => {
engine.cancel();
});
}
public clearCache(cacheKey: string): TPromise<void> {
delete this.caches[cacheKey];
return TPromise.as(undefined);
}
private preventCancellation<C, P>(promise: PPromise<C, P>): PPromise<C, P> {
return new PPromise<C, P>((c, e, p) => {
// Allow for piled up cancellations to come through first.
process.nextTick(() => {
promise.then(c, e, p);
});
}, () => {
// Do not propagate.
});
}
}
class Cache {
public resultsToSearchCache: { [searchValue: string]: PPromise<[ISearchCompleteStats, IInternalFileMatch[]], IInternalFileMatch[]>; } = Object.create(null);
public scorerCache: ScorerCache = Object.create(null);
}
const FileMatchItemAccessor = new class implements IItemAccessor<IInternalFileMatch> {
public getItemLabel(match: IInternalFileMatch): string {
return match.basename; // e.g. myFile.txt
}
public getItemDescription(match: IInternalFileMatch): string {
return match.relativePath.substr(0, match.relativePath.length - match.basename.length - 1); // e.g. some/path/to/file
}
public getItemPath(match: IInternalFileMatch): string {
return match.relativePath; // e.g. some/path/to/file/myFile.txt
}
};
interface CacheStats {
cacheWasResolved: boolean;
cacheFilterStartTime: number;
cacheFilterResultCount: number;
}

172
src/vs/workbench/api/node/extHostSearch.ts
View file

@ -16,6 +16,7 @@ import { IFileMatch, IFolderQuery, IPatternInfo, IRawSearchQuery, ISearchComplet
import * as vscode from 'vscode';
import { ExtHostSearchShape, IMainContext, MainContext, MainThreadSearchShape } from './extHost.protocol';
import { toDisposable } from 'vs/base/common/lifecycle';
import { IInternalFileMatch, QueryGlobTester, resolvePatternsForProvider, IDirectoryTree, IDirectoryEntry, FileIndexSearchManager } from 'vs/workbench/api/node/extHostSearch.fileIndex';
export interface ISchemeTransformer {
transformOutgoing(scheme: string): string;
@ -24,15 +25,18 @@ export interface ISchemeTransformer {
export class ExtHostSearch implements ExtHostSearchShape {
private readonly _proxy: MainThreadSearchShape;
private readonly _searchProvider = new Map<number, vscode.SearchProvider>();
private readonly _fileSearchProvider = new Map<number, vscode.SearchProvider>();
private readonly _textSearchProvider = new Map<number, vscode.TextSearchProvider>();
private readonly _fileIndexProvider = new Map<number, vscode.FileIndexProvider>();
private _handlePool: number = 0;
private _fileSearchManager: FileSearchManager;
private _fileIndexSearchManager: FileIndexSearchManager;
constructor(mainContext: IMainContext, private _schemeTransformer: ISchemeTransformer, private _extfs = extfs) {
this._proxy = mainContext.getProxy(MainContext.MainThreadSearch);
this._fileSearchManager = new FileSearchManager();
this._fileIndexSearchManager = new FileIndexSearchManager();
}
private _transformScheme(scheme: string): string {
@ -42,12 +46,12 @@ export class ExtHostSearch implements ExtHostSearchShape {
return scheme;
}
registerSearchProvider(scheme: string, provider: vscode.SearchProvider) {
registerFileSearchProvider(scheme: string, provider: vscode.SearchProvider) {
const handle = this._handlePool++;
this._searchProvider.set(handle, provider);
this._proxy.$registerSearchProvider(handle, this._transformScheme(scheme));
this._fileSearchProvider.set(handle, provider);
this._proxy.$registerFileSearchProvider(handle, this._transformScheme(scheme));
return toDisposable(() => {
this._searchProvider.delete(handle);
this._fileSearchProvider.delete(handle);
this._proxy.$unregisterProvider(handle);
});
}
@ -55,27 +59,44 @@ export class ExtHostSearch implements ExtHostSearchShape {
registerTextSearchProvider(scheme: string, provider: vscode.TextSearchProvider) {
const handle = this._handlePool++;
this._textSearchProvider.set(handle, provider);
this._proxy.$registerSearchProvider(handle, this._transformScheme(scheme));
this._proxy.$registerTextSearchProvider(handle, this._transformScheme(scheme));
return toDisposable(() => {
this._searchProvider.delete(handle);
this._textSearchProvider.delete(handle);
this._proxy.$unregisterProvider(handle);
});
}
$provideFileSearchResults(handle: number, session: number, rawQuery: IRawSearchQuery): TPromise<ISearchCompleteStats> {
const provider = this._searchProvider.get(handle);
if (!provider.provideFileSearchResults) {
return TPromise.as(undefined);
}
const query = reviveQuery(rawQuery);
return this._fileSearchManager.fileSearch(query, provider, progress => {
this._proxy.$handleFileMatch(handle, session, progress.map(p => p.resource));
registerFileIndexProvider(scheme: string, provider: vscode.FileIndexProvider) {
const handle = this._handlePool++;
this._fileIndexProvider.set(handle, provider);
this._proxy.$registerFileIndexProvider(handle, this._transformScheme(scheme));
return toDisposable(() => {
this._fileSearchProvider.delete(handle);
this._proxy.$unregisterProvider(handle); // TODO@roblou - unregisterFileIndexProvider
});
}
$provideFileSearchResults(handle: number, session: number, rawQuery: IRawSearchQuery): TPromise<ISearchCompleteStats> {
const provider = this._fileSearchProvider.get(handle);
const query = reviveQuery(rawQuery);
if (provider) {
return this._fileSearchManager.fileSearch(query, provider, progress => {
this._proxy.$handleFileMatch(handle, session, progress.map(p => p.resource));
});
} else {
const indexProvider = this._fileIndexProvider.get(handle);
if (indexProvider) {
return this._fileIndexSearchManager.fileSearch(query, indexProvider, progress => {
this._proxy.$handleFileMatch(handle, session, progress.map(p => p.resource));
});
} else {
throw new Error('something went wrong');
}
}
}
$clearCache(handle: number, cacheKey: string): TPromise<void> {
const provider = this._searchProvider.get(handle);
const provider = this._fileSearchProvider.get(handle);
if (!provider.clearCache) {
return TPromise.as(undefined);
}
@ -96,22 +117,6 @@ export class ExtHostSearch implements ExtHostSearchShape {
}
}
/**
* Computes the patterns that the provider handles. Discards sibling clauses and 'false' patterns
*/
function resolvePatternsForProvider(globalPattern: glob.IExpression, folderPattern: glob.IExpression): string[] {
const merged = {
...(globalPattern || {}),
...(folderPattern || {})
};
return Object.keys(merged)
.filter(key => {
const value = merged[key];
return typeof value === 'boolean' && value;
});
}
function reviveQuery(rawQuery: IRawSearchQuery): ISearchQuery {
return {
...rawQuery,
@ -264,107 +269,6 @@ class BatchedCollector<T> {
}
}
interface IDirectoryEntry {
base: URI;
relativePath: string;
basename: string;
}
interface IDirectoryTree {
rootEntries: IDirectoryEntry[];
pathToEntries: { [relativePath: string]: IDirectoryEntry[] };
}
interface IInternalFileMatch {
base: URI;
relativePath?: string; // Not present for extraFiles or absolute path matches
basename: string;
size?: number;
}
class QueryGlobTester {
private _excludeExpression: glob.IExpression;
private _parsedExcludeExpression: glob.ParsedExpression;
private _parsedIncludeExpression: glob.ParsedExpression;
constructor(config: ISearchQuery, folderQuery: IFolderQuery) {
this._excludeExpression = {
...(config.excludePattern || {}),
...(folderQuery.excludePattern || {})
};
this._parsedExcludeExpression = glob.parse(this._excludeExpression);
// Empty includeExpression means include nothing, so no {} shortcuts
let includeExpression: glob.IExpression = config.includePattern;
if (folderQuery.includePattern) {
if (includeExpression) {
includeExpression = {
...includeExpression,
...folderQuery.includePattern
};
} else {
includeExpression = folderQuery.includePattern;
}
}
if (includeExpression) {
this._parsedIncludeExpression = glob.parse(includeExpression);
}
}
/**
* Guaranteed sync - siblingsFn should not return a promise.
*/
public includedInQuerySync(testPath: string, basename?: string, hasSibling?: (name: string) => boolean): boolean {
if (this._parsedExcludeExpression && this._parsedExcludeExpression(testPath, basename, hasSibling)) {
return false;
}
if (this._parsedIncludeExpression && !this._parsedIncludeExpression(testPath, basename, hasSibling)) {
return false;
}
return true;
}
/**
* Guaranteed async.
*/
public includedInQuery(testPath: string, basename?: string, hasSibling?: (name: string) => boolean | TPromise<boolean>): TPromise<boolean> {
const excludeP = this._parsedExcludeExpression ?
TPromise.as(this._parsedExcludeExpression(testPath, basename, hasSibling)).then(result => !!result) :
TPromise.wrap(false);
return excludeP.then(excluded => {
if (excluded) {
return false;
}
return this._parsedIncludeExpression ?
TPromise.as(this._parsedIncludeExpression(testPath, basename, hasSibling)).then(result => !!result) :
TPromise.wrap(true);
}).then(included => {
return included;
});
}
public hasSiblingExcludeClauses(): boolean {
return hasSiblingClauses(this._excludeExpression);
}
}
function hasSiblingClauses(pattern: glob.IExpression): boolean {
for (let key in pattern) {
if (typeof pattern[key] !== 'boolean') {
return true;
}
}
return false;
}
class TextSearchEngine {
private activeCancellationTokens = new Set<CancellationTokenSource>();

89
src/vs/workbench/services/search/node/searchService.ts
View file

@ -20,7 +20,7 @@ import { IModelService } from 'vs/editor/common/services/modelService';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { IDebugParams, IEnvironmentService } from 'vs/platform/environment/common/environment';
import { ILogService } from 'vs/platform/log/common/log';
import { FileMatch, IFileMatch, IFolderQuery, IProgress, ISearchComplete, ISearchConfiguration, ISearchProgressItem, ISearchQuery, ISearchResultProvider, ISearchService, LineMatch, pathIncludedInQuery, QueryType } from 'vs/platform/search/common/search';
import { FileMatch, IFileMatch, IFolderQuery, IProgress, ISearchComplete, ISearchConfiguration, ISearchProgressItem, ISearchQuery, ISearchResultProvider, ISearchService, LineMatch, pathIncludedInQuery, QueryType, SearchProviderType } from 'vs/platform/search/common/search';
import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/untitledEditorService';
@ -31,8 +31,9 @@ export class SearchService extends Disposable implements ISearchService {
public _serviceBrand: any;
private diskSearch: DiskSearch;
private readonly searchProviders: ISearchResultProvider[] = [];
private fileSearchProvider: ISearchResultProvider;
private readonly fileSearchProviders = new Map<string, ISearchResultProvider>();
private readonly textSearchProviders = new Map<string, ISearchResultProvider>();
private readonly fileIndexProviders = new Map<string, ISearchResultProvider>();
constructor(
@IModelService private modelService: IModelService,
@ -50,22 +51,23 @@ export class SearchService extends Disposable implements ISearchService {
}));
}
public registerSearchResultProvider(scheme: string, provider: ISearchResultProvider): IDisposable {
if (scheme === 'file') {
this.fileSearchProvider = provider;
} else {
this.searchProviders.push(provider);
public registerSearchResultProvider(scheme: string, type: SearchProviderType, provider: ISearchResultProvider): IDisposable {
// if (scheme === 'file') {
// this.fileSearchProvider = provider;
let list: Map<string, ISearchResultProvider>;
if (type === SearchProviderType.file) {
list = this.fileSearchProviders;
} else if (type === SearchProviderType.text) {
list = this.textSearchProviders;
} else if (type === SearchProviderType.fileIndex) {
list = this.fileIndexProviders;
}
list.set(scheme, provider);
return toDisposable(() => {
if (scheme === 'file') {
this.fileSearchProvider = null;
} else {
const idx = this.searchProviders.indexOf(provider);
if (idx >= 0) {
this.searchProviders.splice(idx, 1);
}
}
list.delete(scheme);
});
}
@ -122,38 +124,43 @@ export class SearchService extends Disposable implements ISearchService {
};
const startTime = Date.now();
const searchWithProvider = (provider: ISearchResultProvider) => TPromise.as(provider.search(query, onProviderProgress));
const schemesInQuery = query.folderQueries.map(fq => fq.folder.scheme);
const providerActivations = schemesInQuery.map(scheme => this.extensionService.activateByEvent(`onSearch:${scheme}`));
const providerPromise = TPromise.join(providerActivations).then(() => {
// TODO@roblou this is not properly waiting for search-rg to finish registering itself
// If no search provider has been registered for the 'file' schema, fall back on DiskSearch
const providers = [
this.fileSearchProvider || this.diskSearch,
...this.searchProviders
];
return TPromise.join(providers.map(p => searchWithProvider(p)))
.then(completes => {
completes = completes.filter(c => !!c);
if (!completes.length) {
return null;
return TPromise.join(query.folderQueries.map(fq => {
const oneFolderQuery = {
...query,
...{
folderQueries: [fq]
}
};
return <ISearchComplete>{
limitHit: completes[0] && completes[0].limitHit,
stats: completes[0].stats,
results: arrays.flatten(completes.map(c => c.results))
};
}, errs => {
if (!Array.isArray(errs)) {
errs = [errs];
}
const provider = query.type === QueryType.File ?
this.fileSearchProviders.get(fq.folder.scheme) || this.fileIndexProviders.get(fq.folder.scheme) :
this.textSearchProviders.get(fq.folder.scheme);
errs = errs.filter(e => !!e);
return TPromise.wrapError(errs[0]);
});
return TPromise.as(provider.search(oneFolderQuery, onProviderProgress));
})).then(completes => {
completes = completes.filter(c => !!c);
if (!completes.length) {
return null;
}
return <ISearchComplete>{
limitHit: completes[0] && completes[0].limitHit,
stats: completes[0].stats,
results: arrays.flatten(completes.map(c => c.results))
};
}, errs => {
if (!Array.isArray(errs)) {
errs = [errs];
}
errs = errs.filter(e => !!e);
return TPromise.wrapError(errs[0]);
});
});
combinedPromise = providerPromise.then(value => {
@ -262,8 +269,6 @@ export class SearchService extends Disposable implements ISearchService {
public clearCache(cacheKey: string): TPromise<void> {
return TPromise.join([
...this.searchProviders,
this.fileSearchProvider,
this.diskSearch
].map(provider => provider && provider.clearCache(cacheKey)))
.then(() => { });

42
src/vs/workbench/test/electron-browser/api/extHostSearch.test.ts
View file

@ -28,7 +28,11 @@ class MockMainThreadSearch implements MainThreadSearchShape {
results: (UriComponents | IRawFileMatch2)[] = [];
$registerSearchProvider(handle: number, scheme: string): void {
$registerFileSearchProvider(handle: number, scheme: string): void {
this.lastHandle = handle;
}
$registerFileIndexProvider(handle: number, scheme: string): void {
this.lastHandle = handle;
}
@ -62,8 +66,8 @@ suite('ExtHostSearch', () => {
await rpcProtocol.sync();
}
async function registerTestSearchProvider(provider: vscode.SearchProvider, scheme = 'file'): Promise<void> {
disposables.push(extHostSearch.registerSearchProvider(scheme, provider));
async function registerTestFileSearchProvider(provider: vscode.SearchProvider, scheme = 'file'): Promise<void> {
disposables.push(extHostSearch.registerFileSearchProvider(scheme, provider));
await rpcProtocol.sync();
}
@ -162,7 +166,7 @@ suite('ExtHostSearch', () => {
}
test('no results', async () => {
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
return TPromise.wrap(null);
}
@ -180,7 +184,7 @@ suite('ExtHostSearch', () => {
joinPath(rootFolderA, 'file3.ts')
];
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults.forEach(r => progress.report(r));
return TPromise.wrap(null);
@ -195,7 +199,7 @@ suite('ExtHostSearch', () => {
test('Search canceled', async () => {
let cancelRequested = false;
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
return new TPromise((resolve, reject) => {
token.onCancellationRequested(() => {
@ -220,7 +224,7 @@ suite('ExtHostSearch', () => {
'file3.ts',
];
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults
.map(relativePath => joinPath(options.folder, relativePath))
@ -239,7 +243,7 @@ suite('ExtHostSearch', () => {
});
test('provider returns null', async () => {
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
return null;
}
@ -254,7 +258,7 @@ suite('ExtHostSearch', () => {
});
test('all provider calls get global include/excludes', async () => {
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
assert(options.excludes.length === 2 && options.includes.length === 2, 'Missing global include/excludes');
return TPromise.wrap(null);
@ -283,7 +287,7 @@ suite('ExtHostSearch', () => {
});
test('global/local include/excludes combined', async () => {
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
if (options.folder.toString() === rootFolderA.toString()) {
assert.deepEqual(options.includes.sort(), ['*.ts', 'foo']);
@ -325,7 +329,7 @@ suite('ExtHostSearch', () => {
});
test('include/excludes resolved correctly', async () => {
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
assert.deepEqual(options.includes.sort(), ['*.jsx', '*.ts']);
assert.deepEqual(options.excludes.sort(), []);
@ -368,7 +372,7 @@ suite('ExtHostSearch', () => {
'file1.js',
];
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults
.map(relativePath => joinPath(options.folder, relativePath))
@ -401,7 +405,7 @@ suite('ExtHostSearch', () => {
test('multiroot sibling exclude clause', async () => {
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
let reportedResults: URI[];
if (options.folder.fsPath === rootFolderA.fsPath) {
@ -472,7 +476,7 @@ suite('ExtHostSearch', () => {
];
let wasCanceled = false;
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults
.forEach(r => progress.report(r));
@ -510,7 +514,7 @@ suite('ExtHostSearch', () => {
];
let wasCanceled = false;
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults.forEach(r => progress.report(r));
token.onCancellationRequested(() => wasCanceled = true);
@ -546,7 +550,7 @@ suite('ExtHostSearch', () => {
];
let wasCanceled = false;
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults.forEach(r => progress.report(r));
token.onCancellationRequested(() => wasCanceled = true);
@ -577,7 +581,7 @@ suite('ExtHostSearch', () => {
test('multiroot max results', async () => {
let cancels = 0;
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
token.onCancellationRequested(() => cancels++);
@ -623,7 +627,7 @@ suite('ExtHostSearch', () => {
];
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
reportedResults.forEach(r => progress.report(r));
return TPromise.wrap(null);
@ -646,7 +650,7 @@ suite('ExtHostSearch', () => {
test('uses different cache keys for different folders', async () => {
const cacheKeys: string[] = [];
await registerTestSearchProvider({
await registerTestFileSearchProvider({
provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, progress: vscode.Progress<URI>, token: vscode.CancellationToken): Thenable<void> {
cacheKeys.push(query.cacheKey);
return TPromise.wrap(null);