Initial stress test for code completion

Change-Id: I5ec7ff846da2924ef26a7e738a5e30191a3df9c5
Reviewed-on: https://dart-review.googlesource.com/68440
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>

pkg/analysis_server/test/stress/completion/completion.dart

@@ -0,0 +1,100 @@
+// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+
+import 'completion_runner.dart';
+
+/**
+ * The main entry point for the code completion stress test.
+ */
+void main(List<String> args) async {
+  ArgParser parser = createArgParser();
+  ArgResults result = parser.parse(args);
+
+  if (validArguments(parser, result)) {
+    String analysisRoot = result.rest[0];
+
+    CompletionRunner runner = new CompletionRunner(
+        output: stdout,
+        printMissing: result['missing'],
+        timing: result['timing'],
+        useCFE: result['use-cfe'],
+        verbose: result['verbose']);
+    await runner.runAll(analysisRoot);
+    stdout.flush();
+  }
+}
+
+/**
+ * Create a parser that can be used to parse the command-line arguments.
+ */
+ArgParser createArgParser() {
+  ArgParser parser = new ArgParser();
+  parser.addFlag(
+    'help',
+    abbr: 'h',
+    help: 'Print this help message',
+    negatable: false,
+  );
+  parser.addFlag(
+    'missing',
+    help: 'Report locations where the current identifier was not suggested',
+    negatable: false,
+  );
+  parser.addFlag(
+    'timing',
+    help: 'Report timing information',
+    negatable: false,
+  );
+  parser.addFlag(
+    'use-cfe',
+    help: 'Use the CFE to perform the analysis',
+    negatable: false,
+  );
+  parser.addFlag(
+    'verbose',
+    abbr: 'v',
+    help: 'Produce verbose output',
+    negatable: false,
+  );
+  return parser;
+}
+
+/**
+ * Print usage information for this tool.
+ */
+void printUsage(ArgParser parser, {String error}) {
+  if (error != null) {
+    print(error);
+    print('');
+  }
+  print('usage: dart completion path');
+  print('');
+  print('Test the completion engine by requesting completion at the offset of');
+  print('each identifier in the files contained in the given path. The path');
+  print('can be either a single Dart file or a directory.');
+  print('');
+  print(parser.usage);
+}
+
+/**
+ * Return `true` if the command-line arguments (represented by the [result] and
+ * parsed by the [parser]) are valid.
+ */
+bool validArguments(ArgParser parser, ArgResults result) {
+  if (result.wasParsed('help')) {
+    printUsage(parser);
+    return false;
+  } else if (result.rest.length < 1) {
+    printUsage(parser, error: 'Missing path to files');
+    return false;
+  } else if (result.rest.length > 1) {
+    printUsage(parser, error: 'Only one file can be analyzed');
+    return false;
+  }
+  return true;
+}

pkg/analysis_server/test/stress/completion/completion_runner.dart

@@ -0,0 +1,224 @@
+// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:analysis_server/src/services/completion/completion_core.dart';
+import 'package:analysis_server/src/services/completion/completion_performance.dart';
+import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
+import 'package:analysis_server/src/utilities/null_string_sink.dart';
+import 'package:analyzer/dart/analysis/analysis_context.dart';
+import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
+import 'package:analyzer/dart/analysis/results.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/dart/ast/visitor.dart';
+import 'package:analyzer/file_system/overlay_file_system.dart';
+import 'package:analyzer/file_system/physical_file_system.dart';
+import 'package:analyzer/source/line_info.dart';
+import 'package:analyzer/src/generated/source.dart';
+import 'package:analyzer_plugin/protocol/protocol_common.dart';
+
+/**
+ * A runner that can request code completion at the location of each identifier
+ * in a Dart file.
+ */
+class CompletionRunner {
+  /**
+   * The sink to which output is to be written.
+   */
+  final StringSink output;
+
+  /**
+   * A flag indicating whether to produce output about missing suggestions.
+   */
+  final bool printMissing;
+
+  /**
+   * A flag indicating whether to produce timing information.
+   */
+  final bool timing;
+
+  /**
+   * A flag indicating whether to use the CFE when running the tests.
+   */
+  final bool useCFE;
+
+  /**
+   * A flag indicating whether to produce verbose output.
+   */
+  final bool verbose;
+
+  /**
+   * A flag indicating whether we should delete each identifier before
+   * attempting to complete at that offset.
+   */
+  bool deleteBeforeCompletion = false;
+
+  /**
+   * Initialize a newly created completion runner.
+   */
+  CompletionRunner(
+      {StringSink output,
+      bool printMissing,
+      bool timing,
+      bool useCFE,
+      bool verbose})
+      : this.output = output ?? new NullStringSink(),
+        this.printMissing = printMissing ?? false,
+        this.timing = timing ?? false,
+        this.useCFE = useCFE ?? false,
+        this.verbose = verbose ?? false;
+
+  /**
+   * Test the completion engine at the locations of each of the identifiers in
+   * each of the files in the given [analysisRoot].
+   */
+  Future<void> runAll(String analysisRoot) async {
+    OverlayResourceProvider resourceProvider =
+        new OverlayResourceProvider(PhysicalResourceProvider.INSTANCE);
+    AnalysisContextCollection collection = new AnalysisContextCollection(
+        includedPaths: <String>[analysisRoot],
+        resourceProvider: resourceProvider,
+        useCFE: useCFE); // ignore: deprecated_member_use
+    DartCompletionManager contributor = new DartCompletionManager();
+    CompletionPerformance performance = new CompletionPerformance();
+    int stamp = 1;
+
+    int fileCount = 0;
+    int identifierCount = 0;
+    int missingCount = 0;
+
+    // Consider getting individual timings so that we can also report the
+    // longest and shortest times, or even a distribution.
+    Stopwatch timer = new Stopwatch();
+
+    for (AnalysisContext context in collection.contexts) {
+      for (String path in context.contextRoot.analyzedFiles()) {
+        fileCount++;
+        output.write('.');
+        ResolveResult result =
+            await context.currentSession.getResolvedAst(path);
+        String content = result.content;
+        LineInfo lineInfo = result.lineInfo;
+        List<SimpleIdentifier> identifiers = _identifiersIn(result.unit);
+
+        for (SimpleIdentifier identifier in identifiers) {
+          identifierCount++;
+          int offset = identifier.offset;
+          if (deleteBeforeCompletion) {
+            String modifiedContent = content.substring(0, offset) +
+                content.substring(identifier.end);
+            resourceProvider.setOverlay(path,
+                content: modifiedContent, modificationStamp: stamp++);
+            result = await context.currentSession.getResolvedAst(path);
+          }
+
+          timer.start();
+          CompletionRequestImpl request =
+              new CompletionRequestImpl(result, offset, performance);
+          List<CompletionSuggestion> suggestions =
+              await contributor.computeSuggestions(request);
+          timer.stop();
+
+          if (!identifier.inDeclarationContext() &&
+              !_isNamedExpressionName(identifier)) {
+            if (!_hasSuggestion(suggestions, identifier.name)) {
+              missingCount++;
+              if (printMissing) {
+                CharacterLocation location = lineInfo.getLocation(offset);
+                output.writeln('Missing suggestion of "${identifier.name}" at '
+                    '$path:${location.lineNumber}:${location.columnNumber}');
+                if (verbose) {
+                  _printSuggestions(suggestions);
+                }
+              }
+            }
+          }
+        }
+        if (deleteBeforeCompletion) {
+          resourceProvider.removeOverlay(path);
+        }
+      }
+    }
+    output.writeln();
+    if (printMissing) {
+      output.writeln();
+    }
+    if (identifierCount == 0) {
+      output.writeln('No identifiers found in $fileCount files');
+    } else {
+      int percent = (missingCount * 100 / identifierCount).round();
+      output.writeln('$percent% missing suggestions '
+          '($missingCount of $identifierCount in $fileCount files)');
+    }
+    if (timing && identifierCount > 0) {
+      int time = timer.elapsedMilliseconds;
+      int averageTime = (time / identifierCount).round();
+      output.writeln('completion took $time ms, '
+          'which is an average of $averageTime ms per completion');
+    }
+  }
+
+  /**
+   * Return `true` if the given list of [suggestions] includes a suggestion for
+   * the given [identifier].
+   */
+  bool _hasSuggestion(
+      List<CompletionSuggestion> suggestions, String identifier) {
+    for (CompletionSuggestion suggestion in suggestions) {
+      if (suggestion.completion == identifier) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Return a list containing information about the identifiers in the given
+   * compilation [unit].
+   */
+  List<SimpleIdentifier> _identifiersIn(CompilationUnit unit) {
+    IdentifierCollector visitor = new IdentifierCollector();
+    unit.accept(visitor);
+    return visitor.identifiers;
+  }
+
+  /**
+   * Return `true` if the given [identifier] is being used as the name of a
+   * named expression.
+   */
+  bool _isNamedExpressionName(SimpleIdentifier identifier) {
+    AstNode parent = identifier.parent;
+    return parent is NamedExpression && parent.name == identifier;
+  }
+
+  /**
+   * Print information about the given [suggestions].
+   */
+  void _printSuggestions(List<CompletionSuggestion> suggestions) {
+    if (suggestions.length == 0) {
+      output.writeln('  No suggestions');
+      return;
+    }
+    output.writeln('  Suggestions:');
+    for (CompletionSuggestion suggestion in suggestions) {
+      output.writeln('    ${suggestion.completion}');
+    }
+  }
+}
+
+/**
+ * A visitor that will collect simple identifiers in the AST being visited.
+ */
+class IdentifierCollector extends RecursiveAstVisitor<void> {
+  /**
+   * The simple identifiers that were collected.
+   */
+  final List<SimpleIdentifier> identifiers = <SimpleIdentifier>[];
+
+  @override
+  visitSimpleIdentifier(SimpleIdentifier node) {
+    identifiers.add(node);
+  }
+}