Initial analytics for the dartdev cli tool

From the original upload, this change has been updated to include:
- a new flag to disable analytics when running dartdev tests
- some tests on the analytics.dart utility methods
- logic for recording flags and options on dartdev cli calls
- message printed when the user runs the sub command `dart disable-analytics`
- a valid tracking ID

Change-Id: Ifeb4d0669b9fc53b62fea88285e5ac78cc8e14a4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/147823
Commit-Queue: Jaime Wren <jwren@google.com>
Reviewed-by: Devon Carew <devoncarew@google.com>

pkg/dartdev/bin/dartdev.dart

@@ -2,25 +2,9 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:io';
-
-import 'package:args/command_runner.dart';
 import 'package:dartdev/dartdev.dart';
 
 /// The entry point for dartdev.
-main(List<String> args) async {
-  final runner = DartdevRunner(args);
-  try {
-    dynamic result = await runner.run(args);
-    exit(result is int ? result : 0);
-  } catch (e, st) {
-    if (e is UsageException) {
-      stderr.writeln('$e');
-      exit(64);
-    } else {
-      stderr.writeln('$e');
-      stderr.writeln('$st');
-      exit(1);
-    }
-  }
+void main(List<String> args) async {
+  await runDartdev(args);
 }

pkg/dartdev/lib/dartdev.dart

@@ -2,13 +2,15 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:io';
+import 'dart:io' as io;
 
 import 'package:args/args.dart';
 import 'package:args/command_runner.dart';
 import 'package:cli_util/cli_logging.dart';
 import 'package:nnbd_migration/migration_cli.dart';
+import 'package:usage/usage.dart';
 
+import 'src/analytics.dart';
 import 'src/commands/analyze.dart';
 import 'src/commands/create.dart';
 import 'src/commands/format.dart';
@@ -17,6 +19,124 @@ import 'src/commands/run.dart';
 import 'src/commands/test.dart';
 import 'src/core.dart';
 
+/// This is typically called from bin/, but given the length of the method and
+/// analytics logic, it has been moved here. Also note that this method calls
+/// [io.exit(code)] directly.
+void runDartdev(List<String> args) async {
+  final stopwatch = Stopwatch();
+  dynamic result;
+
+  // The Analytics instance used to report information back to Google Analytics,
+  // see lib/src/analytics.dart.
+  Analytics analytics;
+
+  // The exit code for the dartdev process, null indicates that it has not yet
+  // been set yet. The value is set in the catch and finally blocks below.
+  int exitCode;
+
+  // Any caught non-UsageExceptions when running the sub command
+  Exception exception;
+  StackTrace stackTrace;
+
+  var runner;
+
+  analytics =
+      createAnalyticsInstance(args.contains('--disable-dartdev-analytics'));
+
+  // On the first run, print the message to alert users that anonymous data will
+  // be collected by default.
+  if (analytics.firstRun) {
+    print(analyticsNoticeOnFirstRunMessage);
+  }
+
+  // When `--disable-analytics` or `--enable-analytics` are called we perform
+  // the respective intention and print any notices to standard out and exit.
+  if (args.contains('--disable-analytics')) {
+    // This block also potentially catches the case of (disableAnalytics &&
+    // enableAnalytics), in which we favor the disabling of analytics.
+    analytics.enabled = false;
+
+    // Alert the user that analytics have been disabled:
+    print(analyticsDisabledNoticeMessage);
+    io.exit(0);
+  } else if (args.contains('--enable-analytics')) {
+    analytics.enabled = true;
+
+    // Alert the user again that anonymous data will be collected:
+    print(analyticsNoticeOnFirstRunMessage);
+    io.exit(0);
+  }
+
+  var commandName;
+
+  try {
+    stopwatch.start();
+    runner = DartdevRunner(args);
+    // Run can't be called with the '--disable-dartdev-analytics' flag, remove
+    // it if it is contained in args.
+    if (args.contains('--disable-dartdev-analytics')) {
+      args = List.from(args)..remove('--disable-dartdev-analytics');
+    }
+
+    // Before calling to run, send the first ping to analytics to have the first
+    // ping, as well as the command itself, running in parallel.
+    if (analytics.enabled) {
+      analytics.setSessionValue(flagsParam, getFlags(args));
+      commandName = getCommandStr(args, runner.commands.keys.toList());
+      // ignore: unawaited_futures
+      analytics.sendEvent(eventCategory, commandName);
+    }
+
+    // Finally, call the runner to execute the command, see DartdevRunner.
+    result = await runner.run(args);
+  } catch (e, st) {
+    if (e is UsageException) {
+      io.stderr.writeln('$e');
+      exitCode = 64;
+    } else {
+      // Set the exception and stack trace only for non-UsageException cases:
+      exception = e;
+      stackTrace = st;
+      io.stderr.writeln('$e');
+      io.stderr.writeln('$st');
+      exitCode = 1;
+    }
+  } finally {
+    stopwatch.stop();
+
+    // Set the exitCode, if it wasn't set in the catch block above.
+    if (exitCode == null) {
+      exitCode = result is int ? result : 0;
+    }
+
+    // Send analytics before exiting
+    if (analytics.enabled) {
+      analytics.setSessionValue(exitCodeParam, exitCode);
+      // ignore: unawaited_futures
+      analytics.sendTiming(commandName, stopwatch.elapsedMilliseconds,
+          category: 'commands');
+
+      // And now send the exceptions and events to Google Analytics:
+      if (exception != null) {
+        // ignore: unawaited_futures
+        analytics.sendException(
+            '${exception.runtimeType}\n${sanitizeStacktrace(stackTrace)}',
+            fatal: true);
+      }
+
+      await analytics.waitForLastPing(timeout: Duration(milliseconds: 200));
+    }
+
+    // As the notification to the user read on the first run, analytics are
+    // enabled by default, on the first run only.
+    if (analytics.firstRun) {
+      analytics.enabled = true;
+    }
+    analytics.close();
+    io.exit(exitCode);
+  }
+}
+
 class DartdevRunner<int> extends CommandRunner {
   static const String dartdevDescription =
       'A command-line utility for Dart development';
@@ -28,6 +148,18 @@ class DartdevRunner<int> extends CommandRunner {
         abbr: 'v', negatable: false, help: 'Show verbose output.');
     argParser.addFlag('version',
         negatable: false, help: 'Print the Dart SDK version.');
+    argParser.addFlag('enable-analytics',
+        negatable: false, help: 'Enable anonymous analytics.');
+    argParser.addFlag('disable-analytics',
+        negatable: false, help: 'Disable anonymous analytics.');
+
+    // A hidden flag to disable analytics on this run, this constructor can be
+    // called with this flag, but should be removed before run() is called as
+    // the flag has not been added to all sub-commands.
+    argParser.addFlag('disable-dartdev-analytics',
+        negatable: false,
+        help: 'Disable anonymous analytics for this `dart *` run',
+        hide: true);
 
     addCommand(AnalyzeCommand(verbose: verbose));
     addCommand(CreateCommand(verbose: verbose));
@@ -44,14 +176,15 @@ class DartdevRunner<int> extends CommandRunner {
 
   @override
   Future<int> runCommand(ArgResults results) async {
+    assert(!results.arguments.contains('--disable-dartdev-analytics'));
     if (results.command == null && results.arguments.isNotEmpty) {
       final firstArg = results.arguments.first;
       // If we make it this far, it means the VM couldn't find the file on disk.
       if (firstArg.endsWith('.dart')) {
-        stderr.writeln(
+        io.stderr.writeln(
             "Error when reading '$firstArg': No such file or directory.");
         // This is the exit code used by the frontend.
-        exit(254);
+        io.exit(254);
       }
     }
     isVerbose = results['verbose'];

pkg/dartdev/lib/src/analytics.dart

@@ -0,0 +1,161 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+import 'package:telemetry/telemetry.dart' as telemetry show isRunningOnBot;
+import 'package:usage/src/usage_impl.dart';
+import 'package:usage/src/usage_impl_io.dart';
+import 'package:usage/usage_io.dart';
+
+const String analyticsNoticeOnFirstRunMessage = '''
+  ╔════════════════════════════════════════════════════════════════════════════╗
+  ║ The Dart tool uses Google Analytics to anonymously report feature usage    ║
+  ║ statistics, and crash reporting to send basic crash reports. This data is  ║
+  ║ used to help improve the Dart platform and tools over time.                ║
+  ║                                                                            ║
+  ║ To disable reporting of anonymous tool usage statistics in general, run    ║
+  ║ the command: `dart --disable-analytics`.                                   ║
+  ╚════════════════════════════════════════════════════════════════════════════╝
+''';
+const String analyticsDisabledNoticeMessage = '''
+  ╔════════════════════════════════════════════════════════════════════════════╗
+  ║ Anonymous analytics disabled. To enable again, run the command:            ║
+  ║ `dart --enable-analytics`                                                  ║
+  ╚════════════════════════════════════════════════════════════════════════════╝
+''';
+const String _unknown_command = '<unknown>';
+const String _appName = 'dartdev';
+const String _dartDirectoryName = '.dart';
+const String _settingsFileName = 'dartdev.json';
+const String _trackingId = 'UA-26406144-37';
+
+const String eventCategory = 'dartdev';
+const String exitCodeParam = 'exitCode';
+const String flagsParam = 'flags';
+
+Analytics instance;
+
+/// Create and return an [Analytics] instance, this value is cached and returned
+/// on subsequent calls.
+Analytics createAnalyticsInstance(bool disableAnalytics) {
+  if (instance != null) {
+    return instance;
+  }
+
+  // Dartdev tests pass a hidden 'disable-dartdev-analytics' flag which is
+  // handled here
+  if (disableAnalytics) {
+    instance = DisabledAnalytics(_trackingId, _appName);
+    return instance;
+  }
+
+  var settingsDir = getDartStorageDirectory();
+  if (settingsDir == null) {
+    // Some systems don't support user home directories; for those, fail
+    // gracefully by returning a disabled analytics object.
+    instance = DisabledAnalytics(_trackingId, _appName);
+    return instance;
+  }
+
+  if (!settingsDir.existsSync()) {
+    try {
+      settingsDir.createSync();
+    } catch (e) {
+      // If we can't create the directory for the analytics settings, fail
+      // gracefully by returning a disabled analytics object.
+      instance = DisabledAnalytics(_trackingId, _appName);
+      return instance;
+    }
+  }
+
+  var settingsFile = File(path.join(settingsDir.path, _settingsFileName));
+  instance = DartdevAnalytics(_trackingId, settingsFile, _appName);
+  return instance;
+}
+
+/// Return the first member from [args] that occurs in [allCommands], otherwise
+/// '<unknown>' is returned.
+///
+/// 'help' is special cased to have 'dart analyze help', 'dart help analyze',
+/// and 'dart analyze --help' all be recorded as a call to 'help' instead of
+/// 'help' and 'analyze'.
+String getCommandStr(List<String> args, List<String> allCommands) {
+  if (args.contains('help') || args.contains('-h') || args.contains('--help')) {
+    return 'help';
+  }
+  return args.firstWhere((arg) => allCommands.contains(arg),
+      orElse: () => _unknown_command);
+}
+
+/// Given some set of arguments and parameters, this returns a proper subset
+/// of the arguments that start with '-', joined by a space character.
+String getFlags(List<String> args) {
+  if (args == null || args.isEmpty) {
+    return '';
+  }
+  var argSubset = <String>[];
+  for (var arg in args) {
+    if (arg.startsWith('-')) {
+      if (arg.contains('=')) {
+        argSubset.add(arg.substring(0, arg.indexOf('=') + 1));
+      } else {
+        argSubset.add(arg);
+      }
+    }
+  }
+  return argSubset.join(' ');
+}
+
+/// The directory used to store the analytics settings file.
+///
+/// Typically, the directory is `~/.dart/` (and the settings file is
+/// `dartdev.json`).
+///
+/// This can return null under some conditions, including when the user's home
+/// directory does not exist.
+Directory getDartStorageDirectory() {
+  var homeDir = Directory(userHomeDir());
+  if (!homeDir.existsSync()) {
+    return null;
+  }
+  return Directory(path.join(homeDir.path, _dartDirectoryName));
+}
+
+class DartdevAnalytics extends AnalyticsImpl {
+  DartdevAnalytics(String trackingId, File settingsFile, String appName)
+      : super(
+          trackingId,
+          IOPersistentProperties.fromFile(settingsFile),
+          IOPostHandler(),
+          applicationName: appName,
+          applicationVersion: getDartVersion(),
+        );
+
+  @override
+  bool get enabled {
+    if (telemetry.isRunningOnBot()) {
+      return false;
+    }
+
+    // If there's no explicit setting (enabled or disabled) then we don't send.
+    return (properties['enabled'] as bool) ?? false;
+  }
+}
+
+class DisabledAnalytics extends AnalyticsMock {
+  @override
+  final String trackingId;
+  @override
+  final String applicationName;
+
+  DisabledAnalytics(this.trackingId, this.applicationName);
+
+  @override
+  bool get enabled => false;
+
+  @override
+  bool get firstRun => false;
+}

pkg/dartdev/lib/src/core.dart

@@ -20,7 +20,10 @@ abstract class DartdevCommand<int> extends Command {
 
   Project _project;
 
-  DartdevCommand(this._name, this._description);
+  @override
+  final bool hidden;
+
+  DartdevCommand(this._name, this._description, {this.hidden = false});
 
   @override
   String get name => _name;

pkg/dartdev/lib/src/sdk.dart

@@ -54,7 +54,4 @@ class Sdk {
 
   static String _binName(String base) =>
       Platform.isWindows ? '$base.bat' : base;
-
-  static String _exeName(String base) =>
-      Platform.isWindows ? '$base.exe' : base;
 }

pkg/dartdev/test/analytics_test.dart

@@ -0,0 +1,63 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:dartdev/src/analytics.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group('DisabledAnalytics', disabledAnalytics_object);
+  group('utils', utils);
+}
+
+void disabledAnalytics_object() {
+  test('object', () {
+    var diabledAnalytics = DisabledAnalytics('trackingId', 'appName');
+    expect(diabledAnalytics.trackingId, 'trackingId');
+    expect(diabledAnalytics.applicationName, 'appName');
+    expect(diabledAnalytics.enabled, isFalse);
+    expect(diabledAnalytics.firstRun, isFalse);
+  });
+}
+
+void utils() {
+  test('getFlags', () {
+    // base cases
+    expect(getFlags(null), '');
+    expect(getFlags(['']), '');
+    expect(getFlags(['', '']), '');
+
+    // non-trivial tests
+    expect(getFlags(['help', 'foo', 'bar']), '');
+    expect(getFlags(['--some-flag', '--some-option=1', '-v']),
+        '--some-flag --some-option= -v');
+    expect(
+        getFlags(['help', '--some-flag', '--some-option=two', '-v', 'analyze']),
+        '--some-flag --some-option= -v');
+    expect(
+        getFlags([
+          'help',
+          '--some-flag',
+          'analyze',
+          '--some-option=three=three',
+          '-v'
+        ]),
+        '--some-flag --some-option= -v');
+  });
+
+  test('getCommandStr', () {
+    var commands = <String>['help', 'foo', 'bar', 'baz'];
+
+    // base cases
+    expect(getCommandStr(['help'], commands), 'help');
+    expect(getCommandStr(['bar', 'help'], commands), 'help');
+    expect(getCommandStr(['help', 'bar'], commands), 'help');
+    expect(getCommandStr(['bar', '-h'], commands), 'help');
+    expect(getCommandStr(['bar', '--help'], commands), 'help');
+
+    // non-trivial tests
+    expect(getCommandStr(['foo'], commands), 'foo');
+    expect(getCommandStr(['bar', 'baz'], commands), 'bar');
+    expect(getCommandStr(['bazz'], commands), '<unknown>');
+  });
+}

pkg/dartdev/test/commands/flag_test.dart

@@ -17,7 +17,9 @@ void command() {
   // For each command description, assert that the values are not empty, don't
   // have trailing white space and end with a period.
   test('description formatting', () {
-    DartdevRunner([]).commands.forEach((String commandKey, Command command) {
+    DartdevRunner(['--disable-dartdev-analytics'])
+        .commands
+        .forEach((String commandKey, Command command) {
       expect(commandKey, isNotEmpty);
       expect(command.description, isNotEmpty);
       expect(command.description.split('\n').first, endsWith('.'));
@@ -27,7 +29,9 @@ void command() {
 
   // Assert that all found usageLineLengths are the same and null
   test('argParser usageLineLength isNull', () {
-    DartdevRunner([]).commands.forEach((String commandKey, Command command) {
+    DartdevRunner(['--disable-dartdev-analytics'])
+        .commands
+        .forEach((String commandKey, Command command) {
       if (command.argParser != null) {
         expect(command.argParser.usageLineLength, isNull);
       }

pkg/dartdev/test/commands/help_test.dart

@@ -21,7 +21,9 @@ void help() {
   List<String> _commandsNotTested = <String>[
     'help', // `dart help help` is redundant
   ];
-  DartdevRunner([]).commands.forEach((String commandKey, Command command) {
+  DartdevRunner(['--disable-dartdev-analytics'])
+      .commands
+      .forEach((String commandKey, Command command) {
     if (!_commandsNotTested.contains(commandKey)) {
       test('(help $commandKey == $commandKey --help)', () {
         p = project();
@@ -37,14 +39,16 @@ void help() {
   test('(help pub == pub help)', () {
     p = project();
     var result = p.runSync('help', ['pub']);
-
     var pubHelpResult = p.runSync('pub', ['help']);
+
     expect(result.stdout, contains(pubHelpResult.stdout));
     expect(result.stderr, contains(pubHelpResult.stderr));
   });
 
   test('(--help flags also have -h abbr)', () {
-    DartdevRunner([]).commands.forEach((String commandKey, Command command) {
+    DartdevRunner(['--disable-dartdev-analytics'])
+        .commands
+        .forEach((String commandKey, Command command) {
       var helpOption = command.argParser.options['help'];
       // Some commands (like pub which use
       // "argParser = ArgParser.allowAnything()") may not have the help Option

pkg/dartdev/test/test_all.dart

@@ -4,6 +4,7 @@
 
 import 'package:test/test.dart';
 
+import 'analytics_test.dart' as analytics;
 import 'commands/analyze_test.dart' as analyze;
 import 'commands/create_test.dart' as create;
 import 'commands/flag_test.dart' as flag;
@@ -19,6 +20,7 @@ import 'utils_test.dart' as utils;
 
 main() {
   group('dart', () {
+    analytics.main();
     analyze.main();
     create.main();
     flag.main();

pkg/dartdev/test/utils.dart

@@ -64,6 +64,8 @@ class TestProject {
       ...?args,
     ];
 
+    arguments.add('--disable-dartdev-analytics');
+
     return Process.runSync(
       Platform.resolvedExecutable,
       arguments,