flutter/packages/integration_test/lib/integration_test.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:developer' as developer;
import 'dart:ui';

import 'package:flutter/foundation.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
import 'package:flutter_test/flutter_test.dart';
// ignore: implementation_imports
import 'package:test_core/src/direct_run.dart';
// ignore: implementation_imports
import 'package:test_core/src/runner/engine.dart';
import 'package:vm_service/vm_service.dart' as vm;
import 'package:vm_service/vm_service_io.dart' as vm_io;

import '_callback_io.dart' if (dart.library.html) '_callback_web.dart'
    as driver_actions;
import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
import 'common.dart';
import 'src/constants.dart';
import 'src/reporter.dart';

/// Toggles the legacy reporting mechansim where results are only collected
/// for [testWidgets].
///
/// If [run] is called, this will be disabled.
bool _isUsingLegacyReporting = true;

/// Executes a block that contains tests.
///
/// Example Usage:
/// ```
/// import 'package:flutter_test/flutter_test.dart';
/// import 'package:integration_test/integration_test.dart';
///
/// void main() => run(_testMain);
///
/// void _testMain() {
///   test('A test', () {
///     expect(true, true);
///   });
/// }
/// ```
///
/// If not explicitly passed, the default [reporter] will send results over the
/// platform channel to native.
Future<void> run(
  FutureOr<void> Function() testMain, {
  Reporter reporter = const _ReporterImpl(),
}) async {
  _isUsingLegacyReporting = false;
  final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;

  // Pipe detailed exceptions within [testWidgets] to `package:test`.
  reportTestException = (FlutterErrorDetails details, String testDescription) {
    registerException('Test $testDescription failed: $details');
  };

  final Completer<List<TestResult>> resultsCompleter = Completer<List<TestResult>>();

  await directRunTests(
    testMain,
    reporterFactory: (Engine engine) => ResultReporter(engine, resultsCompleter),
  );

  final List<TestResult> results = await resultsCompleter.future;

  binding._updateTestResultState(<String, TestResult>{
    for (final TestResult result in results)
      result.methodName: result,
  });
  await reporter.report(results);
}

/// Abstract interface for a result reporter.
abstract class Reporter {
  /// Reports test results.
  ///
  /// This method will be called at the end of [run] with the [results] of
  /// running the test suite.
  Future<void> report(List<TestResult> results);
}

/// Default implementation of the reporter that sends results over to the
/// platform side.
class _ReporterImpl implements Reporter {
  const _ReporterImpl();

  @override
  Future<void> report(
    List<TestResult> results,
  ) async {
    try {
      await IntegrationTestWidgetsFlutterBinding._channel.invokeMethod<void>(
        'allTestsFinished',
        <String, dynamic>{
          'results': <String, String>{
            for (final TestResult result in results)
              result.methodName: result is Failure
                  ? _formatFailureForPlatform(result)
                  : success
          }
        },
      );
    } on MissingPluginException {
      print('Warning: integration_test test plugin was not detected.');
    }
  }
}

String _formatFailureForPlatform(Failure failure) => '${failure.error} ${failure.details}';

/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
/// on a channel to adapt them to native instrumentation test format.
class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding
    implements IntegrationTestResults {
  /// If [run] is not used, sets up a listener to report that the tests are
  /// finished when everything is torn down.
  ///
  /// This functionality is deprecated – clients are expected to use [run] to
  /// execute their tests instead.
  IntegrationTestWidgetsFlutterBinding() {
    if (!_isUsingLegacyReporting) {
      // TODO(jiahaog): Point users to use the CLI https://github.com/flutter/flutter/issues/66264.
      print('Using the legacy test result reporter, which will not catch all '
          'errors thrown in declared tests. Consider wrapping tests with '
          'https://api.flutter.dev/flutter/integration_test/run.html instead.');
      return;
    }

    tearDownAll(() async {
      _updateTestResultState(results);
      await const _ReporterImpl().report(results.values.toList());
    });

    final TestExceptionReporter oldTestExceptionReporter = reportTestException;
    reportTestException = (FlutterErrorDetails details, String testDescription) {
      results[testDescription] = Failure(
        testDescription,
        details.toString(),
        error: details.exception,
      );
      oldTestExceptionReporter(details, testDescription);
    };
  }

  void _updateTestResultState(Map<String, TestResult> results) {
    this.results = results;
    print('Test execution completed: $results');

    _allTestsPassed.complete(!results.values.any((TestResult result) => result is Failure));
    callbackManager.cleanup();
  }

  @override
  bool get overrideHttpClient => false;

  @override
  bool get registerTestTextInput => false;

  Size _surfaceSize;

  // This flag is used to print warning messages when tracking performance
  // under debug mode.
  static bool _firstRun = false;

  /// Artificially changes the surface size to `size` on the Widget binding,
  /// then flushes microtasks.
  ///
  /// Set to null to use the default surface size.
  @override
  Future<void> setSurfaceSize(Size size) {
    return TestAsyncUtils.guard<void>(() async {
      assert(inTest);
      if (_surfaceSize == size) {
        return;
      }
      _surfaceSize = size;
      handleMetricsChanged();
    });
  }

  @override
  ViewConfiguration createViewConfiguration() {
    final double devicePixelRatio = window.devicePixelRatio;
    final Size size = _surfaceSize ?? window.physicalSize / devicePixelRatio;
    return TestViewConfiguration(
      size: size,
      window: window,
    );
  }

  @override
  Completer<bool> get allTestsPassed => _allTestsPassed;
  final Completer<bool> _allTestsPassed = Completer<bool>();

  @override
  List<Failure> get failureMethodsDetails => _failures;

  /// Similar to [WidgetsFlutterBinding.ensureInitialized].
  ///
  /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
  /// initializing it if necessary.
  static WidgetsBinding ensureInitialized() {
    if (WidgetsBinding.instance == null) {
      IntegrationTestWidgetsFlutterBinding();
    }
    assert(WidgetsBinding.instance is IntegrationTestWidgetsFlutterBinding);
    return WidgetsBinding.instance;
  }

  static const MethodChannel _channel =
      MethodChannel('plugins.flutter.io/integration_test');

  /// Test results that will be populated after the tests have completed.
  @visibleForTesting
  Map<String, TestResult> results = <String, TestResult>{};

  List<Failure> get _failures => results.values.whereType<Failure>().toList();

  /// The extra data for the reported result.
  ///
  /// The values in `reportData` must be json-serializable objects or `null`.
  /// If it's `null`, no extra data is attached to the result.
  ///
  /// The default value is `null`.
  @override
  Map<String, dynamic> reportData;

  /// Manages callbacks received from driver side and commands send to driver
  /// side.
  final CallbackManager callbackManager = driver_actions.callbackManager;

  /// Taking a screenshot.
  ///
  /// Called by test methods. Implementation differs for each platform.
  Future<void> takeScreenshot(String screenshotName) async {
    await callbackManager.takeScreenshot(screenshotName);
  }

  /// The callback function to response the driver side input.
  @visibleForTesting
  Future<Map<String, dynamic>> callback(Map<String, String> params) async {
    return await callbackManager.callback(
        params, this /* as IntegrationTestResults */);
  }

  // Emulates the Flutter driver extension, returning 'pass' or 'fail'.
  @override
  void initServiceExtensions() {
    super.initServiceExtensions();

    if (kIsWeb) {
      registerWebServiceExtension(callback);
    }

    registerServiceExtension(name: 'driver', callback: callback);
  }

  @override
  Future<void> runTest(
    Future<void> testBody(),
    VoidCallback invariantTester, {
    String description = '',
    Duration timeout,
  }) async {
    await super.runTest(
      testBody,
      invariantTester,
      description: description,
      timeout: timeout,
    );
    results[description] ??= Success(description);
  }

  vm.VmService _vmService;

  /// Initialize the [vm.VmService] settings for the timeline.
  @visibleForTesting
  Future<void> enableTimeline({
    List<String> streams = const <String>['all'],
    @visibleForTesting vm.VmService vmService,
  }) async {
    assert(streams != null);
    assert(streams.isNotEmpty);
    if (vmService != null) {
      _vmService = vmService;
    }
    if (_vmService == null) {
      final developer.ServiceProtocolInfo info =
          await developer.Service.getInfo();
      assert(info.serverUri != null);
      _vmService = await vm_io.vmServiceConnectUri(
        'ws://localhost:${info.serverUri.port}${info.serverUri.path}ws',
      );
    }
    await _vmService.setVMTimelineFlags(streams);
  }

  /// Runs [action] and returns a [vm.Timeline] trace for it.
  ///
  /// Waits for the `Future` returned by [action] to complete prior to stopping
  /// the trace.
  ///
  /// The `streams` parameter limits the recorded timeline event streams to only
  /// the ones listed. By default, all streams are recorded.
  /// See `timeline_streams` in
  /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/master/runtime/vm/timeline.cc)
  ///
  /// If [retainPriorEvents] is true, retains events recorded prior to calling
  /// [action]. Otherwise, prior events are cleared before calling [action]. By
  /// default, prior events are cleared.
  Future<vm.Timeline> traceTimeline(
    Future<dynamic> action(), {
    List<String> streams = const <String>['all'],
    bool retainPriorEvents = false,
  }) async {
    await enableTimeline(streams: streams);
    if (retainPriorEvents) {
      await action();
      return await _vmService.getVMTimeline();
    }

    await _vmService.clearVMTimeline();
    final vm.Timestamp startTime = await _vmService.getVMTimelineMicros();
    await action();
    final vm.Timestamp endTime = await _vmService.getVMTimelineMicros();
    return await _vmService.getVMTimeline(
      timeOriginMicros: startTime.timestamp,
      timeExtentMicros: endTime.timestamp,
    );
  }

  /// This is a convenience wrap of [traceTimeline] and send the result back to
  /// the host for the [flutter_driver] style tests.
  ///
  /// This records the timeline during `action` and adds the result to
  /// [reportData] with `reportKey`. The [reportData] contains extra information
  /// from the test other than test success/fail. It will be passed back to the
  /// host and be processed by the [ResponseDataCallback] defined in
  /// [integration_test_driver.integrationDriver]. By default it will be written
  /// to `build/integration_response_data.json` with the key `timeline`.
  ///
  /// For tests with multiple calls of this method, `reportKey` needs to be a
  /// unique key, otherwise the later result will override earlier one.
  ///
  /// The `streams` and `retainPriorEvents` parameters are passed as-is to
  /// [traceTimeline].
  Future<void> traceAction(
    Future<dynamic> action(), {
    List<String> streams = const <String>['all'],
    bool retainPriorEvents = false,
    String reportKey = 'timeline',
  }) async {
    final vm.Timeline timeline = await traceTimeline(
      action,
      streams: streams,
      retainPriorEvents: retainPriorEvents,
    );
    reportData ??= <String, dynamic>{};
    reportData[reportKey] = timeline.toJson();
  }

  /// Watches the [FrameTiming] during `action` and report it to the binding
  /// with key `reportKey`.
  ///
  /// This can be used to implement performance tests previously using
  /// [traceAction] and [TimelineSummary] from [flutter_driver]
  Future<void> watchPerformance(
    Future<void> action(), {
    String reportKey = 'performance',
  }) async {
    assert(() {
      if (_firstRun) {
        debugPrint(kDebugWarning);
        _firstRun = false;
      }
      return true;
    }());

    // The engine could batch FrameTimings and send them only once per second.
    // Delay for a sufficient time so either old FrameTimings are flushed and not
    // interfering our measurements here, or new FrameTimings are all reported.
    // TODO(CareF): remove this when flush FrameTiming is readly in engine.
    //              See https://github.com/flutter/flutter/issues/64808
    //              and https://github.com/flutter/flutter/issues/67593
    Future<void> delayForFrameTimings() => Future<void>.delayed(const Duration(seconds: 2));

    await delayForFrameTimings(); // flush old FrameTimings
    final List<FrameTiming> frameTimings = <FrameTiming>[];
    final TimingsCallback watcher = frameTimings.addAll;
    addTimingsCallback(watcher);
    await action();
    await delayForFrameTimings(); // make sure all FrameTimings are reported
    removeTimingsCallback(watcher);
    final FrameTimingSummarizer frameTimes =
        FrameTimingSummarizer(frameTimings);
    reportData ??= <String, dynamic>{};
    reportData[reportKey] = frameTimes.summary;
  }

  @override
  Timeout get defaultTestTimeout => _defaultTestTimeout ?? super.defaultTestTimeout;

  /// Configures the default timeout for [testWidgets].
  ///
  /// See [TestWidgetsFlutterBinding.defaultTestTimeout] for more details.
  set defaultTestTimeout(Timeout timeout) => _defaultTestTimeout = timeout;
  Timeout _defaultTestTimeout;
}