Remove engine interference in microbenchmarks (#13034)

dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart

@@ -19,10 +19,9 @@ Future<Null> main() async {
   assert(false); // don't run this in checked mode! Use --release.
   stock_data.StockData.actuallyFetchData = false;
 
-  // This allows us to call onBeginFrame even when the engine didn't request it,
-  // and have it actually do something:
+  // We control the framePolicy below to prevent us from scheduling frames in
+  // the engine, so that the engine does not interfere with our timings.
   final LiveTestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized();
-  binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
 
   final Stopwatch watch = new Stopwatch();
   int iterations = 0;
@@ -36,6 +35,7 @@ Future<Null> main() async {
     await tester.pump(const Duration(seconds: 1)); // Complete drawer animation
 
     final Element appState = tester.element(find.byType(stocks.StocksApp));
+    binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmark;
 
     watch.start();
     while (watch.elapsed < kBenchmarkTime) {
@@ -48,7 +48,7 @@ Future<Null> main() async {
       // the two calls below.
       Timer.run(() { ui.window.onBeginFrame(new Duration(milliseconds: iterations * 16)); });
       Timer.run(() { ui.window.onDrawFrame(); });
-      await tester.idle(); // wait until the frame has run
+      await tester.idle(); // wait until the frame has run (also uses Timer.run)
       iterations += 1;
     }
     watch.stop();

dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart

@@ -18,10 +18,9 @@ const Duration kBenchmarkTime = const Duration(seconds: 15);
 Future<Null> main() async {
   stock_data.StockData.actuallyFetchData = false;
 
-  // This allows us to call onBeginFrame even when the engine didn't request it,
-  // and have it actually do something:
+  // We control the framePolicy below to prevent us from scheduling frames in
+  // the engine, so that the engine does not interfere with our timings.
   final LiveTestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized();
-  binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
 
   final Stopwatch watch = new Stopwatch();
   int iterations = 0;
@@ -37,6 +36,7 @@ Future<Null> main() async {
     final TestViewConfiguration big = new TestViewConfiguration(size: const Size(360.0, 640.0));
     final TestViewConfiguration small = new TestViewConfiguration(size: const Size(355.0, 635.0));
     final RenderView renderView = WidgetsBinding.instance.renderView;
+    binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
 
     watch.start();
     while (watch.elapsed < kBenchmarkTime) {
@@ -49,7 +49,7 @@ Future<Null> main() async {
       // the two calls below.
       Timer.run(() { ui.window.onBeginFrame(new Duration(milliseconds: iterations * 16)); });
       Timer.run(() { ui.window.onDrawFrame(); });
-      await tester.idle(); // wait until the frame has run
+      await tester.idle(); // wait until the frame has run (also uses Timer.run)
       iterations += 1;
     }
     watch.stop();

packages/flutter_test/lib/src/binding.dart

@@ -699,6 +699,20 @@ enum LiveTestWidgetsFlutterBindingFramePolicy {
   /// additional frames being pumped beyond those that the test itself requests,
   /// which can cause differences in behavior.
   fullyLive,
+
+  /// Ignore any request to schedule a frame.
+  ///
+  /// This is intended to be used by benchmarks (hence the name) that drive the
+  /// pipeline directly. It tells the binding to entirely ignore requests for a
+  /// frame to be scheduled, while still allowing frames that are pumped
+  /// directly (invoking [Window.onBeginFrame] and [Window.onDrawFrame]) to run.
+  ///
+  /// The [SchedulerBinding.hasScheduledFrame] property will never be true in
+  /// this mode. This can cause unexpected effects. For instance,
+  /// [WidgetTester.pumpAndSettle] does not function in this mode, as it relies
+  /// on the [SchedulerBinding.hasScheduledFrame] property to determine when the
+  /// application has "settled".
+  benchmark,
 }
 
 /// A variant of [TestWidgetsFlutterBinding] for executing tests in
@@ -772,6 +786,16 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
   ///   requests from the engine to be serviced, even those the test did not
   ///   explicitly pump.
   ///
+  /// * [LiveTestWidgetsFlutterBindingFramePolicy.benchmark] allows all frame
+  ///   requests from the engine to be serviced, and allows all frame requests
+  ///   that are artificially triggered to be serviced, but prevents the
+  ///   framework from requesting any frames from the engine itself. The
+  ///   [SchedulerBinding.hasScheduledFrame] property will never be true in this
+  ///   mode. This can cause unexpected effects. For instance,
+  ///   [WidgetTester.pumpAndSettle] does not function in this mode, as it
+  ///   relies on the [SchedulerBinding.hasScheduledFrame] property to determine
+  ///   when the application has "settled".
+  ///
   /// Setting this to anything other than
   /// [LiveTestWidgetsFlutterBindingFramePolicy.onlyPumps] means pumping extra
   /// frames, which might involve calling builders more, or calling paint
@@ -791,6 +815,13 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
   /// ```
   LiveTestWidgetsFlutterBindingFramePolicy framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers;
 
+  @override
+  void scheduleFrame() {
+    if (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmark)
+      return; // In benchmark mode, don't actually schedule any engine frames.
+    super.scheduleFrame();
+  }
+
   bool _doDrawThisFrame;
 
   @override
@@ -798,6 +829,7 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
     assert(_doDrawThisFrame == null);
     if (_expectingFrame ||
         (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.fullyLive) ||
+        (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmark) ||
         (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.fadePointers && _viewNeedsPaint)) {
       _doDrawThisFrame = true;
       super.handleBeginFrame(rawTimeStamp);
@@ -819,6 +851,7 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
       _pendingFrame = null;
       _expectingFrame = false;
     } else {
+      assert(framePolicy != LiveTestWidgetsFlutterBindingFramePolicy.benchmark);
       ui.window.scheduleFrame();
     }
   }

packages/flutter_test/lib/src/widget_tester.dart

@@ -245,6 +245,17 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker
     assert(duration > Duration.ZERO);
     assert(timeout != null);
     assert(timeout > Duration.ZERO);
+    assert(() {
+      final WidgetsBinding binding = this.binding;
+      if (binding is LiveTestWidgetsFlutterBinding &&
+          binding.framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmark) {
+        throw 'When using LiveTestWidgetsFlutterBindingFramePolicy.benchmark, '
+              'hasScheduledFrame is never set to true. This means that pumpAndSettle() '
+              'cannot be used, because it has no way to know if the application has '
+              'stopped registering new frames.';
+      }
+      return true;
+    }());
     int count = 0;
     return TestAsyncUtils.guard(() async {
       final DateTime endTime = binding.clock.fromNowBy(timeout);