[vm/inlining/jit] Allow inlining of force-optimized functions tagged with 'vm:idempotent' pragma.w

If inlined force-optimized function has to deoptimize, it deoptimizes to before-the-call.

BUG=https://github.com/dart-lang/sdk/issues/38985
TEST=Inliner_InlineForceOptimized, Inliner_InlineAndRunForceOptimized

Change-Id: I6aa4bf1b7ce4d19791132d252e01fc38394c9fcc
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/317522
Reviewed-by: Martin Kustermann <kustermann@google.com>
Reviewed-by: Slava Egorov <vegorov@google.com>
Commit-Queue: Alexander Aprelev <aam@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>

runtime/docs/pragmas.md

@@ -37,6 +37,7 @@ These pragmas can cause unsound behavior if used incorrectly and therefore are o
 | --- | --- |
 | `vm:exact-result-type` | [Declaring an exact result type of a method](compiler/pragmas_recognized_by_compiler.md#providing-an-exact-result-type) |
 | `vm:recognized` | [Marking this as a recognized method](compiler/pragmas_recognized_by_compiler.md#marking-recognized-methods) |
+| `vm:idempotent` | Method marked with this pragma can be repeated or restarted multiple times without change to its effect. Loading, storing of memory values are examples of this, while reads and writes from file are examples of non-idempotent methods. At present, use of this pragma is limited to driving inlining of force-optimized functions. |
 
 ## Pragmas ignored in user code
 

runtime/vm/compiler/backend/flow_graph_checker.cc

@@ -121,54 +121,6 @@ static bool IsControlFlow(Instruction* instruction) {
          instruction->IsTailCall();
 }
 
-// Asserts that arguments appear in environment at the right place.
-static void AssertArgumentsInEnv(FlowGraph* flow_graph, Definition* call) {
-  Environment* env = call->env();
-  if (env == nullptr) {
-    // Environments can be removed by EliminateEnvironments pass and
-    // are not present before SSA.
-  } else if (flow_graph->function().IsIrregexpFunction()) {
-    // TODO(dartbug.com/38577): cleanup regexp pipeline too....
-  } else {
-    // Otherwise, the trailing environment entries must
-    // correspond directly with the arguments.
-    const intptr_t env_count = env->Length();
-    const intptr_t arg_count = call->ArgumentCount();
-    // Some calls (e.g. closure calls) have more inputs than actual arguments.
-    // Those extra inputs will be consumed from the stack before the call.
-    const intptr_t after_args_input_count = call->env()->LazyDeoptPruneCount();
-    ASSERT1((arg_count + after_args_input_count) <= env_count, call);
-    const intptr_t env_base = env_count - arg_count - after_args_input_count;
-    for (intptr_t i = 0; i < arg_count; i++) {
-      if (call->HasMoveArguments()) {
-        ASSERT1(call->ArgumentAt(i) == env->ValueAt(env_base + i)
-                                           ->definition()
-                                           ->AsMoveArgument()
-                                           ->value()
-                                           ->definition(),
-                call);
-      } else {
-        // Redefinition instructions and boxing/unboxing are inserted
-        // without updating environment uses (FlowGraph::RenameDominatedUses,
-        // FlowGraph::InsertConversionsFor).
-        // Also, constants may belong to different blocks (e.g. function entry
-        // and graph entry).
-        Definition* arg_def =
-            call->ArgumentAt(i)->OriginalDefinitionIgnoreBoxingAndConstraints();
-        Definition* env_def =
-            env->ValueAt(env_base + i)
-                ->definition()
-                ->OriginalDefinitionIgnoreBoxingAndConstraints();
-        ASSERT2((arg_def == env_def) ||
-                    (arg_def->IsConstant() && env_def->IsConstant() &&
-                     arg_def->AsConstant()->value().ptr() ==
-                         env_def->AsConstant()->value().ptr()),
-                arg_def, env_def);
-      }
-    }
-  }
-}
-
 void FlowGraphChecker::VisitBlocks() {
   const GrowableArray<BlockEntryInstr*>& preorder = flow_graph_->preorder();
   const GrowableArray<BlockEntryInstr*>& postorder = flow_graph_->postorder();
@@ -295,6 +247,14 @@ void FlowGraphChecker::VisitInstruction(Instruction* instruction) {
   ASSERT1(!instruction->MayThrow() || instruction->IsTailCall() ||
               instruction->deopt_id() != DeoptId::kNone,
           instruction);
+
+  // Any instruction that can eagerly deopt cannot come from a force-optimized
+  // function.
+  if (instruction->ComputeCanDeoptimize()) {
+    ASSERT2(!flow_graph_->function().ForceOptimize(), instruction,
+            &flow_graph_->function());
+  }
+
 #endif  // !defined(DART_PRECOMPILER)
 
   // If checking token positions and the flow graph has an inlining ID,
@@ -524,16 +484,79 @@ void FlowGraphChecker::VisitRedefinition(RedefinitionInstr* def) {
   ASSERT1(def->value()->definition() != def, def);
 }
 
+// Asserts that arguments appear in environment at the right place.
+void FlowGraphChecker::AssertArgumentsInEnv(Definition* call) {
+  const auto& function = flow_graph_->function();
+  Environment* env = call->env();
+  if (env == nullptr) {
+    // Environments can be removed by EliminateEnvironments pass and
+    // are not present before SSA.
+  } else if (function.IsIrregexpFunction()) {
+    // TODO(dartbug.com/38577): cleanup regexp pipeline too....
+  } else {
+    // Otherwise, the trailing environment entries must
+    // correspond directly with the arguments.
+    const intptr_t env_count = env->Length();
+    const intptr_t arg_count = call->ArgumentCount();
+    // Some calls (e.g. closure calls) have more inputs than actual arguments.
+    // Those extra inputs will be consumed from the stack before the call.
+    const intptr_t after_args_input_count = call->env()->LazyDeoptPruneCount();
+    ASSERT1((arg_count + after_args_input_count) <= env_count, call);
+    const intptr_t env_base = env_count - arg_count - after_args_input_count;
+    for (intptr_t i = 0; i < arg_count; i++) {
+      if (call->HasMoveArguments()) {
+        ASSERT1(call->ArgumentAt(i) == env->ValueAt(env_base + i)
+                                           ->definition()
+                                           ->AsMoveArgument()
+                                           ->value()
+                                           ->definition(),
+                call);
+      } else {
+        if (env->LazyDeoptToBeforeDeoptId()) {
+          // The deoptimization environment attached to this [call] instruction may
+          // no longer target the same call in unoptimized code. It may target anything.
+          //
+          // As a result, we cannot assume the arguments we pass to the call will also be
+          // in the deopt environment.
+          //
+          // This currently can happen in inlined force-optimized instructions.
+          ASSERT(call->inlining_id() > 0);
+          const auto& function = *inline_id_to_function_[call->inlining_id()];
+          ASSERT(function.ForceOptimize());
+          return;
+        }
+
+        // Redefinition instructions and boxing/unboxing are inserted
+        // without updating environment uses (FlowGraph::RenameDominatedUses,
+        // FlowGraph::InsertConversionsFor).
+        // Also, constants may belong to different blocks (e.g. function entry
+        // and graph entry).
+        Definition* arg_def =
+            call->ArgumentAt(i)->OriginalDefinitionIgnoreBoxingAndConstraints();
+        Definition* env_def =
+            env->ValueAt(env_base + i)
+                ->definition()
+                ->OriginalDefinitionIgnoreBoxingAndConstraints();
+        ASSERT2((arg_def == env_def) ||
+                    (arg_def->IsConstant() && env_def->IsConstant() &&
+                     arg_def->AsConstant()->value().ptr() ==
+                         env_def->AsConstant()->value().ptr()),
+                arg_def, env_def);
+      }
+    }
+  }
+}
+
 void FlowGraphChecker::VisitClosureCall(ClosureCallInstr* call) {
-  AssertArgumentsInEnv(flow_graph_, call);
+  AssertArgumentsInEnv(call);
 }
 
 void FlowGraphChecker::VisitStaticCall(StaticCallInstr* call) {
-  AssertArgumentsInEnv(flow_graph_, call);
+  AssertArgumentsInEnv(call);
 }
 
 void FlowGraphChecker::VisitInstanceCall(InstanceCallInstr* call) {
-  AssertArgumentsInEnv(flow_graph_, call);
+  AssertArgumentsInEnv(call);
   // Force-optimized functions may not have instance calls inside them because
   // we do not reset ICData for these.
   ASSERT(!flow_graph_->function().ForceOptimize());
@@ -541,7 +564,7 @@ void FlowGraphChecker::VisitInstanceCall(InstanceCallInstr* call) {
 
 void FlowGraphChecker::VisitPolymorphicInstanceCall(
     PolymorphicInstanceCallInstr* call) {
-  AssertArgumentsInEnv(flow_graph_, call);
+  AssertArgumentsInEnv(call);
   // Force-optimized functions may not have instance calls inside them because
   // we do not reset ICData for these.
   ASSERT(!flow_graph_->function().ForceOptimize());

runtime/vm/compiler/backend/flow_graph_checker.h

@@ -64,6 +64,7 @@ class FlowGraphChecker : public FlowGraphVisitor {
   void VisitIndirectGoto(IndirectGotoInstr* jmp) override;
   void VisitBranch(BranchInstr* branch) override;
   void VisitRedefinition(RedefinitionInstr* def) override;
+  void AssertArgumentsInEnv(Definition* call);
   void VisitClosureCall(ClosureCallInstr* call) override;
   void VisitStaticCall(StaticCallInstr* call) override;
   void VisitInstanceCall(InstanceCallInstr* call) override;

runtime/vm/compiler/backend/il.cc

@@ -1500,12 +1500,6 @@ void Instruction::InheritDeoptTarget(Zone* zone, Instruction* other) {
   other->env()->DeepCopyTo(zone, this);
 }
 
-void BranchInstr::InheritDeoptTarget(Zone* zone, Instruction* other) {
-  ASSERT(env() == nullptr);
-  Instruction::InheritDeoptTarget(zone, other);
-  comparison()->SetDeoptId(*this);
-}
-
 bool Instruction::IsDominatedBy(Instruction* dom) {
   BlockEntryInstr* block = GetBlock();
   BlockEntryInstr* dom_block = dom->GetBlock();

runtime/vm/compiler/backend/il.h

@@ -1296,7 +1296,7 @@ class Instruction : public ZoneAllocated {
     return false;
   }
 
-  virtual void InheritDeoptTarget(Zone* zone, Instruction* other);
+  void InheritDeoptTarget(Zone* zone, Instruction* other);
 
   bool NeedsEnvironment() const {
     return ComputeCanDeoptimize() || ComputeCanDeoptimizeAfterCall() ||
@@ -1360,7 +1360,7 @@ class Instruction : public ZoneAllocated {
   // Fetch deopt id without checking if this computation can deoptimize.
   intptr_t GetDeoptId() const { return deopt_id_; }
 
-  void CopyDeoptIdFrom(const Instruction& instr) {
+  virtual void CopyDeoptIdFrom(const Instruction& instr) {
     deopt_id_ = instr.deopt_id_;
   }
 
@@ -3776,7 +3776,10 @@ class BranchInstr : public Instruction {
   }
   TargetEntryInstr* constant_target() const { return constant_target_; }
 
-  virtual void InheritDeoptTarget(Zone* zone, Instruction* other);
+  virtual void CopyDeoptIdFrom(const Instruction& instr) {
+    Instruction::CopyDeoptIdFrom(instr);
+    comparison()->CopyDeoptIdFrom(instr);
+  }
 
   virtual bool MayThrow() const { return comparison()->MayThrow(); }
 
@@ -5209,6 +5212,11 @@ class IfThenElseInstr : public Definition {
 
   virtual bool MayThrow() const { return comparison()->MayThrow(); }
 
+  virtual void CopyDeoptIdFrom(const Instruction& instr) {
+    Definition::CopyDeoptIdFrom(instr);
+    comparison()->CopyDeoptIdFrom(instr);
+  }
+
   PRINT_OPERANDS_TO_SUPPORT
 
 #define FIELD_LIST(F)                                                          \
@@ -10170,6 +10178,11 @@ class CheckConditionInstr : public Instruction {
 
   virtual bool MayThrow() const { return false; }
 
+  virtual void CopyDeoptIdFrom(const Instruction& instr) {
+    Instruction::CopyDeoptIdFrom(instr);
+    comparison()->CopyDeoptIdFrom(instr);
+  }
+
   PRINT_OPERANDS_TO_SUPPORT
 
 #define FIELD_LIST(F) F(ComparisonInstr*, comparison_)

runtime/vm/compiler/backend/il_test_helper.cc

@@ -146,14 +146,15 @@ FlowGraph* TestPipeline::RunPasses(
     BlockScheduler::AssignEdgeWeights(flow_graph_);
   }
 
-  SpeculativeInliningPolicy speculative_policy(/*enable_suppression=*/false);
+  pass_state_ =
-  pass_state_ = new CompilerPassState(thread, flow_graph_, &speculative_policy);
+      new CompilerPassState(thread, flow_graph_, speculative_policy_.get());
   pass_state_->reorder_blocks = reorder_blocks;
 
   if (optimized) {
-    JitCallSpecializer jit_call_specializer(flow_graph_, &speculative_policy);
+    JitCallSpecializer jit_call_specializer(flow_graph_,
-    AotCallSpecializer aot_call_specializer(/*precompiler=*/nullptr,
+                                            speculative_policy_.get());
-                                            flow_graph_, &speculative_policy);
+    AotCallSpecializer aot_call_specializer(
+        /*precompiler=*/nullptr, flow_graph_, speculative_policy_.get());
     if (mode_ == CompilerPass::kAOT) {
       pass_state_->call_specializer = &aot_call_specializer;
     } else {
@@ -173,11 +174,10 @@ FlowGraph* TestPipeline::RunPasses(
 
 void TestPipeline::RunAdditionalPasses(
     std::initializer_list<CompilerPass::Id> passes) {
-  SpeculativeInliningPolicy speculative_policy(/*enable_suppression=*/false);
+  JitCallSpecializer jit_call_specializer(flow_graph_,
-
+                                          speculative_policy_.get());
-  JitCallSpecializer jit_call_specializer(flow_graph_, &speculative_policy);
   AotCallSpecializer aot_call_specializer(/*precompiler=*/nullptr, flow_graph_,
-                                          &speculative_policy);
+                                          speculative_policy_.get());
   if (mode_ == CompilerPass::kAOT) {
     pass_state_->call_specializer = &aot_call_specializer;
   } else {

runtime/vm/compiler/backend/il_test_helper.h

@@ -14,6 +14,7 @@
 #include "platform/allocation.h"
 #include "vm/compiler/backend/flow_graph.h"
 #include "vm/compiler/backend/il.h"
+#include "vm/compiler/backend/inliner.h"
 #include "vm/compiler/compiler_pass.h"
 #include "vm/compiler/compiler_state.h"
 #include "vm/compiler/jit/compiler.h"
@@ -79,6 +80,8 @@ class TestPipeline : public ValueObject {
                         mode == CompilerPass::PipelineMode::kAOT,
                         is_optimizing,
                         CompilerState::ShouldTrace(function)),
+        speculative_policy_(std::unique_ptr<SpeculativeInliningPolicy>(
+            new SpeculativeInliningPolicy(/*enable_suppresson=*/false))),
         mode_(mode) {}
   ~TestPipeline() { delete pass_state_; }
 
@@ -99,6 +102,7 @@ class TestPipeline : public ValueObject {
   const Function& function_;
   Thread* thread_;
   CompilerState compiler_state_;
+  std::unique_ptr<SpeculativeInliningPolicy> speculative_policy_;
   CompilerPass::PipelineMode mode_;
   ZoneGrowableArray<const ICData*>* ic_data_array_ = nullptr;
   ParsedFunction* parsed_function_ = nullptr;

runtime/vm/compiler/backend/inliner.cc

@@ -882,10 +882,11 @@ static void ReplaceParameterStubs(Zone* zone,
   //    }
   //    class B extends A<X> { void f(X v) { ... } }
   //
-  // Conside when B.f is inlined into a callsite in A.g (e.g. due to polymorphic
+  // Consider when B.f is inlined into a callsite in A.g (e.g. due to
-  // inlining). v is known to be X within the body of B.f, but not guaranteed to
+  // polymorphic inlining). v is known to be X within the body of B.f, but not
-  // be X outside of it. Thus we must ensure that all operations with v that
+  // guaranteed to be X outside of it. Thus we must ensure that all operations
-  // depend on its type being X are pinned to stay within the inlined body.
+  // with v that depend on its type being X are pinned to stay within the
+  // inlined body.
   //
   // We achieve that by inserting redefinitions for parameters which potentially
   // have narrower types in callee compared to those in the interface target of
@@ -1159,23 +1160,22 @@ class CallSiteInliner : public ValueObject {
     ConstantInstr* constant = argument->definition()->AsConstant();
     if (constant != nullptr) {
       return graph->GetConstant(constant->value());
-    } else {
-      ParameterInstr* param =
-          new (Z) ParameterInstr(/*env_index=*/i, /*param_index=*/i, -1,
-                                 graph->graph_entry(), kNoRepresentation);
-      if (i >= 0) {
-        // Compute initial parameter type using static and inferred types
-        // and combine it with an argument type from the caller.
-        param->UpdateType(
-            *CompileType::ComputeRefinedType(param->Type(), argument->Type()));
-      } else {
-        // Parameter stub for function type arguments.
-        // It doesn't correspond to a real parameter, so don't try to
-        // query its static/inferred type.
-        param->UpdateType(*argument->Type());
-      }
-      return param;
     }
+    ParameterInstr* param = new (Z) ParameterInstr(
+        /*env_index=*/i, /*param_index=*/i,
+        /*param_offset=*/-1, graph->graph_entry(), kNoRepresentation);
+    if (i >= 0) {
+      // Compute initial parameter type using static and inferred types
+      // and combine it with an argument type from the caller.
+      param->UpdateType(
+          *CompileType::ComputeRefinedType(param->Type(), argument->Type()));
+    } else {
+      // Parameter stub for function type arguments.
+      // It doesn't correspond to a real parameter, so don't try to
+      // query its static/inferred type.
+      param->UpdateType(*argument->Type());
+    }
+    return param;
   }
 
   bool TryInlining(const Function& function,
@@ -1243,10 +1243,10 @@ class CallSiteInliner : public ValueObject {
 
     // Type feedback may have been cleared for this function (ClearICDataArray),
     // but we need it for inlining.
-    if (!CompilerState::Current().is_aot() &&
+    if (!CompilerState::Current().is_aot() && !function.ForceOptimize() &&
-        (function.ic_data_array() == Array::null())) {
+        function.ic_data_array() == Array::null()) {
       TRACE_INLINING(THR_Print("     Bailout: type feedback cleared\n"));
-      PRINT_INLINING_TREE("Not compiled", &call_data->caller, &function,
+      PRINT_INLINING_TREE("No ICData", &call_data->caller, &function,
                           call_data->call);
       return false;
     }
@@ -1323,7 +1323,7 @@ class CallSiteInliner : public ValueObject {
         ZoneGrowableArray<const ICData*>* ic_data_array =
             new (Z) ZoneGrowableArray<const ICData*>();
         const bool clone_ic_data = Compiler::IsBackgroundCompilation();
-        ASSERT(CompilerState::Current().is_aot() ||
+        ASSERT(CompilerState::Current().is_aot() || function.ForceOptimize() ||
                function.ic_data_array() != Array::null());
         function.RestoreICDataMap(ic_data_array, clone_ic_data);
 
@@ -1348,10 +1348,11 @@ class CallSiteInliner : public ValueObject {
         } else if (call_data->call->IsClosureCall()) {
           // Closure functions only have one entry point.
         }
+        // context_level_array=nullptr below means we are not building var desc.
         kernel::FlowGraphBuilder builder(
-            parsed_function, ic_data_array, /* not building var desc */ nullptr,
+            parsed_function, ic_data_array, /*context_level_array=*/nullptr,
             exit_collector,
-            /* optimized = */ true, Compiler::kNoOSRDeoptId,
+            /*optimized=*/true, Compiler::kNoOSRDeoptId,
             caller_graph_->max_block_id() + 1,
             entry_kind == Code::EntryKind::kUnchecked);
         {
@@ -1485,7 +1486,11 @@ class CallSiteInliner : public ValueObject {
             CompilerPassState state(Thread::Current(), callee_graph,
                                     inliner_->speculative_policy_);
             state.call_specializer = &call_specializer;
-            CompilerPass::RunInliningPipeline(CompilerPass::kAOT, &state);
+            if (function.ForceOptimize()) {
+              CompilerPass::RunForceOptimizedInliningPipeline(&state);
+            } else {
+              CompilerPass::RunInliningPipeline(CompilerPass::kAOT, &state);
+            }
 #else
             UNREACHABLE();
 #endif  // defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32)
@@ -1496,7 +1501,11 @@ class CallSiteInliner : public ValueObject {
             CompilerPassState state(Thread::Current(), callee_graph,
                                     inliner_->speculative_policy_);
             state.call_specializer = &call_specializer;
-            CompilerPass::RunInliningPipeline(CompilerPass::kJIT, &state);
+            if (function.ForceOptimize()) {
+              CompilerPass::RunForceOptimizedInliningPipeline(&state);
+            } else {
+              CompilerPass::RunInliningPipeline(CompilerPass::kJIT, &state);
+            }
           }
         }
 
@@ -1727,6 +1736,27 @@ class CallSiteInliner : public ValueObject {
     InlineExitCollector* exit_collector = call_data->exit_collector;
     exit_collector->PrepareGraphs(callee_graph);
     ReplaceParameterStubs(zone(), caller_graph_, call_data, nullptr);
+
+    // Inlined force-optimized idempotent functions get deopt-id and
+    // environment from the call, so when deoptimized, the call is repeated.
+    if (callee_graph->function().ForceOptimize()) {
+      // We should only reach here if `Function::CanBeInlined()` returned true,
+      // which only happens if the force-optimized function is idempotent.
+      ASSERT(CompilerState::Current().is_aot() ||
+             callee_graph->function().IsIdempotent());
+      for (BlockIterator block_it = callee_graph->postorder_iterator();
+           !block_it.Done(); block_it.Advance()) {
+        for (ForwardInstructionIterator it(block_it.Current()); !it.Done();
+             it.Advance()) {
+          Instruction* current = it.Current();
+          if (current->env() != nullptr) {
+            call_data->call->env()->DeepCopyTo(zone(), current);
+            current->CopyDeoptIdFrom(*call_data->call);
+            current->env()->MarkAsLazyDeoptToBeforeDeoptId();
+          }
+        }
+      }
+    }
     exit_collector->ReplaceCall(callee_function_entry);
 
     ASSERT(!call_data->call->HasMoveArguments());

runtime/vm/compiler/backend/inliner_test.cc

@@ -8,6 +8,7 @@
 #include "vm/compiler/backend/il_printer.h"
 #include "vm/compiler/backend/il_test_helper.h"
 #include "vm/compiler/compiler_pass.h"
+#include "vm/debugger_api_impl_test.h"
 #include "vm/object.h"
 #include "vm/unit_test.h"
 
@@ -368,4 +369,302 @@ ISOLATE_UNIT_TEST_CASE(Inliner_List_of_inlined) {
 
 #endif  // defined(DART_PRECOMPILER)
 
+// Test that when force-optimized functions get inlined, deopt_id and
+// environment for instructions coming from those functions get overwritten
+// by deopt_id and environment from the call itself.
+ISOLATE_UNIT_TEST_CASE(Inliner_InlineForceOptimized) {
+  const char* kScript = R"(
+    import 'dart:ffi';
+
+    @pragma('vm:never-inline')
+    int foo(int x) {
+      dynamic ptr = Pointer.fromAddress(x);
+      return x + ptr.hashCode;
+    }
+    main() {
+      int r = 0;
+      for (int i = 0; i < 1000; i++) {
+        r += foo(r);
+      }
+      return r;
+    }
+  )";
+
+  const auto& root_library = Library::Handle(LoadTestScript(kScript));
+  const auto& function = Function::Handle(GetFunction(root_library, "foo"));
+
+  Invoke(root_library, "main");
+
+  TestPipeline pipeline(function, CompilerPass::kJIT);
+  FlowGraph* flow_graph = pipeline.RunPasses({
+      CompilerPass::kComputeSSA,
+      CompilerPass::kApplyICData,
+      CompilerPass::kTryOptimizePatterns,
+      CompilerPass::kSetOuterInliningId,
+      CompilerPass::kTypePropagation,
+      CompilerPass::kApplyClassIds,
+  });
+
+  auto entry = flow_graph->graph_entry()->normal_entry();
+  StaticCallInstr* call_instr = nullptr;
+  {
+    ILMatcher cursor(flow_graph, entry);
+    RELEASE_ASSERT(cursor.TryMatch({
+        {kMoveGlob},
+        {kMatchAndMoveStaticCall, &call_instr},
+    }));
+    EXPECT(strcmp(call_instr->function().UserVisibleNameCString(),
+                  "Pointer.fromAddress") == 0);
+  }
+
+  pipeline.RunAdditionalPasses({
+      CompilerPass::kInlining,
+  });
+
+  AllocateObjectInstr* allocate_object_instr = nullptr;
+  {
+    ILMatcher cursor(flow_graph, entry);
+    RELEASE_ASSERT(cursor.TryMatch({
+        {kMoveGlob},
+        {kMatchAndMoveAllocateObject, &allocate_object_instr},
+    }));
+  }
+
+  // Ensure that AllocateObject instruction that came from force-optimized
+  // function has deopt and environment taken from the Call instruction.
+  EXPECT(call_instr->deopt_id() == allocate_object_instr->deopt_id());
+  EXPECT(DeoptId::IsDeoptBefore(call_instr->deopt_id()));
+
+  auto allocate_object_instr_env = allocate_object_instr->env();
+  EXPECT(allocate_object_instr_env->LazyDeoptToBeforeDeoptId());
+  EXPECT(allocate_object_instr_env->Outermost()->GetDeoptId() ==
+         call_instr->deopt_id());
+  const auto call_instr_env = call_instr->env();
+  const intptr_t call_first_index =
+      call_instr_env->Length() - call_instr->InputCount();
+  const intptr_t allocate_first_index =
+      allocate_object_instr_env->Length() - call_instr->InputCount();
+  for (intptr_t i = 0; i < call_instr->InputCount(); i++) {
+    EXPECT(call_instr_env->ValueAt(call_first_index + i)->definition() ==
+           allocate_object_instr_env->ValueAt(allocate_first_index + i)
+               ->definition());
+  }
+}
+
+static void TestPrint(Dart_NativeArguments args) {
+  Dart_EnterScope();
+  Dart_Handle handle = Dart_GetNativeArgument(args, 0);
+  const char* str = nullptr;
+  Dart_StringToCString(handle, &str);
+  OS::Print("%s\n", str);
+  Dart_ExitScope();
+}
+
+void InspectStack(Dart_NativeArguments args) {
+#ifndef PRODUCT
+  Dart_EnterScope();
+
+  Dart_StackTrace stacktrace;
+  Dart_Handle result = Dart_GetStackTrace(&stacktrace);
+  EXPECT_VALID(result);
+
+  intptr_t frame_count = 0;
+  result = Dart_StackTraceLength(stacktrace, &frame_count);
+  EXPECT_VALID(result);
+  EXPECT_EQ(3, frame_count);
+  // Test something bigger than the preallocated size to verify nothing was
+  // truncated.
+  EXPECT(102 > StackTrace::kPreallocatedStackdepth);
+
+  Dart_Handle function_name;
+  Dart_Handle script_url;
+  intptr_t line_number = 0;
+  intptr_t column_number = 0;
+  const char* cstr = "";
+  const char* test_lib = "file:///test-lib";
+
+  // Top frame is InspectStack().
+  Dart_ActivationFrame frame;
+  result = Dart_GetActivationFrame(stacktrace, 0, &frame);
+  EXPECT_VALID(result);
+  result = Dart_ActivationFrameInfo(frame, &function_name, &script_url,
+                                    &line_number, &column_number);
+  EXPECT_VALID(result);
+  Dart_StringToCString(function_name, &cstr);
+  EXPECT_STREQ("InspectStack", cstr);
+  Dart_StringToCString(script_url, &cstr);
+  EXPECT_STREQ(test_lib, cstr);
+  EXPECT_EQ(11, line_number);
+  EXPECT_EQ(24, column_number);
+
+  // Second frame is foo() positioned at call to InspectStack().
+  result = Dart_GetActivationFrame(stacktrace, 1, &frame);
+  EXPECT_VALID(result);
+  result = Dart_ActivationFrameInfo(frame, &function_name, &script_url,
+                                    &line_number, &column_number);
+  EXPECT_VALID(result);
+  Dart_StringToCString(function_name, &cstr);
+  EXPECT_STREQ("foo", cstr);
+  Dart_StringToCString(script_url, &cstr);
+  EXPECT_STREQ(test_lib, cstr);
+
+  // Bottom frame positioned at main().
+  result = Dart_GetActivationFrame(stacktrace, frame_count - 1, &frame);
+  EXPECT_VALID(result);
+  result = Dart_ActivationFrameInfo(frame, &function_name, &script_url,
+                                    &line_number, &column_number);
+  EXPECT_VALID(result);
+  Dart_StringToCString(function_name, &cstr);
+  EXPECT_STREQ("main", cstr);
+  Dart_StringToCString(script_url, &cstr);
+  EXPECT_STREQ(test_lib, cstr);
+
+  // Out-of-bounds frames.
+  result = Dart_GetActivationFrame(stacktrace, frame_count, &frame);
+  EXPECT(Dart_IsError(result));
+  result = Dart_GetActivationFrame(stacktrace, -1, &frame);
+  EXPECT(Dart_IsError(result));
+
+  Dart_SetReturnValue(args, Dart_NewInteger(42));
+  Dart_ExitScope();
+#endif  // !PRODUCT
+}
+
+static Dart_NativeFunction PrintAndInspectResolver(Dart_Handle name,
+                                                   int argument_count,
+                                                   bool* auto_setup_scope) {
+  ASSERT(auto_setup_scope != nullptr);
+  *auto_setup_scope = true;
+  const char* cstr = nullptr;
+  Dart_Handle result = Dart_StringToCString(name, &cstr);
+  EXPECT_VALID(result);
+  if (strcmp(cstr, "testPrint") == 0) {
+    return &TestPrint;
+  } else {
+    return &InspectStack;
+  }
+}
+
+TEST_CASE(Inliner_InlineAndRunForceOptimized) {
+  auto check_handle = [](Dart_Handle handle) {
+    if (Dart_IsError(handle)) {
+      OS::PrintErr("Encountered unexpected error: %s\n", Dart_GetError(handle));
+      FATAL("Aborting");
+    }
+    return handle;
+  };
+
+  auto get_integer = [&](Dart_Handle lib, const char* name) {
+    Dart_Handle handle = Dart_GetField(lib, check_handle(NewString(name)));
+    check_handle(handle);
+
+    int64_t value = 0;
+    handle = Dart_IntegerToInt64(handle, &value);
+    check_handle(handle);
+    OS::Print("Field '%s': %" Pd64 "\n", name, value);
+    return value;
+  };
+
+  const char* kScriptChars = R"(
+import 'dart:ffi';
+import 'dart:_internal';
+
+int _add(int a, int b) => a + b;
+
+@pragma('vm:external-name', "testPrint")
+external void print(String s);
+
+@pragma("vm:external-name", "InspectStack")
+external InspectStack();
+
+@pragma('vm:never-inline')
+void nop() {}
+
+int prologueCount = 0;
+int epilogueCount = 0;
+
+@pragma('vm:never-inline')
+void countPrologue() {
+  prologueCount++;
+  print('countPrologue: $prologueCount');
+}
+
+@pragma('vm:never-inline')
+void countEpilogue() {
+  epilogueCount++;
+  print('countEpilogue: $epilogueCount');
+}
+
+@pragma('vm:force-optimize')
+@pragma('vm:idempotent')
+@pragma('vm:prefer-inline')
+int idempotentForceOptimizedFunction(int x) {
+  countPrologue();
+  print('deoptimizing');
+
+  InspectStack();
+  VMInternalsForTesting.deoptimizeFunctionsOnStack();
+  InspectStack();
+  print('deoptimizing after');
+  countEpilogue();
+  return _add(x, 1);
+}
+
+@pragma('vm:never-inline')
+int foo(int x, {bool onlyOptimizeFunction = false}) {
+  if (onlyOptimizeFunction) {
+    print('Optimizing `foo`');
+    for (int i = 0; i < 100000; i++) nop();
+  }
+  print('Running `foo`');
+  return x + idempotentForceOptimizedFunction(x+1) + 1;
+}
+
+void main() {
+  for (int i = 0; i < 100; i++) {
+    print('\n\nround=$i');
+
+    // Get the `foo` function optimized while leaving prologue/epilogue counters
+    // untouched.
+    final (a, b) = (prologueCount, epilogueCount);
+    foo(i, onlyOptimizeFunction: true);
+    prologueCount = a;
+    epilogueCount = b;
+
+    // Execute the optimized function `foo` function (which has the
+    // `idempotentForceOptimizedFunction` inlined).
+    final result = foo(i);
+    if (result != (2 * i + 3)) throw 'Expected ${2 * i + 3} but got $result!';
+  }
+}
+    )";
+  DisableBackgroundCompilationScope scope;
+
+  Dart_Handle lib =
+      check_handle(TestCase::LoadTestScript(kScriptChars, nullptr));
+  Dart_SetNativeResolver(lib, &PrintAndInspectResolver, nullptr);
+
+  // We disable OSR to ensure we control when the function gets optimized,
+  // namely afer a call to `foo(..., onlyOptimizeFunction:true)` it will be
+  // optimized and during a call to `foo(...)` it will get deoptimized.
+  IsolateGroup::Current()->set_use_osr(false);
+
+  // Run the test.
+  check_handle(Dart_Invoke(lib, NewString("main"), 0, nullptr));
+
+  // Examine the result.
+  const int64_t prologue_count = get_integer(lib, "prologueCount");
+  const int64_t epilogue_count = get_integer(lib, "epilogueCount");
+
+  // We always call the "foo" function when its optimized (and it's optimized
+  // code has the call to `idempotentForceOptimizedFunction` inlined).
+  //
+  // The `idempotentForceOptimizedFunction` will always execute prologue,
+  // lazy-deoptimize the `foo` frame, that will make the `foo` re-try the call
+  // to `idempotentForceOptimizedFunction`.
+  // = > We should see the prologue of the force-optimized function to be
+  // executed twice as many times as epilogue.
+  EXPECT_EQ(epilogue_count * 2, prologue_count);
+}
+
 }  // namespace dart

runtime/vm/compiler/compiler_pass.cc

@@ -298,6 +298,13 @@ void CompilerPass::RunInliningPipeline(PipelineMode mode,
   INVOKE_PASS(TryOptimizePatterns);
 }
 
+void CompilerPass::RunForceOptimizedInliningPipeline(
+    CompilerPassState* pass_state) {
+  INVOKE_PASS(TypePropagation);
+  INVOKE_PASS(Canonicalize);
+  INVOKE_PASS(ConstantPropagation);
+}
+
 // Keep in sync with TestPipeline::RunForcedOptimizedAfterSSAPasses.
 FlowGraph* CompilerPass::RunForceOptimizedPipeline(
     PipelineMode mode,

runtime/vm/compiler/compiler_pass.h

@@ -162,6 +162,7 @@ class CompilerPass {
 
   static void RunInliningPipeline(PipelineMode mode, CompilerPassState* state);
 
+  static void RunForceOptimizedInliningPipeline(CompilerPassState* state);
   // RunPipeline(WithPasses) may have the side effect of changing the FlowGraph
   // stored in the CompilerPassState. However, existing callers may depend on
   // the old invariant that the FlowGraph stored in the CompilerPassState was

runtime/vm/compiler/recognized_methods_list.h

@@ -302,7 +302,7 @@ namespace dart {
   V(::, _storeDouble, FfiStoreDouble, 0x428afcc3)                              \
   V(::, _storeDoubleUnaligned, FfiStoreDoubleUnaligned, 0x3dc047ba)            \
   V(::, _storePointer, FfiStorePointer, 0x8b5a5578)                            \
-  V(::, _fromAddress, FfiFromAddress, 0x810f927f)                              \
+  V(::, _fromAddress, FfiFromAddress, 0x810f9640)                              \
   V(Pointer, get:address, FfiGetAddress, 0x7ccff81d)                           \
   V(::, _asExternalTypedDataInt8, FfiAsExternalTypedDataInt8, 0x767b76f7)      \
   V(::, _asExternalTypedDataInt16, FfiAsExternalTypedDataInt16, 0xd08e6a25)    \

runtime/vm/deopt_instructions.cc

@@ -874,9 +874,6 @@ uword DeoptInstr::GetRetAddress(DeoptInstr* instr,
   ASSERT(instr->kind() == kRetAddress);
   DeoptRetAddressInstr* ret_address_instr =
       static_cast<DeoptRetAddressInstr*>(instr);
-  // The following assert may trigger when displaying a backtrace
-  // from the simulator.
-  ASSERT(DeoptId::IsDeoptAfter(ret_address_instr->deopt_id()));
   ASSERT(!object_table.IsNull());
   Thread* thread = Thread::Current();
   Zone* zone = thread->zone();

runtime/vm/isolate_reload.cc

@@ -1956,10 +1956,7 @@ class InvalidationCollector : public ObjectVisitor {
     if (cid == kFunctionCid) {
       const Function& func =
           Function::Handle(zone_, static_cast<FunctionPtr>(obj));
-      if (!func.ForceOptimize()) {
+      functions_->Add(&func);
-        // Force-optimized functions cannot deoptimize.
-        functions_->Add(&func);
-      }
     } else if (cid == kKernelProgramInfoCid) {
       kernel_infos_->Add(&KernelProgramInfo::Handle(
           zone_, static_cast<KernelProgramInfoPtr>(obj)));
@@ -2061,6 +2058,9 @@ void ProgramReloadContext::InvalidateFunctions(
   for (intptr_t i = 0; i < functions.length(); i++) {
     const Function& func = *functions[i];
 
+    // Force-optimized functions cannot deoptimize.
+    if (func.ForceOptimize()) continue;
+
     // Switch to unoptimized code or the lazy compilation stub.
     func.SwitchToLazyCompiledUnoptimizedCode();
 

runtime/vm/object.cc

@@ -9021,8 +9021,34 @@ void Function::SetIsOptimizable(bool value) const {
 }
 
 bool Function::ForceOptimize() const {
-  return RecognizedKindForceOptimize() || IsFfiTrampoline() ||
+  if (RecognizedKindForceOptimize() || IsFfiTrampoline() ||
-         IsTypedDataViewFactory() || IsUnmodifiableTypedDataViewFactory();
+      IsTypedDataViewFactory() || IsUnmodifiableTypedDataViewFactory()) {
+    return true;
+  }
+
+#if defined(TESTING)
+  // For run_vm_tests we allow marking arbitrary functions as force-optimize
+  // via `@pragma('vm:force-optimize')`.
+  if (has_pragma()) {
+    return Library::FindPragma(Thread::Current(), false, *this,
+                               Symbols::vm_force_optimize());
+  }
+#endif  // defined(TESTING)
+
+  return false;
+}
+
+bool Function::IsIdempotent() const {
+  if (!has_pragma()) return false;
+
+#if defined(TESTING)
+  const bool kAllowOnlyForCoreLibFunctions = false;
+#else
+  const bool kAllowOnlyForCoreLibFunctions = true;
+#endif  // defined(TESTING)
+
+  return Library::FindPragma(Thread::Current(), kAllowOnlyForCoreLibFunctions,
+                             *this, Symbols::vm_idempotent());
 }
 
 bool Function::RecognizedKindForceOptimize() const {
@@ -9092,12 +9118,6 @@ bool Function::RecognizedKindForceOptimize() const {
 
 #if !defined(DART_PRECOMPILED_RUNTIME)
 bool Function::CanBeInlined() const {
-  // Our force-optimized functions cannot deoptimize to an unoptimized frame.
-  // If the instructions of the force-optimized function body get moved via
-  // code motion, we might attempt do deoptimize a frame where the force-
-  // optimized function has only partially finished. Since force-optimized
-  // functions cannot deoptimize to unoptimized frames we prevent them from
-  // being inlined (for now).
   if (ForceOptimize()) {
     if (IsFfiTrampoline()) {
       // We currently don't support inlining FFI trampolines. Some of them
@@ -9107,7 +9127,14 @@ bool Function::CanBeInlined() const {
       // http://dartbug.com/45055.
       return false;
     }
-    return CompilerState::Current().is_aot();
+    if (CompilerState::Current().is_aot()) {
+      return true;
+    }
+    // Inlining of force-optimized functions requires target function to be
+    // idempotent becase if deoptimization is needed in inlined body, the
+    // execution of the force-optimized will be restarted at the beginning of
+    // the function.
+    return IsIdempotent();
   }
 
   if (HasBreakpoint()) {

runtime/vm/object.h

@@ -3569,6 +3569,13 @@ class Function : public Object {
   // run.
   bool ForceOptimize() const;
 
+  // Whether this function is idempotent (i.e. calling it twice has the same
+  // effect as calling it once - no visible side effects).
+  //
+  // If a function is idempotent VM may decide to abort halfway through one call
+  // and retry it again.
+  bool IsIdempotent() const;
+
   // Whether this function's |recognized_kind| requires optimization.
   bool RecognizedKindForceOptimize() const;
 

runtime/vm/symbols.h

@@ -514,15 +514,17 @@ class ObjectPointerVisitor;
   V(string_param_length, ":string_param_length")                               \
   V(system, "system")                                                          \
   V(vm_always_consider_inlining, "vm:always-consider-inlining")                \
+  V(vm_awaiter_link, "vm:awaiter-link")                                        \
   V(vm_entry_point, "vm:entry-point")                                          \
   V(vm_exact_result_type, "vm:exact-result-type")                              \
   V(vm_external_name, "vm:external-name")                                      \
   V(vm_ffi_abi_specific_mapping, "vm:ffi:abi-specific-mapping")                \
   V(vm_ffi_native_assets, "vm:ffi:native-assets")                              \
   V(vm_ffi_struct_fields, "vm:ffi:struct-fields")                              \
+  V(vm_force_optimize, "vm:force-optimize")                                    \
+  V(vm_idempotent, "vm:idempotent")                                            \
   V(vm_invisible, "vm:invisible")                                              \
   V(vm_isolate_unsendable, "vm:isolate-unsendable")                            \
-  V(vm_awaiter_link, "vm:awaiter-link")                                        \
   V(vm_never_inline, "vm:never-inline")                                        \
   V(vm_non_nullable_result_type, "vm:non-nullable-result-type")                \
   V(vm_notify_debugger_on_exception, "vm:notify-debugger-on-exception")        \

runtime/vm/unit_test.cc

@@ -611,8 +611,7 @@ Dart_Handle TestCase::ReloadTestScript(const char* script) {
     return result;
   }
 
-  return TriggerReload(/* kernel_buffer= */ nullptr,
+  return TriggerReload(/*kernel_buffer=*/nullptr, /*kernel_buffer_size=*/0);
-                       /* kernel_buffer_size= */ 0);
 }
 
 Dart_Handle TestCase::ReloadTestKernel(const uint8_t* kernel_buffer,

sdk/lib/_internal/vm/lib/ffi_patch.dart

@@ -77,6 +77,7 @@ int sizeOf<T extends NativeType>() {
   throw UnimplementedError("$T");
 }
 
+@pragma("vm:idempotent")
 @pragma("vm:recognized", "other")
 external Pointer<T> _fromAddress<T extends NativeType>(int ptr);