[cfe/ffi] Improve FFI call mismatched types compile errors

This CL fixes two issues.

1. `FfiNative`s now check the Dart and native type for compatiblity.
2. Both `FfiNative`, `asFunction`, and `lookupFunction` check the type
   correspondence between native and Dart type with a subtype check of
   the expected Dart type and the provided Dart type. For functions,
   any return type is a subtype of a void type. This is fine for Dart,
   but not for native calls. This CL manually checks the return type
   for void.

This CL does not fix the inconsistency between `asFunction` and
`FfiNative` with regard to allowing more strict return types than
`Object` for `Handle`s
Issue: https://github.com/dart-lang/sdk/issues/49518

Analyzer fixes in follow up CL.

TEST=tests/ffi/vmspecific_static_checks_ffinative_test.dart

Closes: https://github.com/dart-lang/sdk/issues/49471
Change-Id: Ibc7bd6a1a0db59cc5fa5d755d76999fd7e9a06a4
Cq-Include-Trybots: luci.dart.try:analyzer-linux-release-try,analyzer-mac-release-try,vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64c-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/252601
Reviewed-by: Martin Kustermann <kustermann@google.com>
Commit-Queue: Daco Harkes <dacoharkes@google.com>

pkg/vm/lib/transformations/ffi/common.dart

@@ -1120,7 +1120,14 @@ class FfiTransformer extends Transformer {
     if (dartType == correspondingDartType) return;
     if (env.isSubtypeOf(correspondingDartType, dartType,
         SubtypeCheckMode.ignoringNullabilities)) {
-      return;
+      // If subtype, manually check the return type is not void.
+      if (dartType is! FunctionType || correspondingDartType is! FunctionType) {
+        return;
+      } else if ((dartType.returnType is VoidType) ==
+          (correspondingDartType.returnType is VoidType)) {
+        return;
+      }
+      // One of the return types is void, the other isn't, report error.
     }
     diagnosticReporter.report(
         templateFfiTypeMismatch.withArguments(dartType, correspondingDartType,

pkg/vm/lib/transformations/ffi/native.dart

@@ -117,9 +117,10 @@ class FfiNativeTransformer extends FfiTransformer {
   // Replaces return type with Object if it is Handle.
   DartType _wrapReturnType(DartType dartReturnType, DartType ffiReturnType) {
     if (env.isSubtypeOf(
-        ffiReturnType,
-        handleClass.getThisType(coreTypes, Nullability.nonNullable),
-        SubtypeCheckMode.ignoringNullabilities)) {
+            ffiReturnType,
+            handleClass.getThisType(coreTypes, Nullability.nonNullable),
+            SubtypeCheckMode.ignoringNullabilities) &&
+        dartReturnType is! VoidType) {
       return objectClass.getThisType(coreTypes, dartReturnType.nullability);
     }
     return dartReturnType;
@@ -411,7 +412,8 @@ class FfiNativeTransformer extends FfiTransformer {
         nativeFunctionClass, Nullability.legacy, [ffiFunctionType]);
     try {
       ensureNativeTypeValid(nativeType, node);
-      ensureNativeTypeToDartType(nativeType, wrappedDartFunctionType, node);
+      ensureNativeTypeToDartType(nativeType, wrappedDartFunctionType, node,
+          allowHandle: true);
       ensureLeafCallDoesNotUseHandles(nativeType, isLeaf, node);
     } on FfiStaticTypeError {
       // It's OK to swallow the exception because the diagnostics issued will

tests/ffi/vmspecific_static_checks_ffinative_test.dart

@@ -37,6 +37,10 @@ class NativeClassy extends NativeFieldWrapperClass1 {
   // Error: Missing receiver in FfiNative annotation.
   @FfiNative<Void Function(IntPtr)>('doesntmatter') //# 05: compile-time error
   external void badMissingReceiver(int v); //# 05: compile-time error
+
+  // Error: wrong return type.
+  @FfiNative<Handle Function(Pointer<Void>, Uint32, Uint32, Handle)>('doesntmatter') //# 49471: compile-time error
+  external void toImageSync(int width, int height, Object outImage);  //# 49471: compile-time error
 }
 
 // Error: Too many FfiNative parameters.

tests/ffi/vmspecific_static_checks_test.dart

@@ -64,6 +64,7 @@ void main() {
   testAsFunctionTakesHandle();
   testLookupFunctionReturnsHandle();
   testAsFunctionReturnsHandle();
+  testReturnVoidNotVoid();
 }
 
 typedef Int8UnOp = Int8 Function(Int8);
@@ -820,3 +821,11 @@ class MyFinalizableStruct extends Struct
 {
   external Pointer<Void> field;
 }
+
+void testReturnVoidNotVoid() {
+  // Taking a more specific argument is okay.
+  testLibrary //# 49471: compile-time error
+      .lookupFunction< //# 49471: compile-time error
+          Handle Function(), //# 49471: compile-time error
+          void Function()>("doesntmatter"); //# 49471: compile-time error
+}