kern_mmap: add a variant that allows caller to inspect fp

Linux mmap rejects mmap() on a write-only file with EACCES.
linux_mmap_common currently does a fun dance to grab the fp associated with
the passed in fd, validates it, then drops the reference and calls into
kern_mmap(). Doing so is perhaps both fragile and premature; there's still
plenty of chance for the request to get rejected with a more appropriate
error, and it's prone to a race where the file we ultimately mmap has
changed after it drops its referenced.

This change alleviates the need to do this by providing a kern_mmap variant
that allows the caller to inspect the fp just before calling into the fileop
layer. The callback takes flags, prot, and maxprot as one could imagine
scenarios where any of these, in conjunction with the file itself, may
influence a caller's decision.

The file type check in the linux compat layer has been removed; EINVAL is
seemingly not an appropriate response to the file not being a vnode or
device. The fileop layer will reject the operation with ENODEV if it's not
supported, which more closely matches the common linux description of
mmap(2) return values.

If we discover that we're allowing an mmap() on a file type that Linux
normally wouldn't, we should restrict those explicitly.

Reviewed by:	kib
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D22977

sys/compat/linux/linux_mmap.c

@@ -62,6 +62,16 @@ __FBSDID("$FreeBSD$");
 static void linux_fixup_prot(struct thread *td, int *prot);
 #endif
 
+static int
+linux_mmap_check_fp(struct file *fp, int flags, int prot, int maxprot)
+{
+
+	/* Linux mmap() just fails for O_WRONLY files */
+	if ((fp->f_flag & FREAD) == 0)
+		return (EACCES);
+
+	return (0);
+}
 
 int
 linux_mmap_common(struct thread *td, uintptr_t addr, size_t len, int prot,
@@ -117,31 +127,6 @@ linux_mmap_common(struct thread *td, uintptr_t addr, size_t len, int prot,
 
 	/* Linux does not check file descriptor when MAP_ANONYMOUS is set. */
 	fd = (bsd_flags & MAP_ANON) ? -1 : fd;
-	if (fd != -1) {
-		/*
-		 * Linux follows Solaris mmap(2) description:
-		 * The file descriptor fildes is opened with
-		 * read permission, regardless of the
-		 * protection options specified.
-		 */
-
-		error = fget(td, fd, &cap_mmap_rights, &fp);
-		if (error != 0)
-			return (error);
-		if (fp->f_type != DTYPE_VNODE && fp->f_type != DTYPE_DEV) {
-			fdrop(fp, td);
-			return (EINVAL);
-		}
-
-		/* Linux mmap() just fails for O_WRONLY files */
-		if (!(fp->f_flag & FREAD)) {
-			fdrop(fp, td);
-			return (EACCES);
-		}
-
-		fdrop(fp, td);
-	}
-
 	if (flags & LINUX_MAP_GROWSDOWN) {
 		/*
 		 * The Linux MAP_GROWSDOWN option does not limit auto
@@ -211,13 +196,15 @@ linux_mmap_common(struct thread *td, uintptr_t addr, size_t len, int prot,
 	 */
 	if (addr != 0 && (bsd_flags & MAP_FIXED) == 0 &&
 	    (bsd_flags & MAP_EXCL) == 0) {
-		error = kern_mmap(td, addr, len, prot,
-		    bsd_flags | MAP_FIXED | MAP_EXCL, fd, pos);
+		error = kern_mmap_fpcheck(td, addr, len, prot,
+		    bsd_flags | MAP_FIXED | MAP_EXCL, fd, pos,
+		    linux_mmap_check_fp);
 		if (error == 0)
 			goto out;
 	}
 
-	error = kern_mmap(td, addr, len, prot, bsd_flags, fd, pos);
+	error = kern_mmap_fpcheck(td, addr, len, prot, bsd_flags, fd, pos,
+	    linux_mmap_check_fp);
 out:
 	LINUX_CTR2(mmap2, "return: %d (%p)", error, td->td_retval[0]);
 

sys/sys/syscallsubr.h

@@ -63,6 +63,8 @@ struct stat;
 struct thr_param;
 struct uio;
 
+typedef int (*mmap_check_fp_fn)(struct file *, int, int, int);
+
 int	kern___getcwd(struct thread *td, char *buf, enum uio_seg bufseg,
 	    size_t buflen, size_t path_max);
 int	kern_accept(struct thread *td, int s, struct sockaddr **name,
@@ -179,6 +181,9 @@ int	kern_mlock(struct proc *proc, struct ucred *cred, uintptr_t addr,
 	    size_t len);
 int	kern_mmap(struct thread *td, uintptr_t addr, size_t len, int prot,
 	    int flags, int fd, off_t pos);
+int	kern_mmap_fpcheck(struct thread *td, uintptr_t addr, size_t len,
+	    int prot, int flags, int fd, off_t pos,
+	    mmap_check_fp_fn check_fp_fn);
 int	kern_mmap_maxprot(struct proc *p, int prot);
 int	kern_mprotect(struct thread *td, uintptr_t addr, size_t size, int prot);
 int	kern_msgctl(struct thread *, int, int, struct msqid_ds *);

sys/vm/vm_mmap.c

@@ -199,6 +199,18 @@ int
 kern_mmap(struct thread *td, uintptr_t addr0, size_t len, int prot, int flags,
     int fd, off_t pos)
 {
+
+	return (kern_mmap_fpcheck(td, addr, len, prot, flags, fd, pos, NULL));
+}
+
+/*
+ * When mmap'ing a file, check_fp_fn may be used for the caller to do any
+ * last-minute validation based on the referenced file in a non-racy way.
+ */
+int
+kern_mmap_fpcheck(struct thread *td, uintptr_t addr0, size_t len, int prot,
+    int flags, int fd, off_t pos, mmap_check_fp_fn check_fp_fn)
+{
 	struct vmspace *vms;
 	struct file *fp;
 	struct proc *p;
@@ -394,7 +406,12 @@ kern_mmap(struct thread *td, uintptr_t addr0, size_t len, int prot, int flags,
 			error = EINVAL;
 			goto done;
 		}
-
+		if (check_fp_fn != NULL) {
+			error = check_fp_fn(fp, prot, max_prot & cap_maxprot,
+			    flags);
+			if (error != 0)
+				goto done;
+		}
 		/* This relies on VM_PROT_* matching PROT_*. */
 		error = fo_mmap(fp, &vms->vm_map, &addr, size, prot,
 		    max_prot & cap_maxprot, flags, pos, td);