Give a less silly response to a silly request.

Prior to this commit, fread/fwrite calls with size * nmemb > SIZE_MAX
were handled by reading or writing (size_t)(size * nmemb) bytes; for
example, on 32-bit platforms, fread(ptr, 641, 6700417, f) would read 1
byte and indicate that the requested 6700417 blocks had been read.

This commit adds a check for such integer overflows, and treats them as
if an overly large request was passed to read/write; i.e., it sets errno
to EINVAL, sets the error indicator on the file, and returns a short
object count (0, to be specific).

The overflow check involves an integer division, so as a performance
optimization we check first to see if both size and nmemb are less than
2^16; if they are, no overflow is possible and we avoid the division.
We assume here that size_t is at least 32 bits; this appears to be true
on all platforms FreeBSD supports.

Although this commit fixes an integer overflow, it is not likely to have
any security implications, since any program which would be affected by
this bug fix is quite clearly already very confused.

Reviewed by:	kib
MFC after:	1 month

lib/libc/stdio/fread.c

@@ -37,6 +37,8 @@ static char sccsid[] = "@(#)fread.c	8.2 (Berkeley) 12/11/93";
 __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
+#include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include "un-namespace.h"
@@ -69,8 +71,27 @@ __fread(void * __restrict buf, size_t size, size_t count, FILE * __restrict fp)
 	/*
 	 * ANSI and SUSv2 require a return value of 0 if size or count are 0.
 	 */
-	if ((resid = count * size) == 0)
+	if ((count == 0) || (size == 0))
 		return (0);
+
+	/*
+	 * Check for integer overflow.  As an optimization, first check that
+	 * at least one of {count, size} is at least 2^16, since if both
+	 * values are less than that, their product can't possible overflow
+	 * (size_t is always at least 32 bits on FreeBSD).
+	 */
+	if (((count | size) > 0xFFFF) &&
+	    (count > SIZE_MAX / size)) {
+		errno = EINVAL;
+		fp->_flags |= __SERR;
+		return (0);
+	}
+
+	/*
+	 * Compute the (now required to not overflow) number of bytes to
+	 * read and actually do the work.
+	 */
+	resid = count * size;
 	ORIENT(fp, -1);
 	if (fp->_r < 0)
 		fp->_r = 0;

lib/libc/stdio/fwrite.c

@@ -37,6 +37,8 @@ static char sccsid[] = "@(#)fwrite.c	8.1 (Berkeley) 6/4/93";
 __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
+#include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include "un-namespace.h"
 #include "local.h"
@@ -60,10 +62,24 @@ fwrite(buf, size, count, fp)
 	/*
 	 * ANSI and SUSv2 require a return value of 0 if size or count are 0.
 	 */
-	n = count * size;
-	if (n == 0)
+	if ((count == 0) || (size == 0))
 		return (0);
 
+	/*
+	 * Check for integer overflow.  As an optimization, first check that
+	 * at least one of {count, size} is at least 2^16, since if both
+	 * values are less than that, their product can't possible overflow
+	 * (size_t is always at least 32 bits on FreeBSD).
+	 */
+	if (((count | size) > 0xFFFF) &&
+	    (count > SIZE_MAX / size)) {
+		errno = EINVAL;
+		fp->_flags |= __SERR;
+		return (0);
+	}
+
+	n = count * size;
+
 	iov.iov_base = (void *)buf;
 	uio.uio_resid = iov.iov_len = n;
 	uio.uio_iov = &iov;