crypto/rsa: return err when key too small to compute salt for RSA PSS

When PSSSaltLengthAuto is passed to SignPSS, and the key size is too
small to create a valid salt, return ErrMessageTooLong

Change-Id: I4e0d70bdd54fcd667eae10e0a70b4f540a4ebe93
Reviewed-on: https://go-review.googlesource.com/c/go/+/450796
Reviewed-by: Filippo Valsorda <filippo@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
Run-TryBot: Roland Shoemaker <roland@golang.org>

src/crypto/rsa/pss.go

@@ -297,6 +297,9 @@ func SignPSS(rand io.Reader, priv *PrivateKey, hash crypto.Hash, digest []byte,
 	switch saltLength {
 	case PSSSaltLengthAuto:
 		saltLength = (priv.N.BitLen()-1+7)/8 - 2 - hash.Size()
+		if saltLength < 0 {
+			return nil, ErrMessageTooLong
+		}
 	case PSSSaltLengthEqualsHash:
 		saltLength = hash.Size()
 	default: