Harden RSA public key unpacking (#693)

RFC 2537 (RSA/MD5) and RFC 3110 (RSA/SHA1) disallow leading zero octets.
RFC 5702 (RSA/SHA256 and RSA/SHA512) isn't specific but defers to these
earlier RFCs in other places.

There is an upper limit of 4096 bits for both the modulus and exponent.
The modulus must be at least 512 bits. No minimum is specified for the
exponent but a quick search suggests single byte exponents are viable.

Exponents larger than 32 bits are already disallowed. This commit adds
checks for the other requirements, general bounds checks, and defers
initialisation of the big num till the other checks have passed.

dnssec.go

@@ -533,6 +533,11 @@ func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey {
 		return nil
 	}
 
+	if len(keybuf) < 1+1+64 {
+		// Exponent must be at least 1 byte and modulus at least 64
+		return nil
+	}
+
 	// RFC 2537/3110, section 2. RSA Public KEY Resource Records
 	// Length is in the 0th byte, unless its zero, then it
 	// it in bytes 1 and 2 and its a 16 bit number
@@ -542,13 +547,22 @@ func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey {
 		explen = uint16(keybuf[1])<<8 | uint16(keybuf[2])
 		keyoff = 3
 	}
-	if explen > 4 {
-		// Larger exponent than supported by the crypto package.
+
+	if explen > 4 || explen == 0 || keybuf[keyoff] == 0 {
+		// Exponent larger than supported by the crypto package,
+		// empty, or contains prohibited leading zero.
 		return nil
 	}
+
+	modoff := keyoff + int(explen)
+	modlen := len(keybuf) - modoff
+	if modlen < 64 || modlen > 512 || keybuf[modoff] == 0 {
+		// Modulus is too small, large, or contains prohibited leading zero.
+		return nil
+	}
+
 	pubkey := new(rsa.PublicKey)
 
-	pubkey.N = big.NewInt(0)
 	expo := uint64(0)
 	for i := 0; i < int(explen); i++ {
 		expo <<= 8
@@ -560,7 +574,9 @@ func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey {
 	}
 	pubkey.E = int(expo)
 
-	pubkey.N.SetBytes(keybuf[keyoff+int(explen):])
+	pubkey.N = big.NewInt(0)
+	pubkey.N.SetBytes(keybuf[modoff:])
+
 	return pubkey
 }