[nsec3] fix crash in nsec3 packing (#973)

Both NSEC and NSEC3 use the same logic to pack the bitmap.
CSYNC.pack also appear to make use of `packDataNsec` so I am giving it
the same treatment by moving the logic in a helper function and making
all those types `len` call use that function.

fuzz_test.go

@@ -105,3 +105,41 @@ func TestCrashNSEC(t *testing.T) {
 		t.Fatalf("expected length of %d, got %d", expectedLength, l)
 	}
 }
+
+// TestCrashNSEC3 tests generated using fuzz.go and with a message pack
+// containing the following bytes:
+// "0000\x00\x00000000\x00\x00200000" +
+// "0\x00\v0000\x00\x00#\x0300\x00\x00\x00\x1a000" +
+// "000\x00\v00\x0200\x00\x03000\x00"
+// That byte sequence, when Unpack() and subsequential Pack() created a
+// panic: runtime error: slice bounds out of range
+// which was attributed to the fact that NSEC3 RR length computation was
+// different (and smaller) then within NSEC3.pack (which relies on
+// packDataNsec).
+func TestCrashNSEC3(t *testing.T) {
+	compression := make(map[string]struct{})
+	nsec3 := &NSEC3{
+		Hdr: RR_Header{
+			Name:     ".",
+			Rrtype:   0x32,
+			Class:    0x3030,
+			Ttl:      0x30303030,
+			Rdlength: 0xb,
+		},
+		Hash:       0x30,
+		Flags:      0x30,
+		Iterations: 0x3030,
+		SaltLength: 0x0,
+		Salt:       "",
+		HashLength: 0x0,
+		NextDomain: ".",
+		TypeBitMap: []uint16{
+			0x2302, 0x2303, 0x230a, 0x230b,
+		},
+	}
+	expectedLength := 24
+	l := nsec3.len(0, compression)
+	if l != expectedLength {
+		t.Fatalf("expected length of %d, got %d", expectedLength, l)
+	}
+}

msg_helpers.go

@@ -587,6 +587,29 @@ func unpackDataNsec(msg []byte, off int) ([]uint16, int, error) {
 	return nsec, off, nil
 }
 
+// typeBitMapLen is a helper function which computes the "maximum" length of
+// a the NSEC Type BitMap field.
+func typeBitMapLen(bitmap []uint16) int {
+	var l int
+	var lastwindow, lastlength uint16
+	for _, t := range bitmap {
+		window := t / 256
+		length := (t-window*256)/8 + 1
+		if window > lastwindow && lastlength != 0 { // New window, jump to the new offset
+			l += int(lastlength) + 2
+			lastlength = 0
+		}
+		if window < lastwindow || length < lastlength {
+			// packDataNsec would return Error{err: "nsec bits out of order"} here, but
+			// when computing the length, we want do be liberal.
+			continue
+		}
+		lastwindow, lastlength = window, length
+	}
+	l += int(lastlength) + 2
+	return l
+}
+
 func packDataNsec(bitmap []uint16, msg []byte, off int) (int, error) {
 	if len(bitmap) == 0 {
 		return off, nil

types.go

@@ -854,22 +854,7 @@ func (rr *NSEC) String() string {
 func (rr *NSEC) len(off int, compression map[string]struct{}) int {
 	l := rr.Hdr.len(off, compression)
 	l += domainNameLen(rr.NextDomain, off+l, compression, false)
-	var lastwindow, lastlength uint16
-	for _, t := range rr.TypeBitMap {
-		window := t / 256
-		length := (t-window*256)/8 + 1
-		if window > lastwindow && lastlength != 0 { // New window, jump to the new offset
-			l += int(lastlength) + 2
-			lastlength = 0
-		}
-		if window < lastwindow || length < lastlength {
-			// packDataNsec would return Error{err: "nsec bits out of order"} here, but
-			// when computing the length, we want do be liberal.
-			continue
-		}
-		lastwindow, lastlength = window, length
-	}
-	l += int(lastlength) + 2
+	l += typeBitMapLen(rr.TypeBitMap)
 	return l
 }
 
@@ -1028,14 +1013,7 @@ func (rr *NSEC3) String() string {
 func (rr *NSEC3) len(off int, compression map[string]struct{}) int {
 	l := rr.Hdr.len(off, compression)
 	l += 6 + len(rr.Salt)/2 + 1 + len(rr.NextDomain) + 1
-	lastwindow := uint32(2 ^ 32 + 1)
-	for _, t := range rr.TypeBitMap {
-		window := t / 256
-		if uint32(window) != lastwindow {
-			l += 1 + 32
-		}
-		lastwindow = uint32(window)
-	}
+	l += typeBitMapLen(rr.TypeBitMap)
 	return l
 }
 
@@ -1352,14 +1330,7 @@ func (rr *CSYNC) String() string {
 func (rr *CSYNC) len(off int, compression map[string]struct{}) int {
 	l := rr.Hdr.len(off, compression)
 	l += 4 + 2
-	lastwindow := uint32(2 ^ 32 + 1)
-	for _, t := range rr.TypeBitMap {
-		window := t / 256
-		if uint32(window) != lastwindow {
-			l += 1 + 32
-		}
-		lastwindow = uint32(window)
-	}
+	l += typeBitMapLen(rr.TypeBitMap)
 	return l
 }