diff --git a/server.go b/server.go
index eec02ef9..b2a63bda 100644
--- a/server.go
+++ b/server.go
@@ -321,6 +321,7 @@ func (srv *Server) ListenAndServe() error {
 		}
 		u := l.(*net.UDPConn)
 		if e := setUDPSocketOptions(u); e != nil {
+			u.Close()
 			return e
 		}
 		srv.PacketConn = l