diff --git a/server.go b/server.go
index 136a9a1c..59135b0f 100644
--- a/server.go
+++ b/server.go
@@ -509,7 +509,10 @@ func (srv *Server) serveUDP(l *net.UDPConn) error {
 		}
 		srv.lock.RUnlock()
 		if err != nil {
-			continue
+			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
+				continue
+			}
+			return err
 		}
 		go srv.serve(s.RemoteAddr(), handler, m, l, s, nil)
 	}