diff --git a/server.go b/server.go
index b79abad5..b6ce5b5f 100644
--- a/server.go
+++ b/server.go
@@ -460,6 +460,12 @@ func (srv *Server) serveTCP(l net.Listener) error {
 	// deadline is not used here
 	for {
 		rw, err := l.Accept()
+		srv.lock.RLock()
+		if !srv.started {
+			srv.lock.RUnlock()
+			return nil
+		}
+		srv.lock.RUnlock()
 		if err != nil {
 			if neterr, ok := err.(net.Error); ok && neterr.Temporary() {
 				continue
@@ -467,12 +473,6 @@ func (srv *Server) serveTCP(l net.Listener) error {
 			return err
 		}
 		m, err := reader.ReadTCP(rw, rtimeout)
-		srv.lock.RLock()
-		if !srv.started {
-			srv.lock.RUnlock()
-			return nil
-		}
-		srv.lock.RUnlock()
 		if err != nil {
 			continue
 		}