diff --git a/p2p/transport/quic/transport.go b/p2p/transport/quic/transport.go
index 8916fbb53..c0d6055af 100644
--- a/p2p/transport/quic/transport.go
+++ b/p2p/transport/quic/transport.go
@@ -221,7 +221,12 @@ func (t *transport) Listen(addr ma.Multiaddr) (tpt.Listener, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newListener(conn, t, t.localPeer, t.privKey, t.identity)
+	ln, err := newListener(conn, t, t.localPeer, t.privKey, t.identity)
+	if err != nil {
+		conn.DecreaseCount()
+		return nil, err
+	}
+	return ln, nil
 }
 
 // Proxy returns true if this transport proxies.