diff --git a/p2p/host/autorelay/autorelay.go b/p2p/host/autorelay/autorelay.go
index fe977366e..871db2a1e 100644
--- a/p2p/host/autorelay/autorelay.go
+++ b/p2p/host/autorelay/autorelay.go
@@ -9,6 +9,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/event"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/network"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/p2p/host/eventbus"
 
 	logging "github.com/libp2p/go-libp2p/gologshim"
@@ -53,6 +54,15 @@ func NewAutoRelay(host host.Host, opts ...Option) (*AutoRelay, error) {
 	return r, nil
 }
 
+// IsPeerInBackoff return true if the peer in backoff list currently
+func (r *AutoRelay) IsPeerInBackoff(peerID peer.ID) bool {
+	r.relayFinder.candidateMx.Lock()
+	defer r.relayFinder.candidateMx.Unlock()
+
+	_, ok := r.relayFinder.backoff[peerID]
+	return ok
+}
+
 func (r *AutoRelay) Start() {
 	r.refCount.Add(1)
 	go func() {
diff --git a/p2p/host/autorelay/autorelay_test.go b/p2p/host/autorelay/autorelay_test.go
index 46a2a4935..cf1eee01e 100644
--- a/p2p/host/autorelay/autorelay_test.go
+++ b/p2p/host/autorelay/autorelay_test.go
@@ -267,6 +267,48 @@ func TestBackoff(t *testing.T) {
 	require.Equal(t, 2, int(reservations.Load()))
 }
 
+func TestRemovePeerFromBackoffAfterSuccess(t *testing.T) {
+	const backoff = 20 * time.Second
+	cl := newMockClock()
+
+	rh := newRelay(t)
+
+	var counter atomic.Int32
+	h, err := libp2p.New(
+		libp2p.ForceReachabilityPrivate(),
+	)
+	require.NoError(t, err)
+	defer h.Close()
+
+	ar, err := autorelay.NewAutoRelay(h,
+		autorelay.WithPeerSource(
+			func(context.Context, int) <-chan peer.AddrInfo {
+				// always return the same node, and make sure we don't try to connect to it too frequently
+				counter.Add(1)
+				peerChan := make(chan peer.AddrInfo, 1)
+				peerChan <- peer.AddrInfo{ID: rh.ID(), Addrs: rh.Addrs()}
+				close(peerChan)
+				return peerChan
+			}),
+		autorelay.WithNumRelays(1),
+		autorelay.WithBootDelay(0),
+		autorelay.WithBackoff(backoff),
+		autorelay.WithMinCandidates(1),
+		autorelay.WithMaxCandidateAge(1),
+		autorelay.WithClock(cl),
+		autorelay.WithMinInterval(0),
+	)
+	require.NoError(t, err)
+	ar.Start()
+	defer ar.Close()
+
+	require.Eventually(t, func() bool {
+		return numRelays(h) > 0
+	}, 5*time.Second, 100*time.Millisecond, "should successfully reserve relay")
+
+	require.False(t, ar.IsPeerInBackoff(rh.ID()), "successfully added relay should not be in backoff list")
+}
+
 func TestStaticRelays(t *testing.T) {
 	const numStaticRelays = 3
 	staticRelays := make([]peer.AddrInfo, 0, numStaticRelays)
diff --git a/p2p/host/autorelay/relay_finder.go b/p2p/host/autorelay/relay_finder.go
index 35ac928b7..7aa6348de 100644
--- a/p2p/host/autorelay/relay_finder.go
+++ b/p2p/host/autorelay/relay_finder.go
@@ -663,13 +663,13 @@ func (rf *relayFinder) connectToRelay(ctx context.Context, cand *candidate) (*ci
 		}
 	}
 
-	rf.candidateMx.Lock()
-	rf.backoff[id] = rf.conf.clock.Now()
-	rf.candidateMx.Unlock()
 	var err error
 	if cand.supportsRelayV2 {
 		rsvp, err = circuitv2.Reserve(ctx, rf.host, cand.ai)
 		if err != nil {
+			rf.candidateMx.Lock()
+			rf.backoff[id] = rf.conf.clock.Now()
+			rf.candidateMx.Unlock()
 			err = fmt.Errorf("failed to reserve slot: %w", err)
 		}
 	}