Skip to content

Commit

Permalink
fix(leader-election): introduce delay before attempting to reacquire …
Browse files Browse the repository at this point in the history
…leadership (#794)

Fixes #785.

Introduce a retry delay before attempting to reacquire leadership,
fixing the log noise generated from the API server being unavailable.
  • Loading branch information
exextatic authored Jul 25, 2024
1 parent d026e3b commit 0965d23
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ static async ValueTask CastAndDispose(IDisposable resource)

private async Task RunAndTryToHoldLeadershipForeverAsync()
{
uint leadershipRetries = 0;

while (!_cts.IsCancellationRequested)
{
try
Expand All @@ -94,7 +96,14 @@ private async Task RunAndTryToHoldLeadershipForeverAsync()
}
catch (Exception exception)
{
logger.LogError(exception, "Failed to hold leadership.");
leadershipRetries++;

var delay = TimeSpan
.FromSeconds(Math.Pow(2, Math.Clamp(leadershipRetries, 0, 5)))
.Add(TimeSpan.FromMilliseconds(new Random().Next(0, 1000)));

logger.LogError(exception, "Failed to hold leadership. Wait {Seconds}s before attempting to reacquire leadership.", delay.TotalSeconds);
await Task.Delay(delay);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ public async Task Elector_Throws_Should_Retry()
await leaderElectionBackgroundService.StartAsync(CancellationToken.None);

// Starting the background service should result in the lock attempt throwing, and then a subsequent attempt being made.
// Wait for the subsequent event to be signalled, if we time out the test fails.
electionLockSubsequentCallEvent.WaitOne(TimeSpan.FromMilliseconds(500)).Should().BeTrue();
// Wait for the subsequent event to be signalled, if we time out the test fails. The retry delay requires us to wait at least 3 seconds.
electionLockSubsequentCallEvent.WaitOne(TimeSpan.FromMilliseconds(3100)).Should().BeTrue();

await leaderElectionBackgroundService.StopAsync(CancellationToken.None);
}
Expand Down

0 comments on commit 0965d23

Please sign in to comment.