/// <inheritdoc/>
public void Dispose()
{
if (isDisposed)
{
return;
}
isDisposed = true;
if (leaderElector != null)
{
leaderElector.Dispose();
try
{
leaderTask.WaitWithoutAggregate();
}
catch (OperationCanceledException)
{
// We're expecting this.
}
leaderElector = null;
leaderTask = null;
}
mutex.Dispose();
GC.SuppressFinalize(this);
}
public void LeaderElectionThrowException()
{
var l = new Mock <ILock>();
l.Setup(obj => obj.GetAsync(CancellationToken.None))
.Throws(new Exception("noxu"));
var leaderElector = new LeaderElector(new LeaderElectionConfig(l.Object)
{
LeaseDuration = TimeSpan.FromMilliseconds(1000),
RetryPeriod = TimeSpan.FromMilliseconds(200),
RenewDeadline = TimeSpan.FromMilliseconds(700),
});
try
{
leaderElector.RunAsync().Wait();
}
catch (Exception e)
{
Assert.Equal("noxu", e.InnerException?.Message);
return;
}
Assert.True(false, "exception not thrown");
}
public async Task Single_WithoutActionsOrCounters()
{
// Verify that the elector works without callback actions.
var leaseName = $"test-{NeonHelper.CreateBase36Uuid()}";
var config = new LeaderElectionConfig(fixture.K8s, @namespace: KubeNamespace.Default, leaseName: leaseName, identity: "instance-0");
var elector = new LeaderElector(config);
Task electorTask;
try
{
using (elector)
{
electorTask = elector.RunAsync();
NeonHelper.WaitFor(() => elector.IsLeader, timeout: MaxWaitTime);
Assert.True(elector.IsLeader);
Assert.Equal("instance-0", elector.Leader);
}
// Ensure that the elector task completes.
await electorTask.WaitAsync(timeout : MaxWaitTime);
}
finally
{
await config.K8s.DeleteNamespacedLeaseWithHttpMessagesAsync(leaseName, config.Namespace);
}
}
/// <summary>
/// Starts the resource manager.
/// </summary>
/// <param name="namespace">Optionally specifies the namespace for namespace scoped operators.</param>
/// <exception cref="InvalidOperationException">Thrown when the resource manager has already been started.</exception>
public async Task StartAsync(string @namespace = null)
{
Covenant.Requires <ArgumentException>(@namespace == null || @namespace != string.Empty, nameof(@namespace));
if (started)
{
throw new InvalidOperationException($"[{nameof(ResourceManager<TEntity, TController>)}] is already running.");
}
//-----------------------------------------------------------------
// Start the leader elector if enabled.
resourceNamespace = @namespace;
started = true;
// Start the leader elector when enabled.
if (leaderConfig != null)
{
leaderElector = new LeaderElector(
leaderConfig,
onStartedLeading: OnPromotion,
onStoppedLeading: OnDemotion,
onNewLeader: OnNewLeader);
leaderTask = leaderElector.RunAsync();
}
await Task.CompletedTask;
}
public LeaderElectorTest()
{
_elector = new LeaderElector(
_logger.Object,
new OperatorSettings {
Name = "test"
},
_election.Object,
_client.Object);
}
public void LeaderElectionReportLeaderOnStart()
{
var l = new Mock <ILock>();
l.Setup(obj => obj.Identity)
.Returns("foo1");
l.SetupSequence(obj => obj.GetAsync(CancellationToken.None))
.ReturnsAsync(() =>
{
return(new LeaderElectionRecord()
{
HolderIdentity = "foo2",
AcquireTime = DateTime.Now,
RenewTime = DateTime.Now,
LeaderTransitions = 1,
LeaseDurationSeconds = 60,
});
})
.ReturnsAsync(() =>
{
return(new LeaderElectionRecord()
{
HolderIdentity = "foo3",
AcquireTime = DateTime.Now,
RenewTime = DateTime.Now,
LeaderTransitions = 1,
LeaseDurationSeconds = 60,
});
});
var leaderElector = new LeaderElector(new LeaderElectionConfig(l.Object)
{
LeaseDuration = TimeSpan.FromMilliseconds(1000),
RetryPeriod = TimeSpan.FromMilliseconds(200),
RenewDeadline = TimeSpan.FromMilliseconds(700),
});
var countdown = new CountdownEvent(2);
var notifications = new List <string>();
leaderElector.OnNewLeader += id =>
{
notifications.Add(id);
countdown.Signal();
};
Task.Run(() => leaderElector.RunAsync());
countdown.Wait(TimeSpan.FromSeconds(10));
Assert.True(notifications.SequenceEqual(new[]
{
"foo2", "foo3",
}));
}
public async Task Single_WithCounters()
{
// Verify that the elector can increment performance counters.
var leaseName = $"test-{NeonHelper.CreateBase36Uuid()}";
var promotionCounter = Metrics.CreateCounter("test_promotions", string.Empty);
var demotionCounter = Metrics.CreateCounter("test_demotions", string.Empty);
var newLeaderCounter = Metrics.CreateCounter("test_newleaders", string.Empty);
var config = new LeaderElectionConfig(
k8s: fixture.K8s,
@namespace: KubeNamespace.Default,
leaseName: leaseName,
identity: "instance-0",
promotionCounter: promotionCounter,
demotionCounter: demotionCounter,
newLeaderCounter: newLeaderCounter);
var elector = new LeaderElector(config);
Task electorTask;
try
{
using (elector)
{
electorTask = elector.RunAsync();
NeonHelper.WaitFor(() => elector.IsLeader, timeout: MaxWaitTime);
Assert.True(elector.IsLeader);
Assert.Equal("instance-0", elector.Leader);
}
// Ensure that the elector task completes.
await electorTask.WaitAsync(timeout : MaxWaitTime);
// Ensure that the counters are correct.
Assert.Equal(1, promotionCounter.Value);
Assert.Equal(0, demotionCounter.Value); // We don't see demotions when the elector is disposed
Assert.Equal(2, newLeaderCounter.Value); // We do see a leadership change when the elector is disposed
}
finally
{
await config.K8s.DeleteNamespacedLeaseWithHttpMessagesAsync(leaseName, config.Namespace);
}
}
public async Task Single_WithoutCounters()
{
// Verify that we can create a single [LeaderElector] instance and that:
//
// 1. The [OnNewLeader] action is called
// 2. The [OnStartedLeader] action is called
// 3. The new leader matches the current instance
// 4. IsLeader and GetLeader() work
var leaseName = $"test-{NeonHelper.CreateBase36Uuid()}";
bool isLeading = false;
string leader = null;
var config = new LeaderElectionConfig(fixture.K8s, @namespace: KubeNamespace.Default, leaseName: leaseName, identity: "instance-0");
Task electorTask;
try
{
var elector = new LeaderElector(
config: config,
onStartedLeading: () => isLeading = true,
onStoppedLeading: () => isLeading = false,
onNewLeader: identity => leader = identity);
using (elector)
{
electorTask = elector.RunAsync();
NeonHelper.WaitFor(() => isLeading, timeout: MaxWaitTime);
Assert.True(isLeading);
Assert.True(elector.IsLeader);
Assert.Equal("instance-0", leader);
Assert.Equal("instance-0", elector.Leader);
}
// Ensure that the elector task completes.
await electorTask.WaitAsync(timeout : MaxWaitTime);
}
finally
{
await config.K8s.DeleteNamespacedLeaseWithHttpMessagesAsync(leaseName, config.Namespace);
}
}
private Task StartLeaderElector(CancellationToken stoppingToken)
{
var configMapLock = new ConfigMapLock(_client, _configuration.PodNamespace, _configuration.ElectionConfigMapName, _configuration.PodName);
var leaderElectionConfig = new LeaderElectionConfig(configMapLock)
{
LeaseDuration = TimeSpan.FromMilliseconds(1000),
RetryPeriod = TimeSpan.FromMilliseconds(500),
RenewDeadline = TimeSpan.FromMilliseconds(600),
};
return(Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfig);
leaderElector.OnStartedLeading += () =>
{
_logger.LogTrace("I am the leader");
_leaderEvent.Set();
};
leaderElector.OnStoppedLeading += () =>
{
_logger.LogTrace("I am NOT the leader");
_leaderEvent.Reset();
};
leaderElector.OnNewLeader += leader =>
{
_logger.LogInformation($"New leader elected. Identity={leader}");
};
while (!stoppingToken.IsCancellationRequested)
{
leaderElector.RunAsync().Wait(stoppingToken);
}
_logger.LogTrace("Election finished");
},
stoppingToken));
}
public void LeaderIntegrationTest()
{
var client = CreateClient();
var namespaceParameter = "default";
void Cleanup()
{
var endpoints = client.ListNamespacedEndpoints(namespaceParameter);
void DeleteEndpoints(string name)
{
while (endpoints.Items.Any(p => p.Metadata.Name == name))
{
try
{
client.DeleteNamespacedEndpoints(name, namespaceParameter);
}
catch (HttpOperationException e)
{
if (e.Response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return;
}
}
}
}
DeleteEndpoints("leaderendpoint");
}
var cts = new CancellationTokenSource();
var leader1acq = new ManualResetEvent(false);
var leader1lose = new ManualResetEvent(false);
try
{
Cleanup();
var tasks = new List <Task>();
// 1
{
var l = new EndpointsLock(client, namespaceParameter, "leaderendpoint", "leader1");
var le = new LeaderElector(new LeaderElectionConfig(l)
{
LeaseDuration = TimeSpan.FromSeconds(1),
RetryPeriod = TimeSpan.FromMilliseconds(400),
});
le.OnStartedLeading += () => leader1acq.Set();
le.OnStoppedLeading += () => leader1lose.Set();
tasks.Add(le.RunAsync(cts.Token));
}
// wait 1 become leader
Assert.True(leader1acq.WaitOne(TimeSpan.FromSeconds(30)));
// 2
{
var l = new EndpointsLock(client, namespaceParameter, "leaderendpoint", "leader2");
var le = new LeaderElector(new LeaderElectionConfig(l)
{
LeaseDuration = TimeSpan.FromSeconds(1),
RetryPeriod = TimeSpan.FromMilliseconds(400),
});
var leader2init = new ManualResetEvent(false);
le.OnNewLeader += _ =>
{
leader2init.Set();
};
tasks.Add(le.RunAsync());
Assert.True(leader2init.WaitOne(TimeSpan.FromSeconds(30)));
Assert.Equal("leader1", le.GetLeader());
cts.Cancel();
Assert.True(leader1lose.WaitOne(TimeSpan.FromSeconds(30)));
Task.Delay(TimeSpan.FromSeconds(3)).Wait();
Assert.True(le.IsLeader());
}
Task.WaitAll(tasks.ToArray(), TimeSpan.FromSeconds(30));
}
finally
{
Cleanup();
}
}
public void LeaderElection()
{
var electionHistory = new List <string>();
var leadershipHistory = new List <string>();
var renewCountA = 3;
var mockLockA = new MockResourceLock("mockA")
{
UpdateWillFail = () => renewCountA <= 0
};
mockLockA.OnCreate += (_) =>
{
renewCountA--;
electionHistory.Add("A creates record");
leadershipHistory.Add("A gets leadership");
};
mockLockA.OnUpdate += (_) =>
{
renewCountA--;
electionHistory.Add("A updates record");
};
mockLockA.OnChange += (_) => { leadershipHistory.Add("A gets leadership"); };
var leaderElectionConfigA = new LeaderElectionConfig(mockLockA)
{
LeaseDuration = TimeSpan.FromMilliseconds(500),
RetryPeriod = TimeSpan.FromMilliseconds(300),
RenewDeadline = TimeSpan.FromMilliseconds(400),
};
var renewCountB = 4;
var mockLockB = new MockResourceLock("mockB")
{
UpdateWillFail = () => renewCountB <= 0
};
mockLockB.OnCreate += (_) =>
{
renewCountB--;
electionHistory.Add("B creates record");
leadershipHistory.Add("B gets leadership");
};
mockLockB.OnUpdate += (_) =>
{
renewCountB--;
electionHistory.Add("B updates record");
};
mockLockB.OnChange += (_) => { leadershipHistory.Add("B gets leadership"); };
var leaderElectionConfigB = new LeaderElectionConfig(mockLockB)
{
LeaseDuration = TimeSpan.FromMilliseconds(500),
RetryPeriod = TimeSpan.FromMilliseconds(300),
RenewDeadline = TimeSpan.FromMilliseconds(400),
};
var lockAStopLeading = new ManualResetEvent(false);
var testLeaderElectionLatch = new CountdownEvent(4);
Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfigA);
leaderElector.OnStartedLeading += () =>
{
leadershipHistory.Add("A starts leading");
testLeaderElectionLatch.Signal();
};
leaderElector.OnStoppedLeading += () =>
{
leadershipHistory.Add("A stops leading");
testLeaderElectionLatch.Signal();
lockAStopLeading.Set();
};
leaderElector.RunAsync().Wait();
});
lockAStopLeading.WaitOne(TimeSpan.FromSeconds(3));
Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfigB);
leaderElector.OnStartedLeading += () =>
{
leadershipHistory.Add("B starts leading");
testLeaderElectionLatch.Signal();
};
leaderElector.OnStoppedLeading += () =>
{
leadershipHistory.Add("B stops leading");
testLeaderElectionLatch.Signal();
};
leaderElector.RunAsync().Wait();
});
testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(10));
Assert.Equal(7, electionHistory.Count);
Assert.True(electionHistory.SequenceEqual(
new[]
{
"A creates record", "A updates record", "A updates record",
"B updates record", "B updates record", "B updates record", "B updates record",
}));
Assert.True(leadershipHistory.SequenceEqual(
new[]
{
"A gets leadership", "A starts leading", "A stops leading",
"B gets leadership", "B starts leading", "B stops leading",
}));
}
public void LeaderElectionWithRenewDeadline()
{
var electionHistory = new List <string>();
var leadershipHistory = new List <string>();
var renewCount = 3;
var mockLock = new MockResourceLock("mock")
{
UpdateWillFail = () => renewCount <= 0,
};
mockLock.OnCreate += _ =>
{
renewCount--;
electionHistory.Add("create record");
leadershipHistory.Add("get leadership");
};
mockLock.OnUpdate += _ =>
{
renewCount--;
electionHistory.Add("update record");
};
mockLock.OnChange += _ => { electionHistory.Add("change record"); };
mockLock.OnTryUpdate += _ => { electionHistory.Add("try update record"); };
var leaderElectionConfig = new LeaderElectionConfig(mockLock)
{
LeaseDuration = TimeSpan.FromMilliseconds(1000),
RetryPeriod = TimeSpan.FromMilliseconds(200),
RenewDeadline = TimeSpan.FromMilliseconds(650),
};
var countdown = new CountdownEvent(2);
Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfig);
leaderElector.OnStartedLeading += () =>
{
leadershipHistory.Add("start leading");
countdown.Signal();
};
leaderElector.OnStoppedLeading += () =>
{
leadershipHistory.Add("stop leading");
countdown.Signal();
};
leaderElector.RunAsync().Wait();
});
countdown.Wait(TimeSpan.FromSeconds(10));
// TODO flasky
// Assert.Equal(9, electionHistory.Count);
// Assert.True(electionHistory.SequenceEqual(new[]
// {
// "create record", "try update record", "update record", "try update record", "update record",
// "try update record", "try update record", "try update record", "try update record",
// }));
Assert.True(electionHistory.Take(7).SequenceEqual(new[]
{
"create record", "try update record", "update record", "try update record", "update record",
"try update record", "try update record",
}));
Assert.True(leadershipHistory.SequenceEqual(new[] { "get leadership", "start leading", "stop leading" }));
}
