public async Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation(@"Startup Leader Elector for operator ""{operatorName}"".", _settings.Name);
_leaseCheck?.Dispose();
_leaseCheck = new Timer(
TimeSpan.FromSeconds(_settings.LeaderElectionCheckInterval).TotalMilliseconds)
{
AutoReset = true,
};
_logger.LogTrace("Fetching namespace for leader election.");
_namespace = await _client.GetCurrentNamespace();
_operatorDeployment = (await _client.List <V1Deployment>(
_namespace,
new EqualsSelector("operator-deployment", _settings.Name))).FirstOrDefault();
if (_operatorDeployment != null)
{
_operatorDeployment.Kind = V1Deployment.KubeKind;
_operatorDeployment.ApiVersion = $"{V1Deployment.KubeGroup}/{V1Deployment.KubeApiVersion}";
}
#if DEBUG
_election.LeadershipChanged(LeaderState.Leader);
#else
_leaseCheck.Start();
_leaseCheck.Elapsed += async(_, __) => await CheckLeaderLease();
await CheckLeaderLease();
#endif
}
/// <summary>
/// Check the <see cref="V1Lease"/> object for the leader election.
/// </summary>
/// <returns>A Task.</returns>
internal async Task CheckLeaderLease()
{
if (_namespace.Length == 0)
{
_logger.LogTrace("Fetching namespace for leader election.");
_namespace = await _client.GetCurrentNamespace();
}
_logger.LogTrace(@"Fetch V1Lease object for operator ""{operator}"".", _settings.Name);
var lease = await _client.Get <V1Lease>(_leaseName, _namespace);
// If the lease does not exist, create it, set this instance as leader
// fire the appropriate event, return.
if (lease == null)
{
_logger.LogInformation(
@"There was no lease for operator ""{operator}"". Creating one and electing ""{hostname}"" as leader.",
_settings.Name,
_hostname);
try
{
await _client.Create(
new V1Lease(
$"{V1Lease.KubeGroup}/{V1Lease.KubeApiVersion}",
V1Lease.KubeKind,
new V1ObjectMeta(
name: _leaseName,
namespaceProperty: _namespace,
annotations: new Dictionary <string, string> {
{ "leader-elector", _settings.Name }
}),
new V1LeaseSpec(
DateTime.UtcNow,
_hostname,
_settings.LeaderElectionLeaseDuration,
0,
DateTime.UtcNow)));
_election.LeadershipChanged(LeaderState.Leader);
}
catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogInformation("Another instance of the operator was faster. Falling back to candiate.");
_election.LeadershipChanged(LeaderState.Candidate);
}
catch (HttpOperationException ex)
{
_logger.LogCritical(
ex,
@"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""",
_hostname,
_settings.Name,
$"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}");
}
catch (Exception ex)
{
_logger.LogCritical(
ex,
@"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".",
_hostname,
_settings.Name);
}
return;
}
/*
* If the lease exists, check if this instance is the leader.
* If it is, update the renew time, and update the entity.
* If it isn't and the lease time is in the past,
* set the leader, update the entity, trigger event.
*/
if (lease.Spec.HolderIdentity == _hostname)
{
_logger.LogDebug(
@"The instance ""{hostname}"" is still the leader for operator ""{operator}"".",
_hostname,
_settings.Name);
lease.Spec.RenewTime = DateTime.UtcNow;
try
{
await _client.Update(lease);
_election.LeadershipChanged(LeaderState.Leader);
}
catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogWarning("Another instance updated the lease. Retry on next cycle.");
}
catch (HttpOperationException ex)
{
_logger.LogCritical(
ex,
@"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""",
_hostname,
_settings.Name,
$"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}");
}
catch (Exception ex)
{
_logger.LogCritical(
ex,
@"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".",
_hostname,
_settings.Name);
}
return;
}
if (lease.Spec.RenewTime.HasValue &&
lease.Spec.RenewTime.Value +
TimeSpan.FromSeconds(lease.Spec.LeaseDurationSeconds ?? _settings.LeaderElectionLeaseDuration) <
DateTime.UtcNow)
{
_logger.LogInformation(
@"The lease for operator ""{operator}"" ran out. Electing ""{hostname}"" as leader.",
_settings.Name,
_hostname);
lease.Spec.AcquireTime = DateTime.UtcNow;
lease.Spec.RenewTime = DateTime.UtcNow;
lease.Spec.HolderIdentity = _hostname;
lease.Spec.LeaseTransitions ??= 0;
lease.Spec.LeaseTransitions += 1;
try
{
await _client.Update(lease);
_election.LeadershipChanged(LeaderState.Leader);
}
catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogWarning("Another instance updated the lease. Retry on next cycle.");
}
catch (HttpOperationException ex)
{
_logger.LogCritical(
ex,
@"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""",
_hostname,
_settings.Name,
$"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}");
}
catch (Exception ex)
{
_logger.LogCritical(
ex,
@"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".",
_hostname,
_settings.Name);
}
return;
}
_logger.LogDebug(
@"The lease for operator ""{operator}"" did not ran out, staying/becoming candidate.",
_settings.Name);
_election.LeadershipChanged(LeaderState.Candidate);
}
public async Task Should_Return_Namespace()
{
var ns = await _client.GetCurrentNamespace();
ns.Should().Be("default");
}
public async Task <int> OnExecuteAsync(CommandLineApplication app)
{
var @namespace = await _client.GetCurrentNamespace();
using var certManager = new CertificateGenerator(app.Out);
#if DEBUG
CertificatesPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
CaCertificatesPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
await certManager.CreateCaCertificateAsync(CaCertificatesPath);
#endif
Directory.CreateDirectory(CertificatesPath);
File.Copy(Path.Join(CaCertificatesPath, "ca.pem"), Path.Join(Path.Join(CertificatesPath, "ca.pem")));
await certManager.CreateServerCertificateAsync(
CertificatesPath,
_settings.Name,
@namespace,
Path.Join(CaCertificatesPath, "ca.pem"),
Path.Join(CaCertificatesPath, "ca-key.pem"));
var deployment = (await _client.List <V1Deployment>(
@namespace,
new EqualsSelector("operator-deployment", _settings.Name))).FirstOrDefault();
if (deployment != null)
{
deployment.Kind = V1Deployment.KubeKind;
deployment.ApiVersion = $"{V1Deployment.KubeGroup}/{V1Deployment.KubeApiVersion}";
}
await app.Out.WriteLineAsync("Create service.");
await _client.Delete <V1Service>(_settings.Name, @namespace);
await _client.Create(
new V1Service(
V1Service.KubeApiVersion,
V1Service.KubeKind,
new V1ObjectMeta(
name: _settings.Name,
namespaceProperty: @namespace,
ownerReferences: deployment != null
? new List <V1OwnerReference>
{
deployment.MakeOwnerReference(),
}
: null,
labels: new Dictionary <string, string>
{
{ "operator", _settings.Name },
{ "usage", "webhook-service" },
}),
new V1ServiceSpec
{
Ports = new List <V1ServicePort>
{
new()
{
Name = "https",
TargetPort = "https",
Port = 443,
},
},