/// <summary>
/// Check the <see cref="V1Lease"/> object for the leader election.
/// </summary>
/// <returns>A Task.</returns>
internal async Task CheckLeaderLease()
{
if (_namespace.Length == 0)
{
_logger.LogTrace("Fetching namespace for leader election.");
_namespace = await _client.GetCurrentNamespace();
}
_logger.LogTrace(@"Fetch V1Lease object for operator ""{operator}"".", _settings.Name);
var lease = await _client.Get <V1Lease>(_leaseName, _namespace);
// If the lease does not exist, create it, set this instance as leader
// fire the appropriate event, return.
if (lease == null)
{
_logger.LogInformation(
@"There was no lease for operator ""{operator}"". Creating one and electing ""{hostname}"" as leader.",
_settings.Name,
_hostname);
try
{
await _client.Create(
new V1Lease(
$"{V1Lease.KubeGroup}/{V1Lease.KubeApiVersion}",
V1Lease.KubeKind,
new V1ObjectMeta(
name: _leaseName,
namespaceProperty: _namespace,
annotations: new Dictionary <string, string> {
{ "leader-elector", _settings.Name }
}),
new V1LeaseSpec(
DateTime.UtcNow,
_hostname,
_settings.LeaderElectionLeaseDuration,
0,
DateTime.UtcNow)));
_election.LeadershipChanged(LeaderState.Leader);
}
catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogInformation("Another instance of the operator was faster. Falling back to candiate.");
_election.LeadershipChanged(LeaderState.Candidate);
}
catch (HttpOperationException ex)
{
_logger.LogCritical(
ex,
@"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""",
_hostname,
_settings.Name,
$"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}");
}
catch (Exception ex)
{
_logger.LogCritical(
ex,
@"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".",
_hostname,
_settings.Name);
}
return;
}
/*
* If the lease exists, check if this instance is the leader.
* If it is, update the renew time, and update the entity.
* If it isn't and the lease time is in the past,
* set the leader, update the entity, trigger event.
*/
if (lease.Spec.HolderIdentity == _hostname)
{
_logger.LogDebug(
@"The instance ""{hostname}"" is still the leader for operator ""{operator}"".",
_hostname,
_settings.Name);
lease.Spec.RenewTime = DateTime.UtcNow;
try
{
await _client.Update(lease);
_election.LeadershipChanged(LeaderState.Leader);
}
catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogWarning("Another instance updated the lease. Retry on next cycle.");
}
catch (HttpOperationException ex)
{
_logger.LogCritical(
ex,
@"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""",
_hostname,
_settings.Name,
$"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}");
}
catch (Exception ex)
{
_logger.LogCritical(
ex,
@"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".",
_hostname,
_settings.Name);
}
return;
}
if (lease.Spec.RenewTime.HasValue &&
lease.Spec.RenewTime.Value +
TimeSpan.FromSeconds(lease.Spec.LeaseDurationSeconds ?? _settings.LeaderElectionLeaseDuration) <
DateTime.UtcNow)
{
_logger.LogInformation(
@"The lease for operator ""{operator}"" ran out. Electing ""{hostname}"" as leader.",
_settings.Name,
_hostname);
lease.Spec.AcquireTime = DateTime.UtcNow;
lease.Spec.RenewTime = DateTime.UtcNow;
lease.Spec.HolderIdentity = _hostname;
lease.Spec.LeaseTransitions ??= 0;
lease.Spec.LeaseTransitions += 1;
try
{
await _client.Update(lease);
_election.LeadershipChanged(LeaderState.Leader);
}
catch (HttpOperationException e) when(e.Response.StatusCode == HttpStatusCode.Conflict)
{
_logger.LogWarning("Another instance updated the lease. Retry on next cycle.");
}
catch (HttpOperationException ex)
{
_logger.LogCritical(
ex,
@"A http error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"". Response Message: ""{response}""",
_hostname,
_settings.Name,
$"Phrase: {ex.Response.ReasonPhrase}\nContent: {ex.Response.Content}");
}
catch (Exception ex)
{
_logger.LogCritical(
ex,
@"A generic error happened during leader election check of instance ""{hostname}"" of operator ""{operator}"".",
_hostname,
_settings.Name);
}
return;
}
_logger.LogDebug(
@"The lease for operator ""{operator}"" did not ran out, staying/becoming candidate.",
_settings.Name);
_election.LeadershipChanged(LeaderState.Candidate);
}
public async Task PublishAsync(
IKubernetesObject <V1ObjectMeta> resource,
string reason,
string message,
EventType type = EventType.Normal)
{
_logger.LogTrace(
"Encoding event name with: {resourceName}.{resourceNamespace}.{reason}.{message}.{type}.",
resource.Name(),
resource.Namespace(),
reason,
message,
type);
var eventName =
Base32.Rfc4648.Encode(
SHA512.HashData(
Encoding.UTF8.GetBytes($"{resource.Name()}.{resource.Namespace()}.{reason}.{message}.{type}")));
_logger.LogTrace(@"Search or create event with name ""{name}"".", eventName);
var @event = await _client.Get <Corev1Event>(eventName, resource.Namespace()) ??
new Corev1Event
{
Kind = Corev1Event.KubeKind,
ApiVersion = $"{Corev1Event.KubeGroup}/{Corev1Event.KubeApiVersion}",
Metadata = new V1ObjectMeta
{
Name = eventName,
NamespaceProperty = resource.Namespace(),
Annotations = new Dictionary <string, string>
{
{ "nameHash", "sha512" },
{ "nameEncoding", "Base32 / RFC 4648" },
},
},
Type = type.ToString(),
Reason = reason,
Message = message,
ReportingComponent = _settings.Name,
ReportingInstance = Environment.MachineName,
Source = new V1EventSource {
Component = _settings.Name
},
InvolvedObject = resource.MakeObjectReference(),
FirstTimestamp = DateTime.UtcNow,
LastTimestamp = DateTime.UtcNow,
Count = 0,
};
@event.Count++;
@event.LastTimestamp = DateTime.UtcNow;
_logger.LogTrace(
"Save event with new count {count} and last timestamp {timestamp}",
@event.Count,
@event.LastTimestamp);
await _client.Save(@event);
_logger.LogInformation(
@"Created or updated event with name ""{name}"" to new count {count} on resource ""{kind}/{name}"".",
eventName,
@event.Count,
resource.Kind,
resource.Name());
}