public async Task <bool> UpdateAsync(LeaderElectionRecord record, CancellationToken cancellationToken = default)
{
var metaObj = Interlocked.CompareExchange(ref metaObjCache, null, null);
if (metaObj == null)
{
throw new InvalidOperationException("endpoint not initialized, call get or create first");
}
metaObj = SetLeaderElectionRecord(record, metaObj);
try
{
var replacedObj = await ReplaceMetaObjectAsync(client, metaObj, name, ns, cancellationToken).ConfigureAwait(false);
Interlocked.Exchange(ref metaObjCache, replacedObj);
return(true);
}
catch (HttpOperationException)
{
// ignore
}
return(false);
}
public async Task <bool> CreateAsync(LeaderElectionRecord record, CancellationToken cancellationToken = default)
{
var metaObj = new T
{
Metadata = new V1ObjectMeta()
{
Name = name, NamespaceProperty = ns
},
};
metaObj = SetLeaderElectionRecord(record, metaObj);
try
{
var createdObj = await CreateMetaObjectAsync(client, metaObj, ns, cancellationToken)
.ConfigureAwait(false);
Interlocked.Exchange(ref metaObjCache, createdObj);
return(true);
}
catch (HttpOperationException)
{
// ignore
}
return(false);
}
protected override T SetLeaderElectionRecord(LeaderElectionRecord record, T metaObj)
{
metaObj.SetAnnotation(
LeaderElectionRecordAnnotationKey,
JsonConvert.SerializeObject(record, serializationSettings));
return(metaObj);
}
private async Task UpdateLeaderRecord(LeaderElectionRecord record, CancellationToken token)
{
this.logger.LogInformation("Updating leader record {leaderObjectName}", this.leaderObjectName);
var annotations = new Dictionary <string, string>()
{
[LeaderAnnotationName] = JsonSerializer.Serialize(record),
};
await this.client.PutEndpointAnnotationsAsync(this.ns, this.leaderObjectName, annotations, token);
}
public Task <bool> CreateAsync(
LeaderElectionRecord record,
CancellationToken cancellationToken = default)
{
lock (Lockobj)
{
if (leaderRecord != null)
{
return(Task.FromResult(false));
}
leaderRecord = record;
OnCreate?.Invoke(record);
return(Task.FromResult(true));
}
}
public Task <bool> UpdateAsync(LeaderElectionRecord record, CancellationToken cancellationToken = default)
{
lock (Lockobj)
{
OnTryUpdate?.Invoke(record);
if (UpdateWillFail?.Invoke() == true)
{
return(Task.FromResult(false));
}
var oldRecord = leaderRecord;
leaderRecord = record;
OnUpdate?.Invoke(record);
if (oldRecord?.HolderIdentity != record.HolderIdentity)
{
OnChange?.Invoke(record);
}
return(Task.FromResult(true));
}
}
protected override V1Lease SetLeaderElectionRecord(LeaderElectionRecord record, V1Lease metaObj)
{
if (record == null)
{
throw new NullReferenceException(nameof(record));
}
if (metaObj == null)
{
throw new NullReferenceException(nameof(metaObj));
}
metaObj.Spec = new V1LeaseSpec()
{
AcquireTime = record.AcquireTime,
HolderIdentity = record.HolderIdentity,
LeaseTransitions = record.LeaderTransitions,
LeaseDurationSeconds = record.LeaseDurationSeconds,
RenewTime = record.RenewTime,
};
return(metaObj);
}
public static void ResetGloablRecord()
{
leaderRecord = null;
}
protected abstract T SetLeaderElectionRecord(LeaderElectionRecord record, T metaObj);
public async Task BackgroundTaskLoop(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
var record = await this.EnsureEndpointExists(token);
this.IsLeader = record.HolderIdentity == this.identity;
this.logger.LogInformation("LeaderElection/BackgroundTaskLoop: IsLeader={isLeader}", this.IsLeader);
var now = DateTimeOffset.UtcNow;
var expireDate = new DateTimeOffset((DateTime)(record.RenewTime !)) + TimeSpan.FromSeconds(record.LeaseDurationSeconds);
var timeToLive = expireDate - now;
if (this.IsLeader)
{
if (timeToLive > TimeToRefreshBeforeExpiry)
{
await Task.Delay(timeToLive - TimeToRefreshBeforeExpiry, token);
}
var newRecord = new LeaderElectionRecord()
{
HolderIdentity = this.identity,
LeaseDurationSeconds = (int)this.leaseDuration.TotalSeconds,
AcquireTime = record.AcquireTime,
RenewTime = DateTime.UtcNow,
LeaderTransitions = record.LeaderTransitions,
};
await this.UpdateLeaderRecord(newRecord, token);
}
else
{
if (timeToLive < TimeSpan.Zero)
{
var newRecord = new LeaderElectionRecord()
{
HolderIdentity = this.identity,
LeaseDurationSeconds = (int)this.leaseDuration.TotalSeconds,
AcquireTime = DateTime.UtcNow,
RenewTime = DateTime.UtcNow,
LeaderTransitions = record.LeaderTransitions + 1,
};
// it is expired--claim leadership and restart loop
await this.UpdateLeaderRecord(newRecord, token);
await Task.Delay(TimeToWaitAfterSettingLeader, token);
continue;
}
else
{
// Give the leader a little extra time to renew
await Task.Delay(timeToLive + TimeToWaitAfterSettingLeader, token);
}
}
}
catch (TaskCanceledException e)
{
this.logger.LogError(e, "BackgroundTaskLoop cancelled: {exception}", e.ToString());
}
}
}
protected bool Equals(LeaderElectionRecord other)
{
return(this.HolderIdentity == other?.HolderIdentity &&
Nullable.Equals(this.AcquireTime, other?.AcquireTime) &&
Nullable.Equals(this.RenewTime, other?.RenewTime));
}
private async Task <bool> TryAcquireOrRenew(CancellationToken cancellationToken)
{
var l = config.Lock;
var leaderElectionRecord = new LeaderElectionRecord()
{
HolderIdentity = l.Identity,
LeaseDurationSeconds = ( int )config.LeaseDuration.TotalSeconds,
AcquireTime = DateTime.UtcNow,
RenewTime = DateTime.UtcNow,
LeaderTransitions = 0,
};
// 1. obtain or create the ElectionRecord
LeaderElectionRecord oldLeaderElectionRecord = null;
try
{
oldLeaderElectionRecord = await l.GetAsync(cancellationToken).ConfigureAwait(false);
}
catch (HttpOperationException e)
{
Logger.LogWarning(e, $"ERR {e.Response.StatusCode}");
if (e.Response.StatusCode != HttpStatusCode.NotFound)
{
return(false);
}
}
if (oldLeaderElectionRecord?.AcquireTime == null ||
oldLeaderElectionRecord?.RenewTime == null ||
oldLeaderElectionRecord?.HolderIdentity == null)
{
var created = await l.CreateAsync(leaderElectionRecord, cancellationToken).ConfigureAwait(false);
if (created)
{
observedRecord = leaderElectionRecord;
observedTime = DateTimeOffset.Now;
return(true);
}
return(false);
}
// 2. Record obtained, check the Identity & Time
if (!Equals(observedRecord, oldLeaderElectionRecord))
{
observedRecord = oldLeaderElectionRecord;
observedTime = DateTimeOffset.Now;
}
if (!string.IsNullOrEmpty(oldLeaderElectionRecord.HolderIdentity) &&
observedTime + config.LeaseDuration > DateTimeOffset.Now &&
!IsLeader())
{
// lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity
return(false);
}
// 3. We're going to try to update. The leaderElectionRecord is set to it's default
// here. Let's correct it before updating.
if (IsLeader())
{
leaderElectionRecord.AcquireTime = oldLeaderElectionRecord.AcquireTime;
leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions;
}
else
{
leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions + 1;
}
var updated = await l.UpdateAsync(leaderElectionRecord, cancellationToken).ConfigureAwait(false);
if (!updated)
{
return(false);
}
observedRecord = leaderElectionRecord;
observedTime = DateTimeOffset.Now;
return(true);
}
public async Task <bool> UpdateAsync(LeaderElectionRecord record, CancellationToken cancellationToken = default)
{
return(await primary.UpdateAsync(record, cancellationToken).ConfigureAwait(false) &&
await secondary.UpdateAsync(record, cancellationToken).ConfigureAwait(false));
}
protected override T SetLeaderElectionRecord(LeaderElectionRecord record, T metaObj)
{
metaObj.SetAnnotation(LeaderElectionRecordAnnotationKey, KubernetesJson.Serialize(record));
return(metaObj);
}
