public async Task Single_WithoutActionsOrCounters()
{
// Verify that the elector works without callback actions.
var leaseName = $"test-{NeonHelper.CreateBase36Uuid()}";
var config = new LeaderElectionConfig(fixture.K8s, @namespace: KubeNamespace.Default, leaseName: leaseName, identity: "instance-0");
var elector = new LeaderElector(config);
Task electorTask;
try
{
using (elector)
{
electorTask = elector.RunAsync();
NeonHelper.WaitFor(() => elector.IsLeader, timeout: MaxWaitTime);
Assert.True(elector.IsLeader);
Assert.Equal("instance-0", elector.Leader);
}
// Ensure that the elector task completes.
await electorTask.WaitAsync(timeout : MaxWaitTime);
}
finally
{
await config.K8s.DeleteNamespacedLeaseWithHttpMessagesAsync(leaseName, config.Namespace);
}
}
/// <summary>
/// Default constructor.
/// </summary>
/// <param name="k8s">The <see cref="IKubernetes"/> client used by the controller.</param>
/// <param name="options">
/// Optionally specifies options that customize the resource manager's behavior. Reasonable
/// defaults will be used when this isn't specified.
/// </param>
/// <param name="filter">
/// <para>
/// Optionally specifies a predicate to be use for filtering the resources to be managed.
/// This can be useful for situations where multiple operator instances will partition
/// and handle the resources amongst themselves. A good example is a node based operator
/// that handles only the resources associated with the node.
/// </para>
/// <para>
/// Your filter should examine the resource passed and return <c>true</c> when the resource
/// should be managed by this resource manager. The default filter always returns <c>true</c>.
/// </para>
/// </param>
/// <param name="logger">Optionally specifies the logger to be used by the instance.</param>
/// <param name="leaderConfig">
/// Optionally specifies the <see cref="LeaderElectionConfig"/> to be used to control
/// whether only a single entity is managing a specific resource kind at a time. See
/// the <b>LEADER ELECTION SECTION</b> in the <see cref="ResourceManager{TResource, TController}"/>
/// remarks for more information.
/// </param>
public ResourceManager(
IKubernetes k8s,
ResourceManagerOptions options = null,
Func <TEntity, bool> filter = null,
INeonLogger logger = null,
LeaderElectionConfig leaderConfig = null)
{
Covenant.Requires <ArgumentNullException>(k8s != null, nameof(k8s));
this.k8s = k8s; // $todo(jefflill): Can we obtain this from KubeOps or the [IServiceProvider] somehow?
this.options = options ?? new ResourceManagerOptions();
this.filter = filter ?? new Func <TEntity, bool>(resource => true);
this.log = logger ?? LogManager.Default.GetLogger($"Neon.Kube.Operator.ResourceManager({typeof(TEntity).Name})");
this.leaderConfig = leaderConfig;
options.Validate();
// $todo(jefflill): https://github.com/nforgeio/neonKUBE/issues/1589
//
// Locate the controller's constructor that has a single [IKubernetes] parameter.
var controllerType = typeof(TController);
this.controllerConstructor = controllerType.GetConstructor(new Type[] { typeof(IKubernetes) });
if (this.controllerConstructor == null)
{
throw new NotSupportedException($"Controller type [{controllerType.FullName}] does not have a constructor accepting a single [{nameof(IKubernetes)}] parameter. This is currently required.");
}
}
public async Task Single_WithCounters()
{
// Verify that the elector can increment performance counters.
var leaseName = $"test-{NeonHelper.CreateBase36Uuid()}";
var promotionCounter = Metrics.CreateCounter("test_promotions", string.Empty);
var demotionCounter = Metrics.CreateCounter("test_demotions", string.Empty);
var newLeaderCounter = Metrics.CreateCounter("test_newleaders", string.Empty);
var config = new LeaderElectionConfig(
k8s: fixture.K8s,
@namespace: KubeNamespace.Default,
leaseName: leaseName,
identity: "instance-0",
promotionCounter: promotionCounter,
demotionCounter: demotionCounter,
newLeaderCounter: newLeaderCounter);
var elector = new LeaderElector(config);
Task electorTask;
try
{
using (elector)
{
electorTask = elector.RunAsync();
NeonHelper.WaitFor(() => elector.IsLeader, timeout: MaxWaitTime);
Assert.True(elector.IsLeader);
Assert.Equal("instance-0", elector.Leader);
}
// Ensure that the elector task completes.
await electorTask.WaitAsync(timeout : MaxWaitTime);
// Ensure that the counters are correct.
Assert.Equal(1, promotionCounter.Value);
Assert.Equal(0, demotionCounter.Value); // We don't see demotions when the elector is disposed
Assert.Equal(2, newLeaderCounter.Value); // We do see a leadership change when the elector is disposed
}
finally
{
await config.K8s.DeleteNamespacedLeaseWithHttpMessagesAsync(leaseName, config.Namespace);
}
}
public async Task Single_WithoutCounters()
{
// Verify that we can create a single [LeaderElector] instance and that:
//
// 1. The [OnNewLeader] action is called
// 2. The [OnStartedLeader] action is called
// 3. The new leader matches the current instance
// 4. IsLeader and GetLeader() work
var leaseName = $"test-{NeonHelper.CreateBase36Uuid()}";
bool isLeading = false;
string leader = null;
var config = new LeaderElectionConfig(fixture.K8s, @namespace: KubeNamespace.Default, leaseName: leaseName, identity: "instance-0");
Task electorTask;
try
{
var elector = new LeaderElector(
config: config,
onStartedLeading: () => isLeading = true,
onStoppedLeading: () => isLeading = false,
onNewLeader: identity => leader = identity);
using (elector)
{
electorTask = elector.RunAsync();
NeonHelper.WaitFor(() => isLeading, timeout: MaxWaitTime);
Assert.True(isLeading);
Assert.True(elector.IsLeader);
Assert.Equal("instance-0", leader);
Assert.Equal("instance-0", elector.Leader);
}
// Ensure that the elector task completes.
await electorTask.WaitAsync(timeout : MaxWaitTime);
}
finally
{
await config.K8s.DeleteNamespacedLeaseWithHttpMessagesAsync(leaseName, config.Namespace);
}
}
private Task StartLeaderElector(CancellationToken stoppingToken)
{
var configMapLock = new ConfigMapLock(_client, _configuration.PodNamespace, _configuration.ElectionConfigMapName, _configuration.PodName);
var leaderElectionConfig = new LeaderElectionConfig(configMapLock)
{
LeaseDuration = TimeSpan.FromMilliseconds(1000),
RetryPeriod = TimeSpan.FromMilliseconds(500),
RenewDeadline = TimeSpan.FromMilliseconds(600),
};
return(Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfig);
leaderElector.OnStartedLeading += () =>
{
_logger.LogTrace("I am the leader");
_leaderEvent.Set();
};
leaderElector.OnStoppedLeading += () =>
{
_logger.LogTrace("I am NOT the leader");
_leaderEvent.Reset();
};
leaderElector.OnNewLeader += leader =>
{
_logger.LogInformation($"New leader elected. Identity={leader}");
};
while (!stoppingToken.IsCancellationRequested)
{
leaderElector.RunAsync().Wait(stoppingToken);
}
_logger.LogTrace("Election finished");
},
stoppingToken));
}
/// <summary>
/// Starts the controller.
/// </summary>
/// <param name="k8s">The <see cref="IKubernetes"/> client to use.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
public static async Task StartAsync(IKubernetes k8s)
{
Covenant.Requires <ArgumentNullException>(k8s != null, nameof(k8s));
// Load the configuration settings.
var leaderConfig =
new LeaderElectionConfig(
k8s,
@namespace: KubeNamespace.NeonSystem,
leaseName: $"{Program.Service.Name}.nodetask",
identity: Pod.Name,
promotionCounter: Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_promoted", "Leader promotions"),
demotionCounter: Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_demoted", "Leader demotions"),
newLeaderCounter: Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_newLeader", "Leadership changes"));
var options = new ResourceManagerOptions()
{
IdleInterval = Program.Service.Environment.Get("NODETASK_IDLE_INTERVAL", TimeSpan.FromSeconds(1)),
ErrorMinRequeueInterval = Program.Service.Environment.Get("NODETASK_ERROR_MIN_REQUEUE_INTERVAL", TimeSpan.FromSeconds(15)),
ErrorMaxRetryInterval = Program.Service.Environment.Get("NODETASK_ERROR_MAX_REQUEUE_INTERVAL", TimeSpan.FromSeconds(60)),
IdleCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle", "IDLE events processed."),
ReconcileCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle", "RECONCILE events processed."),
DeleteCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle", "DELETED events processed."),
StatusModifyCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle", "STATUS-MODIFY events processed."),
IdleErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle_error", "Failed NodeTask IDLE event processing."),
ReconcileErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_reconcile_error", "Failed NodeTask RECONCILE event processing."),
DeleteErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_delete_error", "Failed NodeTask DELETE event processing."),
StatusModifyErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_statusmodify_error", "Failed NodeTask STATUS-MODIFY events processing.")
};
resourceManager = new ResourceManager <V1NeonNodeTask, NodeTaskController>(
k8s,
options: options,
leaderConfig: leaderConfig);
await resourceManager.StartAsync();
}
public void LeaderElection()
{
var electionHistory = new List <string>();
var leadershipHistory = new List <string>();
var renewCountA = 3;
var mockLockA = new MockResourceLock("mockA")
{
UpdateWillFail = () => renewCountA <= 0
};
mockLockA.OnCreate += (_) =>
{
renewCountA--;
electionHistory.Add("A creates record");
leadershipHistory.Add("A gets leadership");
};
mockLockA.OnUpdate += (_) =>
{
renewCountA--;
electionHistory.Add("A updates record");
};
mockLockA.OnChange += (_) => { leadershipHistory.Add("A gets leadership"); };
var leaderElectionConfigA = new LeaderElectionConfig(mockLockA)
{
LeaseDuration = TimeSpan.FromMilliseconds(500),
RetryPeriod = TimeSpan.FromMilliseconds(300),
RenewDeadline = TimeSpan.FromMilliseconds(400),
};
var renewCountB = 4;
var mockLockB = new MockResourceLock("mockB")
{
UpdateWillFail = () => renewCountB <= 0
};
mockLockB.OnCreate += (_) =>
{
renewCountB--;
electionHistory.Add("B creates record");
leadershipHistory.Add("B gets leadership");
};
mockLockB.OnUpdate += (_) =>
{
renewCountB--;
electionHistory.Add("B updates record");
};
mockLockB.OnChange += (_) => { leadershipHistory.Add("B gets leadership"); };
var leaderElectionConfigB = new LeaderElectionConfig(mockLockB)
{
LeaseDuration = TimeSpan.FromMilliseconds(500),
RetryPeriod = TimeSpan.FromMilliseconds(300),
RenewDeadline = TimeSpan.FromMilliseconds(400),
};
var lockAStopLeading = new ManualResetEvent(false);
var testLeaderElectionLatch = new CountdownEvent(4);
Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfigA);
leaderElector.OnStartedLeading += () =>
{
leadershipHistory.Add("A starts leading");
testLeaderElectionLatch.Signal();
};
leaderElector.OnStoppedLeading += () =>
{
leadershipHistory.Add("A stops leading");
testLeaderElectionLatch.Signal();
lockAStopLeading.Set();
};
leaderElector.RunAsync().Wait();
});
lockAStopLeading.WaitOne(TimeSpan.FromSeconds(3));
Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfigB);
leaderElector.OnStartedLeading += () =>
{
leadershipHistory.Add("B starts leading");
testLeaderElectionLatch.Signal();
};
leaderElector.OnStoppedLeading += () =>
{
leadershipHistory.Add("B stops leading");
testLeaderElectionLatch.Signal();
};
leaderElector.RunAsync().Wait();
});
testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(10));
Assert.Equal(7, electionHistory.Count);
Assert.True(electionHistory.SequenceEqual(
new[]
{
"A creates record", "A updates record", "A updates record",
"B updates record", "B updates record", "B updates record", "B updates record",
}));
Assert.True(leadershipHistory.SequenceEqual(
new[]
{
"A gets leadership", "A starts leading", "A stops leading",
"B gets leadership", "B starts leading", "B stops leading",
}));
}
public void LeaderElectionWithRenewDeadline()
{
var electionHistory = new List <string>();
var leadershipHistory = new List <string>();
var renewCount = 3;
var mockLock = new MockResourceLock("mock")
{
UpdateWillFail = () => renewCount <= 0,
};
mockLock.OnCreate += _ =>
{
renewCount--;
electionHistory.Add("create record");
leadershipHistory.Add("get leadership");
};
mockLock.OnUpdate += _ =>
{
renewCount--;
electionHistory.Add("update record");
};
mockLock.OnChange += _ => { electionHistory.Add("change record"); };
mockLock.OnTryUpdate += _ => { electionHistory.Add("try update record"); };
var leaderElectionConfig = new LeaderElectionConfig(mockLock)
{
LeaseDuration = TimeSpan.FromMilliseconds(1000),
RetryPeriod = TimeSpan.FromMilliseconds(200),
RenewDeadline = TimeSpan.FromMilliseconds(650),
};
var countdown = new CountdownEvent(2);
Task.Run(() =>
{
var leaderElector = new LeaderElector(leaderElectionConfig);
leaderElector.OnStartedLeading += () =>
{
leadershipHistory.Add("start leading");
countdown.Signal();
};
leaderElector.OnStoppedLeading += () =>
{
leadershipHistory.Add("stop leading");
countdown.Signal();
};
leaderElector.RunAsync().Wait();
});
countdown.Wait(TimeSpan.FromSeconds(10));
// TODO flasky
// Assert.Equal(9, electionHistory.Count);
// Assert.True(electionHistory.SequenceEqual(new[]
// {
// "create record", "try update record", "update record", "try update record", "update record",
// "try update record", "try update record", "try update record", "try update record",
// }));
Assert.True(electionHistory.Take(7).SequenceEqual(new[]
{
"create record", "try update record", "update record", "try update record", "update record",
"try update record", "try update record",
}));
Assert.True(leadershipHistory.SequenceEqual(new[] { "get leadership", "start leading", "stop leading" }));
}
public KElector(LeaderElectionConfig config)
{
Logger = KLog.GetLogger <KElector>();
this.config = config;
}
/// <summary>
/// Starts the controller.
/// </summary>
/// <param name="k8s">The <see cref="IKubernetes"/> client to use.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
public static async Task StartAsync(IKubernetes k8s)
{
Covenant.Requires <ArgumentNullException>(k8s != null, nameof(k8s));
if (NeonHelper.IsLinux)
{
// Ensure that the [/var/run/neonkube/node-tasks] folder exists on the node.
var scriptPath = Path.Combine(Node.HostMount, $"tmp/node-agent-folder-{NeonHelper.CreateBase36Uuid()}.sh");
var script =
$@"#!/bin/bash
set -euo pipefail
# Ensure that the nodetask runtime folders exist and have the correct permissions.
if [ ! -d {hostNeonRunFolder} ]; then
mkdir -p {hostNeonRunFolder}
chmod 700 {hostNeonRunFolder}
fi
if [ ! -d {hostNeonTasksFolder} ]; then
mkdir -p {hostNeonTasksFolder}
chmod 700 {hostNeonTasksFolder}
fi
# Remove this script.
rm $0
";
File.WriteAllText(scriptPath, NeonHelper.ToLinuxLineEndings(script));
try
{
(await Node.BashExecuteCaptureAsync(scriptPath)).EnsureSuccess();
}
finally
{
NeonHelper.DeleteFile(scriptPath);
}
}
// Load the configuration settings.
var leaderConfig =
new LeaderElectionConfig(
k8s,
@namespace: KubeNamespace.NeonSystem,
leaseName: $"{Program.Service.Name}.nodetask-{Node.Name}",
identity: Pod.Name,
promotionCounter: Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_promoted", "Leader promotions"),
demotionCounter: Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_demoted", "Leader demotions"),
newLeaderCounter: Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_newLeader", "Leadership changes"));
var options = new ResourceManagerOptions()
{
IdleInterval = Program.Service.Environment.Get("NODETASK_IDLE_INTERVAL", TimeSpan.FromSeconds(60)),
ErrorMinRequeueInterval = Program.Service.Environment.Get("NODETASK_ERROR_MIN_REQUEUE_INTERVAL", TimeSpan.FromSeconds(15)),
ErrorMaxRetryInterval = Program.Service.Environment.Get("NODETASK_ERROR_MAX_REQUEUE_INTERVAL", TimeSpan.FromSeconds(60)),
IdleCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle", "IDLE events processed."),
ReconcileCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_reconcile", "RECONCILE events processed."),
DeleteCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_delete", "DELETED events processed."),
IdleErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_idle_error", "Failed NodeTask IDLE event processing."),
ReconcileErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_reconcile_error", "Failed NodeTask RECONCILE event processing."),
DeleteErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_delete_error", "Failed NodeTask DELETE event processing."),
StatusModifyErrorCounter = Metrics.CreateCounter($"{Program.Service.MetricsPrefix}nodetask_statusmodify_error", "Failed NodeTask STATUS-MODIFY events processing.")
};
resourceManager = new ResourceManager <V1NeonNodeTask, NodeTaskController>(
k8s,
options: options,
filter: NodeTaskFilter,
leaderConfig: leaderConfig);
await resourceManager.StartAsync();
}
