public async Task NeonTask_Timeout()
{
//-----------------------------------------------------------------
// Verify that task timeouts are honored.
await DeleteExistingTasksAsync();
var taskName = $"test-timeout-{CreateUuidString()}";
var nodeTask = new V1NeonNodeTask();
var metadata = nodeTask.Metadata;
var spec = nodeTask.Spec;
spec.Node = fixture.Cluster.FirstControlNode.Name;
spec.TimeoutSeconds = 15;
spec.RetentionSeconds = 30;
spec.BashScript =
@"
sleep 30
";
metadata.SetLabel(NeonLabel.RemoveOnClusterReset);
await fixture.K8s.CreateClusterCustomObjectAsync <V1NeonNodeTask>(nodeTask, name : taskName);
//-----------------------------------------------------------------
// Wait the node task to report completion.
var phase = V1NeonNodeTask.Phase.New;
var exitCode = int.MaxValue;
await NeonHelper.WaitForAsync(
async() =>
{
var task = await fixture.K8s.ReadClusterCustomObjectAsync <V1NeonNodeTask>(taskName);
switch (task.Status.Phase)
{
case V1NeonNodeTask.Phase.New:
case V1NeonNodeTask.Phase.Pending:
case V1NeonNodeTask.Phase.Running:
return(false);
default:
phase = task.Status.Phase;
exitCode = task.Status.ExitCode;
return(true);
}
},
timeout : timeout,
pollInterval : pollInterval);
//-----------------------------------------------------------------
// Verify that the node task timed out.
Assert.Equal(V1NeonNodeTask.Phase.Timeout, phase);
Assert.Equal(-1, exitCode);
}
public async Task NeonTask_StartAfter()
{
//-----------------------------------------------------------------
// Verify that a task scheduled in the future with a StartAfterTimestamp
// is actually executed in the future.
await DeleteExistingTasksAsync();
var taskName = $"test-scheduled-{CreateUuidString()}";
var nodeTask = new V1NeonNodeTask();
var metadata = nodeTask.Metadata;
var spec = nodeTask.Spec;
var scheduledUtc = DateTime.UtcNow + TimeSpan.FromSeconds(90);
spec.Node = fixture.Cluster.FirstControlNode.Name;
spec.StartAfterTimestamp = scheduledUtc;
spec.TimeoutSeconds = 15;
spec.RetentionSeconds = 30;
spec.BashScript =
@"
sleep 5
";
metadata.SetLabel(NeonLabel.RemoveOnClusterReset);
await fixture.K8s.CreateClusterCustomObjectAsync <V1NeonNodeTask>(nodeTask, name : taskName);
//-----------------------------------------------------------------
// Wait the node task to reported as SUCCESS.
var actualUtc = DateTime.MinValue;
await NeonHelper.WaitForAsync(
async() =>
{
var task = await fixture.K8s.ReadClusterCustomObjectAsync <V1NeonNodeTask>(taskName);
if (task.Status.Phase == V1NeonNodeTask.Phase.Success)
{
actualUtc = task.Status.StartTimestamp.Value;
return(true);
}
else
{
return(false);
}
},
timeout : timeout,
pollInterval : pollInterval);
//-----------------------------------------------------------------
// Verify that the task actually started at or after the scheduled time.
Assert.True(actualUtc >= scheduledUtc);
}
public async Task NeonTask_MissingNode()
{
//-----------------------------------------------------------------
// Verify that the [V1NodeTask] controller in [neon-cluster-operator] deletes
// tasks assigned to nodes that don't exist.
await DeleteExistingTasksAsync();
var taskName = $"test-badnode-{CreateUuidString()}";
var nodeTask = new V1NeonNodeTask();
var metadata = nodeTask.Metadata;
var spec = nodeTask.Spec;
var scheduledUtc = DateTime.UtcNow + TimeSpan.FromHours(1);
spec.Node = $"missing-node-{CreateUuidString()}";
spec.StartAfterTimestamp = scheduledUtc;
spec.TimeoutSeconds = 15;
spec.RetentionSeconds = 30;
spec.BashScript =
@"
sleep 5
";
metadata.SetLabel(NeonLabel.RemoveOnClusterReset);
await fixture.K8s.CreateClusterCustomObjectAsync <V1NeonNodeTask>(nodeTask, name : taskName);
//-----------------------------------------------------------------
// Wait the node task to be deleted.
await NeonHelper.WaitForAsync(
async() =>
{
try
{
await fixture.K8s.ReadClusterCustomObjectAsync <V1NeonNodeTask>(taskName);
return(false);
}
catch (HttpOperationException e)
{
return(e.Response.StatusCode == HttpStatusCode.NotFound);
}
},
timeout : timeout,
pollInterval : pollInterval);
}
public async Task NeonTask_StartBefore()
{
//-----------------------------------------------------------------
// Verify that a task scheduled with a StartBeforeTimestamp that is
// already too late is detected and its status is set to TARDY.
await DeleteExistingTasksAsync();
var taskName = $"test-tardy-{CreateUuidString()}";
var nodeTask = new V1NeonNodeTask();
var metadata = nodeTask.Metadata;
var spec = nodeTask.Spec;
spec.Node = fixture.Cluster.FirstControlNode.Name;
spec.StartBeforeTimestamp = DateTime.UtcNow - TimeSpan.FromHours(1);
spec.TimeoutSeconds = 15;
spec.RetentionSeconds = 30;
spec.BashScript =
@"
sleep 5
";
metadata.SetLabel(NeonLabel.RemoveOnClusterReset);
await fixture.K8s.CreateClusterCustomObjectAsync <V1NeonNodeTask>(nodeTask, name : taskName);
//-----------------------------------------------------------------
// Wait the node task to reported as TARDY.
await NeonHelper.WaitForAsync(
async() =>
{
var task = await fixture.K8s.ReadClusterCustomObjectAsync <V1NeonNodeTask>(taskName);
return(task.Status.Phase == V1NeonNodeTask.Phase.Tardy);
},
timeout : timeout,
pollInterval : pollInterval);
}
public async Task NodeTask_ExitCodeAndStreams()
{
//-----------------------------------------------------------------
// Submit a task to the first control-plane node that returns a non-zero
// exit code as well as writes to the standard output and error
// streams.
//
// Then we'll verify that the task [Phase==Failed] and confirm that
// the exitcode and streams are present in the task status.
await DeleteExistingTasksAsync();
var taskName = $"test-exitcode-{CreateUuidString()}";
var nodeTask = new V1NeonNodeTask();
var metadata = nodeTask.Metadata;
var spec = nodeTask.Spec;
spec.Node = fixture.Cluster.FirstControlNode.Name;
spec.RetentionSeconds = 30;
spec.BashScript =
@"
echo 'HELLO WORLD!' >&1
echo 'GOODBYE WORLD!' >&2
exit 123
";
metadata.SetLabel(NeonLabel.RemoveOnClusterReset);
await fixture.K8s.CreateClusterCustomObjectAsync <V1NeonNodeTask>(nodeTask, name : taskName);
//-----------------------------------------------------------------
// Wait the node task to report completion.
await NeonHelper.WaitForAsync(
async() =>
{
var task = await fixture.K8s.ReadClusterCustomObjectAsync <V1NeonNodeTask>(taskName);
switch (task.Status.Phase)
{
case V1NeonNodeTask.Phase.New:
case V1NeonNodeTask.Phase.Pending:
case V1NeonNodeTask.Phase.Running:
return(false);
default:
return(true);
}
},
timeout : timeout,
pollInterval : pollInterval);
//-----------------------------------------------------------------
// Verify that the node task failed (due to the non-zero exit code)
// and that the exit code as well as the output/error streams were
// captured.
var task = await fixture.K8s.ReadClusterCustomObjectAsync <V1NeonNodeTask>(taskName);
Assert.Equal(V1NeonNodeTask.Phase.Failed, task.Status.Phase);
Assert.Equal(123, task.Status.ExitCode);
Assert.StartsWith("HELLO WORLD!", task.Status.Output);
Assert.StartsWith("GOODBYE WORLD!", task.Status.Error);
}
public async Task NodeTask_Basic()
{
try
{
//-----------------------------------------------------------------
// We're going to schedule simple node tasks for all cluster nodes that
// touch a temporary file and then verify that the file was written
// to the nodes and that the node task status indicates the the operation
// succeeded.
await DeleteExistingTasksAsync();
// Create a string dictionary that maps cluster node names to the unique
// name to use for the test tasks targeting each node.
var nodeToTaskName = new Dictionary <string, string>();
foreach (var node in fixture.Cluster.Nodes)
{
nodeToTaskName.Add(node.Name, $"test-basic-{node.Name}-{CreateUuidString()}");
}
// Initalize a test folder on each node where the task will update a file
// indicating that it ran and then submit a task for each node.
foreach (var node in fixture.Cluster.Nodes)
{
// Clear and recreate the node test folder.
node.Connect();
node.SudoCommand($"rm -rf {testFolderPath}");
node.SudoCommand($"mkdir -p {testFolderPath}");
// Create the node task for the target node.
var nodeTask = new V1NeonNodeTask();
var metadata = nodeTask.Metadata;
var spec = nodeTask.Spec;
metadata.SetLabel(NeonLabel.RemoveOnClusterReset);
var filePath = GetTestFilePath(node.Name);
var folderPath = LinuxPath.GetDirectoryName(filePath);
spec.Node = node.Name;
spec.RetentionSeconds = 30;
spec.BashScript =
$@"
set -euo pipefail
mkdir -p $NODE_ROOT{folderPath}
touch $NODE_ROOT{filePath}
";
await fixture.K8s.CreateClusterCustomObjectAsync <V1NeonNodeTask>(nodeTask, name : nodeToTaskName[node.Name]);
}
// Wait for all of the node tasks to report completion.
var taskNames = new HashSet <string>();
foreach (var taskName in nodeToTaskName.Values)
{
taskNames.Add(taskName);
}
await NeonHelper.WaitForAsync(
async() =>
{
foreach (var task in (await fixture.K8s.ListClusterCustomObjectAsync <V1NeonNodeTask>()).Items.Where(task => taskNames.Contains(task.Metadata.Name)))
{
switch (task.Status.Phase)
{
case V1NeonNodeTask.Phase.New:
case V1NeonNodeTask.Phase.Pending:
case V1NeonNodeTask.Phase.Running:
return(false);
}
}
return(true);
},
timeout : timeout,
pollInterval : pollInterval);
//-----------------------------------------------------------------
// Verify that the node tasks completeted successfully and are being
// retained for a while.
var nodeTasks = await fixture.K8s.ListClusterCustomObjectAsync <V1NeonNodeTask>();
foreach (var task in nodeTasks.Items)
{
if (taskNames.Contains(task.Metadata.Name))
{
Assert.Equal(V1NeonNodeTask.Phase.Success, task.Status.Phase);
Assert.Equal(0, task.Status.ExitCode);
Assert.Equal(string.Empty, task.Status.Output);
Assert.Equal(string.Empty, task.Status.Error);
}
}
//-----------------------------------------------------------------
// Connect to each of the nodes and verify that the files touched by
// the scripts actually exist.
foreach (var node in fixture.Cluster.Nodes)
{
var filePath = GetTestFilePath(node.Name);
// Clear and recreate the node test folder.
node.Connect();
Assert.True(node.FileExists(filePath));
}
}
finally
{
// Remove the test folders on the nodes.
foreach (var node in fixture.Cluster.Nodes)
{
var filePath = GetTestFilePath(node.Name);
// Clear and recreate the node test folder.
node.Connect();
node.SudoCommand($"rm -rf {testFolderPath}");
}
}
}
