public IRunspaceInfo StartCreate()
{
_logger.LogInformation("Create Runspace");
K8sRunspaceInfo result = null;
try {
_logger.LogDebug("GenerateRunspaceId");
var runspaceId = GenerateRunspaceId();
_logger.LogDebug($"RunspaceId: {runspaceId}");
var runspacePod = CreateK8sPod(runspaceId);
result = new K8sRunspaceInfo {
Id = runspaceId,
CreationState = RunspaceCreationState.Creating
};
_logger.LogDebug($"RunspaceInfo.Id: {result.Id}");
} catch (Exception exc) {
_logger.LogError(exc.ToString());
var error = new RunspaceProviderException(
Resources.K8sRunspaceProvider_Create_K8sRunspaceCreateFail,
exc);
result = new K8sRunspaceInfo {
Id = result?.Id,
CreationState = RunspaceCreationState.Error,
CreationError = error
};
}
return(result);
}
public IRunspaceInfo[] List()
{
_logger.LogInformation("List Runspaces");
List <IRunspaceInfo> result = new List <IRunspaceInfo>();
try {
var runspacePods = _client.ListNamespacedPod(
_namespace,
labelSelector: $"{LABEL_KEY}={RUNSPACE_TYPE}");
foreach (var runspacePod in runspacePods.Items)
{
if (runspacePod?.Status?.Phase != "Running" &&
runspacePod?.Status?.Phase != "Pending")
{
continue;
}
var runspaceInfo = new K8sRunspaceInfo {
Id = runspacePod.Metadata.Name
};
if (runspacePod?.Status?.Phase == "Pending")
{
runspaceInfo.CreationState = RunspaceCreationState.Creating;
}
else
{
runspaceInfo.Endpoint = new IPEndPoint(
IPAddress.Parse(runspacePod.Status.PodIP),
_runspaceApiPort);
runspaceInfo.CreationState = RunspaceCreationState.Ready;
}
result.Add(runspaceInfo);
}
} catch (Exception exc) {
_logger.LogError(exc.ToString());
throw new RunspaceProviderException(
Resources.K8sRunspaceProvider_List_K8sRunspaceListFail,
exc);
}
return(result.ToArray());
}
public IRunspaceInfo WaitCreateCompletion(IRunspaceInfo runspaceInfo)
{
var result = runspaceInfo;
if (result != null && result.CreationState == RunspaceCreationState.Creating)
{
V1Pod pod = null;
try {
_logger.LogDebug($"Waiting k8s Pod '{runspaceInfo.Id}' to become ready");
_logger.LogDebug($"K8s API Call ReadNamespacedPod: {runspaceInfo.Id}");
pod = _client.ReadNamespacedPod(runspaceInfo.Id, _namespace);
} catch (Exception exc) {
result = new K8sRunspaceInfo {
Id = result.Id,
CreationState = RunspaceCreationState.Error,
CreationError = new RunspaceProviderException(
string.Format(
Resources.K8sRunspaceProvider_WaitCreateComplation_PodNotFound, result.Id),
exc)
};
}
if (pod != null)
{
// Set 10 minutes timeout for container creation.
// Worst case would be image pulling from server.
int maxRetryCount = 6000;
int retryIntervalMs = 100;
int retryCount = 1;
// Wait Pod to become running and obtain IP Address
_logger.LogDebug($"Start wating K8s Pod to become running: {pod.Metadata.Name}");
// There are three possible phases of a POD
// Pending - awaiting containers to start
// Running - Pod is initialized and all containers in the Pod are running or completed successfully
// Terminating - Pod is terminating
// The Pending phase could last forever when container image pull error occurred or some other error
// in the container initialization happens. In order to stop waiting below we first monitor for Pod status to
// phase to switch from pending to running. While Pod is pending phase we monitor the container
// creation for errors and if such occur we break the waiting with error.
//
// The Container creation errors that are related to image pulling failura are stored in the
// pod.Status.ContainerStatuses[0].State.Waiting propery is not null which is instance of V1ContainerStateWaiting
// The errors are returned as strings in Reason property of the V1ContainerStateWaiting
// The strings that represent errors are:
//
// ImagePullBackOff - Container image pull failed, kubelet is backing off image pull
// ImageInspectError - Unable to inspect image
// ErrImagePull - General image pull error
// ErrImageNeverPull - Required Image is absent on host and PullPolicy is NeverPullImage
// RegistryUnavailable - Get http error when pulling image from registry
// InvalidImageName - Unable to parse the image name.
while (
pod != null &&
string.IsNullOrEmpty(pod.Status?.PodIP) &&
(pod.Status?.Phase != "Running" ||
(pod.Status?.Phase == "Pending" &&
!HasErrrorInContainerStatus(pod.Status, out var _))) &&
retryCount < maxRetryCount)
{
Thread.Sleep(retryIntervalMs);
_logger.LogDebug($"K8s API Call ReadNamespacedPod: {pod.Metadata.Name}");
pod = _client.ReadNamespacedPod(pod.Metadata.Name, _namespace);
retryCount++;
}
if (retryCount >= maxRetryCount)
{
// Timeout
result = new K8sRunspaceInfo {
Id = result.Id,
CreationState = RunspaceCreationState.Error,
CreationError = new RunspaceProviderException(Resources.K8sRunspaceProvider_WaitCreateComplition_TimeOut)
};
}
else
if (HasErrrorInContainerStatus(pod.Status, out var errorMessage))
{
// Container Creation Error
result = new K8sRunspaceInfo {
Id = result.Id,
CreationState = RunspaceCreationState.Error,
CreationError = new RunspaceProviderException(errorMessage)
};
}
else
{
// Success, everything should be in place
result = new K8sRunspaceInfo {
Id = result.Id,
Endpoint =
new IPEndPoint(
IPAddress.Parse(pod.Status.PodIP),
_runspaceApiPort),
CreationState = RunspaceCreationState.Ready
};
}
if (result.CreationState == RunspaceCreationState.Ready && _verifyRunspaceApiIsAccessibleOnCreate)
{
try {
_logger.LogDebug($"EnsureRunspaceEndpointIsAccessible: Start");
// Ensure Container is accessible over the network after creation
EnsureRunspaceEndpointIsAccessible(result);
_logger.LogDebug($"EnsureRunspaceEndpointIsAccessible: Success");
} catch (RunspaceProviderException exc) {
_logger.LogError(exc.ToString());
// Kill the container that is not accessible, otherwise it will leak
try {
Kill(result.Id);
} catch (RunspaceProviderException rexc) {
_logger.LogError(rexc.ToString());
}
result = new K8sRunspaceInfo {
Id = result.Id,
CreationState = RunspaceCreationState.Error,
CreationError = exc
};
}
}
}
}
return(result);
}