public IActionResult Post([FromQuery] string name, string namespaceName)
{
var job = new V1Job
{
ApiVersion = "batch/v1",
Kind = "Job",
Metadata = new V1ObjectMeta
{
Name = name
},
Spec = new V1JobSpec
{
Template = new V1PodTemplateSpec()
{
Spec = new V1PodSpec()
{
Containers = new List <V1Container>()
{
new V1Container()
{
Name = "container-test",
Image = "hello-world"
}
},
RestartPolicy = "Never"
}
}
}
};
var result = kubeClient.CreateNamespacedJob(job, namespaceName);
return(Ok());
}
public ActionResult <String> Post()
{
// get the namespace the pod is running in
// this is set with a FieldRef as part of the deployment
var pod_namespace = Environment.GetEnvironmentVariable("MY_POD_NAMESPACE");
var task_image = Environment.GetEnvironmentVariable("RUNNER_IMAGE");
// get the kube client
var kube = GetKubernetes();
// create the job definition
var job = new V1Job
{
Metadata = new V1ObjectMeta {
Name = "test-" + Guid.NewGuid()
},
Spec = new V1JobSpec {
Template = new V1PodTemplateSpec {
Metadata = new V1ObjectMeta {
Name = "runner",
Labels = new Dictionary <string, string> {
{ "task", "test-task" }
}
},
Spec = new V1PodSpec {
Containers = new List <V1Container> {
new V1Container {
Name = "runner",
Image = task_image,
}
},
RestartPolicy = "Never"
}
}
}
};
Console.WriteLine(job);
// schedule the job
var result = kube.CreateNamespacedJob(job, pod_namespace);
return(job.Metadata.Name);
}
public V1JobStatus CreateK8sJob(dynamic messageBody)
{
var job = new V1Job();
var container = new V1Container();
container.Image = kubernetesJob.ImageName;
container.Name = kubernetesJob.KubernetesNamespace;
container.Command = new List <string>();
// container entry point
container.Command.Add("dotnet");
container.Command.Add(kubernetesJob.EntryPoint);
// job arguments
container.Command.Add(messageBody);
job.Kind = "Job";
job.ApiVersion = "batch/v1";
job.Metadata = new V1ObjectMeta();
job.Metadata.Name = $"{kubernetesJob.JobName.ToLower()}-{DateTime.Now.Ticks.ToString()}";
job.Metadata.NamespaceProperty = kubernetesJob.KubernetesNamespace;
job.Spec = new V1JobSpec();
job.Spec.Template = new V1PodTemplateSpec();
job.Spec.Template.Spec = new V1PodSpec();
job.Spec.Template.Spec.Containers = new List <V1Container>();
job.Spec.Template.Spec.Containers.Add(container);
job.Spec.Template.Spec.RestartPolicy = "Never";
job.Validate();
var config = KubernetesClientConfiguration.BuildDefaultConfig();
Console.WriteLine(config);
IKubernetes client = new Kubernetes(config);
var createdJob = client.CreateNamespacedJob(job, kubernetesJob.KubernetesNamespace);
return(createdJob.Status);
}
public V1Job GetEUConverterJob(int id)
{
V1Job job = new V1Job()
{
ApiVersion = "batch/v1",
Kind = V1Job.KubeKind,
Metadata = new V1ObjectMeta()
{
Name = $"eu-converter-sliceid-{id}"
},
Spec = new V1JobSpec()
{
TtlSecondsAfterFinished = 0,
BackoffLimit = 3,
Template = new V1PodTemplateSpec()
{
Spec = new V1PodSpec()
{
Containers = new List <V1Container>()
{
new V1Container()
{
Image = _appConfigService.Image,
Name = $"eu-converter-sliceid-{id}",
//Command = new List<string>() { "/bin/bash", "-c", "--" },
Env = new List <V1EnvVar>()
{
new V1EnvVar("SliceId", id.ToString()),
new V1EnvVar("RetryCount", _appConfigService.RetryCount.ToString()),
new V1EnvVar("SleepDuration", _appConfigService.DevAttributeContainerLifeDuration.ToString())
},
VolumeMounts = new List <V1VolumeMount>()
{
new V1VolumeMount(
mountPath: "/app/appsettings.json",
name: "config-volume",
subPath: "appsettings.json")
},
ImagePullPolicy = "Always",
},
},
Volumes = new List <V1Volume>()
{
new V1Volume()
{
Name = "config-volume",
ConfigMap = new V1ConfigMapVolumeSource()
{
Name = "app-config"
}
}
},
RestartPolicy = "OnFailure",
},
},
}
};
_logger.LogInformation($"Kubernetes job eu-converter-sliceid-{id} started.");
return(job);
}
/// <summary>
/// Generate Kubernetes Job manifest
/// </summary>
/// <remarks>
/// manifest would be equivalant to YAML manifest.
/// ---
/// apiVersion: batch/v1
/// kind: Job
/// metadata:
/// name: {name}
/// labels:
/// app: {name}
/// spec:
/// parallelism: {parallelism}
/// completions: {parallelism}
/// backoffLimit: 0
/// template:
/// metadata:
/// labels:
/// app: {name}
/// spec:
/// restartPolicy: Never
/// containers:
/// - name: {name}
/// image: {image}:{imageTag}
/// imagePullPolicy: {imagePullPolicy}
/// args: [""--worker-flag""]
/// env:
/// - name: DFRAME_MASTER_CONNECT_TO_HOST
/// value: ""{host}""
/// - name: DFRAME_MASTER_CONNECT_TO_PORT
/// value: ""{port}""
/// resources:
/// requests:
/// cpu: 100m
/// memory: 100Mi
/// limits:
/// cpu: 2000m
/// memory: 1000Mi
/// imagePullSecrets:
/// - name: {imagePullSecret}
/// ---
/// </remarks>
/// <param name="name"></param>
/// <param name="image"></param>
/// <param name="imageTag"></param>
/// <param name="host"></param>
/// <param name="port"></param>
/// <param name="imagePullPolicy"></param>
/// <param name="imagePullSecret"></param>
/// <param name="parallelism"></param>
/// <returns></returns>
public V1Job CreateJobDefinition(string name, string image, string imageTag, string host, int port, string imagePullPolicy = "IfNotPresent", string imagePullSecret = "", int parallelism = 1)
{
var labels = new Dictionary <string, string>
{
{ "app", name },
};
var definition = new V1Job
{
ApiVersion = "batch/v1",
Kind = "Job",
Metadata = new V1ObjectMeta
{
Name = name,
Labels = labels
},
Spec = new V1JobSpec
{
Parallelism = parallelism,
Completions = parallelism,
// note: must be 0 to prevent pod restart during load testing.
BackoffLimit = 0,
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = labels,
},
Spec = new V1PodSpec
{
// note: must be Never to prevent pod restart during load testing.
RestartPolicy = "Never",
Containers = new[] {
new V1Container
{
Name = name,
Image = $"{image}:{imageTag}",
// "IfNotPresent" to reuse existing, "Never" to always use latest image for same tag.
ImagePullPolicy = imagePullPolicy,
Args = new [] { "--worker-flag" },
Env = new []
{
new V1EnvVar
{
Name = "DFRAME_MASTER_CONNECT_TO_HOST",
Value = host,
},
new V1EnvVar
{
Name = "DFRAME_MASTER_CONNECT_TO_PORT",
Value = port.ToString(),
}
},
Resources = new V1ResourceRequirements
{
// todo: should be configuable
Limits = new Dictionary <string, ResourceQuantity>
{
{ "cpu", new ResourceQuantity {
Value = "2000m"
} },
{ "memory", new ResourceQuantity {
Value = "1000Mi"
} },
},
Requests = new Dictionary <string, ResourceQuantity>
{
{ "cpu", new ResourceQuantity {
Value = "100m"
} },
{ "memory", new ResourceQuantity {
Value = "100Mi"
} },
}
},
},
},
},
}
}
};
if (!string.IsNullOrEmpty(imagePullSecret))
{
definition.Spec.Template.Spec.ImagePullSecrets = new[]
{
new V1LocalObjectReference
{
Name = imagePullSecret,
},
};
}
return(definition);
}
public async ValueTask <V1Job> CreateJobAsync(string ns, V1Job job, CancellationToken ct = default)
{
using var res = await CreateJobHttpAsync(ns, JsonConvert.Serialize(job), ct).ConfigureAwait(false);
return(res.Body);
}
async Task StartJobFor(IPerformerLog log, Context context, Read.Configuration.Build build)
{
var @namespace = "dolittle";
var metadata = new V1ObjectMeta
{
Name = Guid.NewGuid().ToString() //,
//Labels = { { "type", "build" } }
};
log.Information($"---");
log.Information($"Type : {build.Type}");
log.Information($"BasePath : {build.BasePath}");
log.Information($"Package : {build.Package}");
log.Information($"Publish : {build.Publish}");
log.Information($"Folder with project to publish : {build.FolderWithProjectToPublish}");
log.Information($"---");
var job = new V1Job
{
Metadata = metadata,
Spec = new V1JobSpec
{
Completions = 1,
Template = new V1PodTemplateSpec
{
Metadata = metadata,
Spec = new V1PodSpec
{
Containers = new [] {
new V1Container {
Name = "build",
Image = $"dolittlebuild/{build.Type}",
ImagePullPolicy = "Always",
Env = new [] {
new V1EnvVar("REPOSITORY", context.Project.Repository.ToString()),
new V1EnvVar("COMMIT", context.SourceControl.Commit),
new V1EnvVar("PULL_REQUEST", context.IsPullRequest.ToString()),
new V1EnvVar("VERSION", context.Version),
new V1EnvVar("BASE_PATH", build.BasePath),
new V1EnvVar("PACKAGE", build.Package.ToString()),
new V1EnvVar("PUBLISH", build.Publish.ToString()),
new V1EnvVar("FOLDER_WITH_PROJECT_TO_PUBLISH", build.FolderWithProjectToPublish),
new V1EnvVar("CALLBACK", $"http://continuousimprovement/buildJobDone?jobName={metadata.Name}")
},
VolumeMounts = new[] {
new V1VolumeMount {
Name = "azure",
MountPath = "/repository",
SubPath = context.Volumes.SourcePath
},
new V1VolumeMount {
Name = "azure",
MountPath = "/packages",
SubPath = context.Volumes.PackagePath
},
new V1VolumeMount {
Name = "azure",
MountPath = "/output",
SubPath = context.Volumes.OutputPath
},
new V1VolumeMount {
Name = "azure",
MountPath = "/publish",
SubPath = context.Volumes.PublishPath
},
new V1VolumeMount {
Name = "azure",
MountPath = "/testresults",
SubPath = context.Volumes.TestResultsPath
}
}
}
},
Volumes = new[] {
new V1Volume {
Name = "azure",
AzureFile = new V1AzureFileVolumeSource {
SecretName = "azure-storage-secret",
ShareName = "continuousimprovement",
ReadOnlyProperty = false
}
}
},
RestartPolicy = "Never",
}
}
}
};
await _kubernetes.CreateNamespacedJobAsync(job, @namespace);
}
public async Task <V1Job> CreateJobAsync(string jobName,
int parallelism,
int completions,
string containerName,
string containerImage,
string imagePullSecret,
string label,
IDictionary <string, string> environmentVariables,
string _namespace = "default",
string cpuRequest = "250m",
string memRequest = "50Mi",
string cpuLimit = "500m",
string memLimit = "100Mi")
{
if (string.IsNullOrEmpty(jobName))
{
throw new ArgumentNullException(nameof(jobName));
}
if (string.IsNullOrEmpty(containerName))
{
throw new ArgumentNullException(nameof(containerName));
}
if (string.IsNullOrEmpty(containerImage))
{
throw new ArgumentNullException(nameof(containerImage));
}
if (string.IsNullOrEmpty(imagePullSecret))
{
throw new ArgumentNullException(nameof(imagePullSecret));
}
if (parallelism < 1)
{
throw new ArgumentOutOfRangeException(nameof(parallelism));
}
if (completions < 1)
{
throw new ArgumentOutOfRangeException(nameof(completions));
}
if (environmentVariables == null || environmentVariables.Count == 0)
{
throw new ApplicationException("No Environment variable provided!");
}
var labels = new Dictionary <string, string>();
labels.Add("nanny", label);
var job = new V1Job();
job.Metadata = new V1ObjectMeta()
{
Name = jobName
};
var podLimits = new Dictionary <string, ResourceQuantity>();
podLimits.Add("cpu", new ResourceQuantity(cpuLimit));
podLimits.Add("memory", new ResourceQuantity(memLimit));
var podRequest = new Dictionary <string, ResourceQuantity>();
podRequest.Add("cpu", new ResourceQuantity(cpuRequest));
podRequest.Add("memory", new ResourceQuantity(memRequest));
List <V1EnvVar> envVars = environmentVariables.Select(e => new V1EnvVar(e.Key, e.Value)).ToList <V1EnvVar>();
job.Spec = new V1JobSpec()
{
Parallelism = parallelism,
Completions = completions,
Template = new V1PodTemplateSpec()
{
Metadata = new V1ObjectMeta()
{
Labels = labels
},
Spec = new V1PodSpec()
{
Containers = new List <V1Container>()
{
new V1Container()
{
Name = containerName,
Image = containerImage,
ImagePullPolicy = "Always",
Resources = new V1ResourceRequirements()
{
Limits = podLimits,
Requests = podRequest
},
Env = envVars
}
},
RestartPolicy = "Never",
ImagePullSecrets = new List <V1LocalObjectReference>()
{
new V1LocalObjectReference()
{
Name = imagePullSecret
}
}
}
}
};
return(await _k8client.CreateNamespacedJobAsync(job, _namespace));
}
public async Task <V1Job> CreateJobAsync(string jobName, int parallelism, int completions, string containerName, string containerImage, string imagePullSecret, string _namespace = "default")
{
if (string.IsNullOrEmpty(jobName))
{
throw new ArgumentNullException(nameof(jobName));
}
if (string.IsNullOrEmpty(containerName))
{
throw new ArgumentNullException(nameof(containerName));
}
if (string.IsNullOrEmpty(containerImage))
{
throw new ArgumentNullException(nameof(containerImage));
}
if (string.IsNullOrEmpty(imagePullSecret))
{
throw new ArgumentNullException(nameof(imagePullSecret));
}
if (parallelism < 1)
{
throw new ArgumentOutOfRangeException(nameof(parallelism));
}
if (completions < 1)
{
throw new ArgumentOutOfRangeException(nameof(completions));
}
var job = new V1Job();
job.Metadata = new V1ObjectMeta()
{
Name = jobName
};
job.Spec = new V1JobSpec()
{
Parallelism = parallelism,
Completions = completions,
Template = new V1PodTemplateSpec()
{
Spec = new V1PodSpec()
{
Containers = new List <V1Container>()
{
new V1Container()
{
Name = containerName,
Image = containerImage,
ImagePullPolicy = "Always"
}
},
RestartPolicy = "Never",
ImagePullSecrets = new List <V1LocalObjectReference>()
{
new V1LocalObjectReference()
{
Name = imagePullSecret
}
}
}
}
};
return(await _k8client.CreateNamespacedJobAsync(job, _namespace));
}
private async Task <string> DescribeObject(Kubernetes client, V1Namespace ns, V1Job o, StringBuilder buffer)
{
var fetched = await client.ReadNamespacedJobAsync(o.Metadata.Name, ns.Metadata.Name).ConfigureAwait(false);
buffer.AppendLine($"API Veresion: {fetched.ApiVersion}");
buffer.AppendLine($"Kind: {fetched.Kind}");
buffer.AppendLine(DescribeMetadata(fetched.Metadata));
return($"Job - {fetched.Metadata.Name}");
}
public async Task <Job> SubmitJob(Job job)
{
var resources = new V1ResourceRequirements
{
Requests = new Dictionary <string, ResourceQuantity>()
{
{ "cpu", new ResourceQuantity("1") },
{ "memory", new ResourceQuantity($"1Gi") }
}
};
var podSpec = new V1PodSpec
{
Containers = new[]
{
new V1Container
{
Name = job.Id,
Image = _config["Pi:Docker:Image"],
Command = new []
{
"/app/Pi.Runtime.NetFx",
"-dp",
job.DecimalPlaces.ToString()
},
Resources = resources
}
},
RestartPolicy = "Never"
};
//for running in AKS with ACI integration:
if (_config.GetValue <bool>("Pi:Processors:Kubernetes:UseAci"))
{
AddAciConfiguration(podSpec);
}
var jobSpec = new V1JobSpec()
{
Template = new V1PodTemplateSpec(spec: podSpec)
};
var jobMetadata = new V1ObjectMeta(name: job.Id)
{
Labels = new Dictionary <string, string>()
{
{ "com.pi", "1" }
}
};
var k8sJob = new V1Job(metadata: jobMetadata, spec: jobSpec);
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("*** Generated YAML: ***");
var yaml = Yaml.SaveToString(k8sJob);
_logger.LogDebug(yaml);
_logger.LogDebug("---");
}
await _kubernetes.CreateNamespacedJobAsync(k8sJob, _namespace);
job.ProcessingId = k8sJob.Metadata.Name;
return(job);
}
private async Task <bool> CreateJobAsync(WebPingerArchive archive)
{
// check a pinger exists to archive:
var services = await _kubernetes.ListNamespacedServiceAsync(
Program.NamespaceName,
labelSelector : $"app=web-ping,target={archive.Spec.Target}");
if (!services.Items.Any())
{
Console.WriteLine($"** No WebPinger Service exists for target: {archive.Spec.Target}, in namespace: {Program.NamespaceName}");
return(false);
}
var pingerServiceName = services.Items.First().Metadata.Name;
var name = $"wpa-{archive.Metadata.Name}-{archive.Metadata.CreationTimestamp:yyMMdd-HHmm}";
var jobs = await _kubernetes.ListNamespacedJobAsync(
Program.NamespaceName,
fieldSelector : $"metadata.name={name}");
if (!jobs.Items.Any())
{
var job = new V1Job
{
Metadata = new V1ObjectMeta
{
Name = name,
Labels = new Dictionary <string, string>()
{
{ "kiamol", "ch20" },
}
},
Spec = new V1JobSpec
{
Completions = 1,
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = new Dictionary <string, string>()
{
{ "app", "web-ping-archive" },
{ "target", archive.Spec.Target }
}
},
Spec = new V1PodSpec
{
AutomountServiceAccountToken = false,
RestartPolicy = "Never",
Containers = new List <V1Container>
{
new V1Container
{
Name = "archiver",
Image = "kiamol/ch20-web-ping-archiver",
Env = new List <V1EnvVar>
{
new V1EnvVar
{
Name = "WEB_PING_URL",
Value = $"http://{pingerServiceName}:8080/archive"
}
}
}
}
}
}
}
};
await _kubernetes.CreateNamespacedJobAsync(job, Program.NamespaceName);
Console.WriteLine($"** Created Job: {name}, in namespace: {Program.NamespaceName}");
return(true);
}
else
{
Console.WriteLine($"** Job exists: {name}, in namespace: {Program.NamespaceName}");
return(false);
}
}
