V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
Dictionary <string, string> annotations = module.Config.CreateOptions.Labels
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
var(proxyContainer, proxyVolumes) = this.PrepareProxyContainer(module);
var(moduleContainer, moduleVolumes) = this.PrepareModuleContainer(name, identity, module);
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig
.Map(auth => new List <V1LocalObjectReference> {
new V1LocalObjectReference(auth.Name)
});
Option <IDictionary <string, string> > nodeSelector = Option.Maybe(module.Config.CreateOptions).FlatMap(options => options.NodeSelector);
var modulePodSpec = new V1PodSpec(
containers: new List <V1Container> {
proxyContainer, moduleContainer
},
volumes: proxyVolumes.Concat(moduleVolumes).ToList(),
imagePullSecrets: imageSecret.OrDefault(),
serviceAccountName: name,
nodeSelector: nodeSelector.OrDefault());
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}
public static bool ImageEquals(this V1PodSpec self, V1PodSpec other)
{
if (ReferenceEquals(null, other))
{
return(false);
}
if (ReferenceEquals(self, other))
{
return(true);
}
List <V1Container> otherList = other.Containers.ToList();
foreach (V1Container selfContainer in self.Containers)
{
if (otherList.Exists(c => string.Equals(c.Name, selfContainer.Name)))
{
V1Container otherContainer = otherList.Find(c => string.Equals(c.Name, selfContainer.Name));
if (!string.Equals(selfContainer.Image, otherContainer.Image))
{
// Container has a new image name.
return(false);
}
}
else
{
// container names don't match
return(false);
}
}
return(true);
}
V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule module, IDictionary <string, string> labels)
{
List <V1EnvVar> envVars = this.CollectEnv(module, identity);
// Convert docker labels to annotations because docker labels don't have the same restrictions as Kubernetes labels.
var annotations = Option.Maybe(module.Config.CreateOptions?.Labels)
.Map(dockerLabels => dockerLabels.ToDictionary(label => KubeUtils.SanitizeAnnotationKey(label.Key), label => label.Value))
.GetOrElse(() => new Dictionary <string, string>());
annotations[KubernetesConstants.K8sEdgeOriginalModuleId] = ModuleIdentityHelper.GetModuleName(identity.ModuleId);
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPorts = Option.Maybe(module.Config?.CreateOptions?.ExposedPorts)
.Map(PortExtensions.GetContainerPorts);
// privileged container
Option <V1SecurityContext> securityContext = Option.Maybe(module.Config?.CreateOptions?.HostConfig?.Privileged)
.Map(config => new V1SecurityContext(privileged: true));
// Bind mounts
(List <V1Volume> volumes, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMounts) = this.GetVolumesFromModule(module);
var containers = new List <V1Container>
{
new V1Container(
name,
env: envVars,
image: module.Config.Image,
volumeMounts: volumeMounts,
securityContext: securityContext.OrDefault(),
ports: exposedPorts.OrDefault()),
new V1Container(
"proxy",
env: envVars, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = module.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containers, volumes: volumes, imagePullSecrets: imageSecret.OrDefault(), serviceAccountName: name);
var objectMeta = new V1ObjectMeta(name: name, labels: labels, annotations: annotations);
return(new V1PodTemplateSpec(objectMeta, modulePodSpec));
}
/// <summary>
/// Creates a new <see cref="V1Deployment"/> for the specified <see cref="Resources.Channel"/>
/// </summary>
/// <param name="channel">The <see cref="Resources.Channel"/> to deploy</param>
/// <returns>A new awaitable <see cref="Task"/></returns>
protected virtual async Task CreateChannelDeploymentAsync(Resources.Channel channel)
{
V1Deployment deployment;
try
{
this.Logger.LogInformation("Creating a new deployment for the channel with name '{resourceName}'...", channel.Name());
V1PodSpec podSpec = new V1PodSpec();
V1Container container = channel.Spec.Container;
container.Env.Add(new V1EnvVar("SINK", $"http://gateway.{channel.Namespace()}.svc.cluster.local/events/"));
podSpec.Containers = new List <V1Container>()
{
container
};
V1ObjectMeta podMetadata = new V1ObjectMeta();
podMetadata.Annotations = new Dictionary <string, string>()
{
{ "sidecar.istio.io/inject", "true" }
};
V1PodTemplateSpec podTemplateSpec = new V1PodTemplateSpec(podMetadata, podSpec);
podTemplateSpec.Metadata.Labels = new Dictionary <string, string>()
{
{ "app", channel.Name() },
{ "version", "1.0" }
};
V1DeploymentSpec deploymentSpec = new V1DeploymentSpec(new V1LabelSelector(), podTemplateSpec);
deploymentSpec.Selector.MatchLabels = new Dictionary <string, string>()
{
{ "app", channel.Name() },
{ "version", "1.0" }
};
V1ObjectMeta deploymentMetadata = new V1ObjectMeta(namespaceProperty: channel.Namespace(), name: channel.Name());
deploymentMetadata.Labels = new Dictionary <string, string>()
{
{ "type", EventingDefaults.Labels.Channel }
};
deploymentMetadata.Name = channel.Name();
deploymentMetadata.NamespaceProperty = channel.Namespace();
deployment = new V1Deployment(KubernetesDefaults.ApiVersions.AppsV1, KubernetesDefaults.Kinds.Deployment, deploymentMetadata, deploymentSpec);
await this.KubernetesClient.CreateNamespacedDeploymentAsync(deployment, channel.Namespace());
this.Logger.LogInformation("A new deployment for the channel with name '{resourceName}' has been successfully created.", channel.Name());
}
catch (HttpOperationException ex)
{
this.Logger.LogError($"An error occured while creating the deployment for the channel with name '{{resourceName}}': the server responded with a non-success status code '{{statusCode}}'.{Environment.NewLine}Details: {{responseContent}}", channel.Name(), ex.Response.StatusCode, ex.Response.Content);
throw;
}
}
public PodBuilder AddToleration(string _Effect = null, string _Key = null, string _Value = null, string _Operator = null, long?TolerationSeconds = null)
{
if (Pod.Spec == null)
{
V1PodSpec PodSpec = new V1PodSpec();
}
if (Pod.Spec.Tolerations == null)
{
Pod.Spec.Tolerations = new List <V1Toleration>();
}
Pod.Spec.Tolerations.Add(new V1Toleration(_Effect, _Key, _Operator, TolerationSeconds, _Value));
return(this);
}
public static V1Pod CreatePod(string id)
{
var spec = new V1PodSpec()
{
Containers = new List <V1Container>()
{
new V1Container()
{
Name = $"minecraft-pod-{id}",
Image = "openhack/minecraft-server:2.0",
Ports = new List <V1ContainerPort>()
{
new V1ContainerPort()
{
ContainerPort = 25565
},
new V1ContainerPort()
{
ContainerPort = 25575
},
},
Env = new List <V1EnvVar>()
{
new V1EnvVar()
{
Name = "EULA", Value = "TRUE"
},
}
}
}
};
var metadata = new V1ObjectMeta()
{
Name = $"minecraft-pod-{id}",
Labels = new Dictionary <string, string>()
{
{ "pod-name", $"minecraft-pod-{id}" }
}
};
return(new V1Pod(V1Pod.KubeApiVersion, V1Pod.KubeKind, metadata, spec));
}
private void AddAciConfiguration(V1PodSpec podSpec)
{
//bursting out to ACI is controlled by node selectors and tolerations:
podSpec.NodeSelector = new Dictionary <string, string>
{
{ "beta.kubernetes.io/os", _config["Pi:Docker:OSPlatform"].ToLower() },
{ "kubernetes.io/role", "agent" },
{ "type", "virtual-kubelet" }
};
podSpec.Tolerations = new List <V1Toleration>
{
new V1Toleration
{
Key = "virtual-kubelet.io/provider",
OperatorProperty = "Exists"
},
new V1Toleration
{
Key = "azure.com/aci",
Effect = "NoSchedule"
}
};
}
public static IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req, TraceWriter log, ExecutionContext context)
{
log.Info("C# HTTP trigger function processed a request.");
//var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
var configpath = $"{context.FunctionAppDirectory}\\config\\config.txt";
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath: configpath);
Console.WriteLine("Starting Request!");
IKubernetes client = new Kubernetes(config);
var labels = new Dictionary <string, string>();
labels.Add("app", "raincraft");
var ports = new List <V1ContainerPort>();
ports.Add(new V1ContainerPort()
{
ContainerPort = 25565
});
ports.Add(new V1ContainerPort()
{
ContainerPort = 25575
});
var env = new List <V1EnvVar>();
env.Add(new V1EnvVar()
{
Name = "EULA", Value = "true"
});
var mounts = new List <V1VolumeMount>();
mounts.Add(new V1VolumeMount()
{
Name = "minedb", MountPath = "/data"
});
var volumes = new List <V1Volume>();
volumes.Add(new V1Volume()
{
Name = "minedb",
PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource()
{
ClaimName = "minevol2"
}
});
var spec = new V1PodSpec();
spec.Containers = new List <V1Container>();
spec.Containers.Add(new V1Container()
{
Name = "raincraft-pod",
Image = "openhack/minecraft-server:2.0-alpine",
Ports = ports,
Env = env,
VolumeMounts = mounts
});
spec.Volumes = volumes;
var template = new V1PodTemplateSpec()
{
Metadata = new V1ObjectMeta()
{
Name = "raincraft-pod",
Labels = labels
},
Spec = spec
};
var deployment = new V1Deployment()
{
ApiVersion = "apps/v1",
Kind = "Deployment",
Metadata = new V1ObjectMeta()
{
Name = "raincraft",
},
Spec = new V1DeploymentSpec()
{
Replicas = 1,
Template = template,
Selector = new V1LabelSelector()
{
MatchLabels = labels
}
}
};
deployment.Validate();
var dresult = client.CreateNamespacedDeployment(deployment, "default");
var list = new List <string>();
//foreach (var i in client.ListNamespacedService("default").Items)
//{
// if (i.Metadata.Labels.Contains(new KeyValuePair<string, string>("app", "minepod-service")))
// list.Add(i.Metadata.Name);
//}
var pods = client.ListNamespacedPod("default").Items;
foreach (var i in pods)
{
if (i.Metadata.Labels.Contains(new KeyValuePair <string, string>("app", "raincraft")))
{
list.Add(i.Metadata.Name);
}
}
return(new OkObjectResult(list) as ActionResult);
}
private static ApiJobSubmitRequest CreateJobRequest(string jobSet)
{
var pod = new V1PodSpec
{
Volumes = new List <V1Volume>
{
new V1Volume
{
Name = "root-dir",
FlexVolume = new V1FlexVolumeSource
{
Driver = "gr/cifs",
FsType = "cifs",
SecretRef = new V1LocalObjectReference {
Name = "secret-name"
},
Options = new Dictionary <string, string> {
{ "networkPath", "" }
}
}
}
},
Containers = new[]
{
new V1Container
{
Name = "Container1",
Image = "index.docker.io/library/ubuntu:latest",
Args = new[] { "sleep", "10s" },
SecurityContext = new V1SecurityContext {
RunAsUser = 1000
},
Resources = new V1ResourceRequirements
{
Requests = new V1ResourceList
{
["cpu"] = "120m",
["memory"] = "512Mi"
},
Limits = new V1ResourceList
{
["cpu"] = "120m",
["memory"] = "512Mi"
}
}
}
}
};
return(new ApiJobSubmitRequest
{
Queue = "test",
JobSetId = jobSet,
JobRequestItems = new[]
{
new ApiJobSubmitRequestItem
{
Priority = 1,
PodSpec = pod
}
},
});
}
V1PodTemplateSpec GetPodFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
// pod labels
var podLabels = new Dictionary <string, string>(labels);
// pod annotations
var podAnnotations = new Dictionary <string, string>();
podAnnotations.Add(Constants.K8sEdgeOriginalModuleId, moduleIdentity.ModuleId);
// Convert docker labels to annotations because docker labels don't have the same restrictions as
// Kuberenetes labels.
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
podAnnotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
}
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPortsOption = (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
? this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts).Map(
servicePorts =>
servicePorts.Select(tuple => new V1ContainerPort(tuple.Port, protocol: tuple.Protocol)).ToList())
: Option.None <List <V1ContainerPort> >();
// privileged container
Option <V1SecurityContext> securityContext = (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Privileged == true) ? Option.Some(new V1SecurityContext(privileged: true)) : Option.None <V1SecurityContext>();
// Environment Variables.
List <V1EnvVar> env = this.CollectEnv(moduleWithDockerConfig, (KubernetesModuleIdentity)moduleIdentity);
// Bind mounts
(List <V1Volume> volumeList, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMountList) = this.GetVolumesFromModule(moduleWithDockerConfig).GetOrElse((null, null, null));
// Image
string moduleImage = moduleWithDockerConfig.Config.Image;
var containerList = new List <V1Container>()
{
new V1Container(
KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId),
env: env,
image: moduleImage,
volumeMounts: volumeMountList,
securityContext: securityContext.GetOrElse(() => null),
ports: exposedPortsOption.GetOrElse(() => null)),
// TODO: Add Proxy container here - configmap for proxy configuration.
new V1Container(
"proxy",
env: env, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = moduleWithDockerConfig.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containerList, volumes: volumeList, imagePullSecrets: imageSecret.GetOrElse(() => null));
var objectMeta = new V1ObjectMeta(labels: podLabels, annotations: podAnnotations);
return(new V1PodTemplateSpec(metadata: objectMeta, spec: modulePodSpec));
}
else
{
Events.InvalidModuleType(module);
}
return(new V1PodTemplateSpec());
}
private static V1Pod CreatePodDefinition()
{
Guid podIdentifier = Guid.NewGuid();
var podidentifierstring = podIdentifier.ToString();
string podName = "pod" + podidentifierstring;
string containerPortName = "containerport";
string containerName = "container";
V1ObjectMeta meta = new V1ObjectMeta();
meta.Name = podName;
V1EnvVar addr = new V1EnvVar();
addr.Name = "ACTIVITY_PARTITION_NAME";
addr.Value = "bam_EnrichedFPAttributesV2_Instances";
//V1EnvVar port = new V1EnvVar();
//addr.name("var2");
//addr.value("value2");
//V1ResourceRequirements res = new V1ResourceRequirements();
//Map<String, String> limits = new HashMap<>();
//limits.put("cpu", "300m");
//limits.put("memory", "500Mi");
//res.limits(limits);
V1ContainerPort port = new V1ContainerPort();
port.Name = containerPortName;
port.ContainerPort = 8080;
V1Container container = new V1Container();
container.Name = containerName;
container.Image = "bamreplacementwebapp:dev";
//container.Image = "nbsbamreplacementmigrationservice:dev";
//container.Image = "migrationservice:dev";
container.Ports = new List <V1ContainerPort>();
container.Ports.Add(port);
container.Env = new List <V1EnvVar>();
container.Env.Add(addr);
//container.Args = new List<string>();
//container.Args.Add("bam_EnrichedFPAttributesV2_Instances");
// container.Args.Add("bar");
// container.env(Arrays.asList(addr, port));
//container.resources(res);
V1PodSpec spec = new V1PodSpec();
spec.Containers = new List <V1Container>();
spec.Containers.Add(container);
V1Pod podBody = new V1Pod();
podBody.ApiVersion = "v1";
podBody.Kind = "Pod";
podBody.Metadata = meta;
podBody.Spec = spec;
return(podBody);
}
private static V1Deployment CreateDeploymentDefinition()
{
Guid podIdentifier = Guid.NewGuid();
var podidentifierstring = podIdentifier.ToString();
string podName = "pod-" + podidentifierstring;
string containerPortName = "containerport";
string containerName = "container";
string deploymentName = "poddeployment-" + podIdentifier;
V1ObjectMeta deplMetadata = new V1ObjectMeta();
deplMetadata.Name = deploymentName;
deplMetadata.Labels = new Dictionary <string, string>();
deplMetadata.Labels.Add("component", "migrationservice");
V1DeploymentSpec deplSpec = new V1DeploymentSpec();
deplSpec.Selector = new V1LabelSelector();
deplSpec.Selector.MatchLabels = new Dictionary <string, string>();
deplSpec.Selector.MatchLabels.Add("app", "bamreplacement");
deplSpec.Replicas = 3; // to be tokenized
deplSpec.Strategy = new V1DeploymentStrategy();
deplSpec.Strategy.Type = "RollingUpdate";
deplSpec.Strategy.RollingUpdate = new V1RollingUpdateDeployment();
deplSpec.Strategy.RollingUpdate.MaxSurge = 1;
deplSpec.Strategy.RollingUpdate.MaxUnavailable = 0;
V1ObjectMeta podMetadata = new V1ObjectMeta();
podMetadata.Name = podName;
podMetadata.Labels = new Dictionary <string, string>();
podMetadata.Labels.Add("app", "bamreplacement");
V1ContainerPort port = new V1ContainerPort();
port.Name = containerPortName;
port.ContainerPort = 8080;
V1EnvVar addr = new V1EnvVar();
addr.Name = "ACTIVITY_PARTITION_NAME";
addr.Value = "bam_EnrichedFPAttributesV2_Instances";
V1Container container = new V1Container();
container.Name = containerName;
// container.Image = "abhishekagg24/stableapp:v1";
//container.Image = "nbsbamreplacementmigrationservice:dev";
container.Image = "bamreplacementwebapp:dev";
container.Ports = new List <V1ContainerPort>();
container.Ports.Add(port);
container.Env = new List <V1EnvVar>();
container.Env.Add(addr);
V1PodSpec podSpec = new V1PodSpec();
podSpec.Containers = new List <V1Container>();
podSpec.Containers.Add(container);
deplSpec.Template = new V1PodTemplateSpec();
deplSpec.Template.Metadata = podMetadata;
deplSpec.Template.Spec = podSpec;
V1Deployment deployment = new V1Deployment();
deployment.ApiVersion = "apps/v1";
deployment.Kind = "Deployment";
deployment.Metadata = deplMetadata;
deployment.Spec = deplSpec;
return(deployment);
}
public async Task <Job> SubmitJob(Job job)
{
var resources = new V1ResourceRequirements
{
Requests = new Dictionary <string, ResourceQuantity>()
{
{ "cpu", new ResourceQuantity("1") },
{ "memory", new ResourceQuantity($"1Gi") }
}
};
var podSpec = new V1PodSpec
{
Containers = new[]
{
new V1Container
{
Name = job.Id,
Image = _config["Pi:Docker:Image"],
Command = new []
{
"/app/Pi.Runtime.NetFx",
"-dp",
job.DecimalPlaces.ToString()
},
Resources = resources
}
},
RestartPolicy = "Never"
};
//for running in AKS with ACI integration:
if (_config.GetValue <bool>("Pi:Processors:Kubernetes:UseAci"))
{
AddAciConfiguration(podSpec);
}
var jobSpec = new V1JobSpec()
{
Template = new V1PodTemplateSpec(spec: podSpec)
};
var jobMetadata = new V1ObjectMeta(name: job.Id)
{
Labels = new Dictionary <string, string>()
{
{ "com.pi", "1" }
}
};
var k8sJob = new V1Job(metadata: jobMetadata, spec: jobSpec);
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("*** Generated YAML: ***");
var yaml = Yaml.SaveToString(k8sJob);
_logger.LogDebug(yaml);
_logger.LogDebug("---");
}
await _kubernetes.CreateNamespacedJobAsync(k8sJob, _namespace);
job.ProcessingId = k8sJob.Metadata.Name;
return(job);
}
