/// <summary>
/// Initializes a new <see cref="VirtualService"/>
/// </summary>
/// <param name="metadata">The <see cref="VirtualService"/>'s <see cref="V1ObjectMeta"/></param>
public VirtualService(V1ObjectMeta metadata)
: this(metadata, null)
{
}
private async Task ManageDeployments(V1ServiceList currentServices, V1DeploymentList currentDeployments, EdgeDeploymentDefinition <TConfig> customObject)
{
var desiredModules = ModuleSet.Create(customObject.Spec.ToArray());
var moduleIdentities = await this.moduleIdentityLifecycleManager.GetModuleIdentitiesAsync(desiredModules, this.currentModules);
// Pull current configuration from annotations.
Dictionary <string, string> currentV1ServicesFromAnnotations = this.GetCurrentServiceConfig(currentServices);
// strip out edgeAgent so edgeAgent doesn't update itself.
// TODO: remove this filter.
var agentDeploymentName = this.DeploymentName(CoreConstants.EdgeAgentModuleName);
Dictionary <string, string> currentDeploymentsFromAnnotations = this.GetCurrentDeploymentConfig(currentDeployments)
.ToDictionary(
pair => pair.Key,
pair => pair.Value);
var desiredServices = new List <V1Service>();
var desiredDeployments = new List <V1Deployment>();
foreach (KubernetesModule <TConfig> module in customObject.Spec)
{
var moduleId = moduleIdentities[module.Name];
if (string.Equals(module.Type, "docker"))
{
// Default labels
var labels = new Dictionary <string, string>
{
[Constants.K8sEdgeModuleLabel] = KubeUtils.SanitizeLabelValue(moduleId.ModuleId),
[Constants.K8sEdgeDeviceLabel] = KubeUtils.SanitizeLabelValue(this.deviceId),
[Constants.K8sEdgeHubNameLabel] = KubeUtils.SanitizeLabelValue(this.iotHubHostname)
};
// Create a Service for every network interface of each module. (label them with hub, device and module id)
Option <V1Service> moduleService = this.GetServiceFromModule(labels, module, moduleId);
moduleService.ForEach(service => desiredServices.Add(service));
// Create a Pod for each module, and a proxy container.
V1PodTemplateSpec v1PodSpec = this.GetPodFromModule(labels, module, moduleId);
// if this is the edge agent's deployment then it needs to run under a specific service account
if (moduleIdentities[module.Name].ModuleId == CoreConstants.EdgeAgentModuleIdentityName)
{
v1PodSpec.Spec.ServiceAccountName = this.serviceAccountName;
}
// Bundle into a deployment
string deploymentName = this.DeploymentName(moduleIdentities[module.Name].ModuleId);
// Deployment data
var deploymentMeta = new V1ObjectMeta(name: deploymentName, labels: labels);
var selector = new V1LabelSelector(matchLabels: labels);
var deploymentSpec = new V1DeploymentSpec(replicas: 1, selector: selector, template: v1PodSpec);
desiredDeployments.Add(new V1Deployment(metadata: deploymentMeta, spec: deploymentSpec));
}
else
{
Events.InvalidModuleType(module);
}
}
// Find current Services/Deployments which need to be removed and updated
var servicesRemoved = new List <V1Service>(currentServices.Items);
servicesRemoved.RemoveAll(s => desiredServices.Exists(i => string.Equals(i.Metadata.Name, s.Metadata.Name)));
var deploymentsRemoved = new List <V1Deployment>(currentDeployments.Items);
deploymentsRemoved.RemoveAll(
d =>
{
return(desiredDeployments.Exists(i => string.Equals(i.Metadata.Name, d.Metadata.Name)));
});
var newServices = new List <V1Service>();
desiredServices.ForEach(
s =>
{
string creationString = JsonConvert.SerializeObject(s);
if (currentV1ServicesFromAnnotations.ContainsKey(s.Metadata.Name))
{
string serviceAnnotation = currentV1ServicesFromAnnotations[s.Metadata.Name];
// If configuration matches, no need to update service
if (string.Equals(serviceAnnotation, creationString))
{
return;
}
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
servicesRemoved.Add(s);
newServices.Add(s);
Events.UpdateService(s.Metadata.Name);
}
else
{
if (s.Metadata.Annotations == null)
{
s.Metadata.Annotations = new Dictionary <string, string>();
}
s.Metadata.Annotations[Constants.CreationString] = creationString;
newServices.Add(s);
Events.CreateService(s.Metadata.Name);
}
});
var deploymentsUpdated = new List <V1Deployment>();
var newDeployments = new List <V1Deployment>();
List <V1Deployment> currentDeploymentsList = currentDeployments.Items.ToList();
desiredDeployments.ForEach(
d =>
{
if (currentDeploymentsFromAnnotations.ContainsKey(d.Metadata.Name))
{
V1Deployment current = currentDeploymentsList.Find(i => string.Equals(i.Metadata.Name, d.Metadata.Name));
string currentFromAnnotation = currentDeploymentsFromAnnotations[d.Metadata.Name];
string creationString = JsonConvert.SerializeObject(d);
// If configuration matches, or this is edgeAgent deployment and the images match,
// no need to do update deployment
if (string.Equals(currentFromAnnotation, creationString) ||
(string.Equals(d.Metadata.Name, this.DeploymentName(CoreConstants.EdgeAgentModuleName)) && V1DeploymentEx.ImageEquals(current, d)))
{
return;
}
d.Metadata.ResourceVersion = current.Metadata.ResourceVersion;
if (d.Metadata.Annotations == null)
{
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
}
else
{
d.Metadata.Annotations[Constants.CreationString] = creationString;
}
deploymentsUpdated.Add(d);
Events.UpdateDeployment(d.Metadata.Name);
}
else
{
string creationString = JsonConvert.SerializeObject(d);
var annotations = new Dictionary <string, string>
{
[Constants.CreationString] = creationString
};
d.Metadata.Annotations = annotations;
newDeployments.Add(d);
Events.CreateDeployment(d.Metadata.Name);
}
});
// Remove the old
IEnumerable <Task <V1Status> > removeServiceTasks = servicesRemoved.Select(
i =>
{
Events.DeletingService(i);
return(this.client.DeleteNamespacedServiceAsync(i.Metadata.Name, this.k8sNamespace, new V1DeleteOptions()));
});
await Task.WhenAll(removeServiceTasks);
IEnumerable <Task <V1Status> > removeDeploymentTasks = deploymentsRemoved.Select(
d =>
{
Events.DeletingDeployment(d);
return(this.client.DeleteNamespacedDeployment1Async(d.Metadata.Name, this.k8sNamespace, new V1DeleteOptions(propagationPolicy: "Foreground"), propagationPolicy: "Foreground"));
});
await Task.WhenAll(removeDeploymentTasks);
// Create the new.
IEnumerable <Task <V1Service> > createServiceTasks = newServices.Select(
s =>
{
Events.CreatingService(s);
return(this.client.CreateNamespacedServiceAsync(s, this.k8sNamespace));
});
await Task.WhenAll(createServiceTasks);
IEnumerable <Task <V1Deployment> > createDeploymentTasks = newDeployments.Select(
deployment =>
{
Events.CreatingDeployment(deployment);
return(this.client.CreateNamespacedDeploymentAsync(deployment, this.k8sNamespace));
});
await Task.WhenAll(createDeploymentTasks);
// Update the existing - should only do this when different.
IEnumerable <Task <V1Deployment> > updateDeploymentTasks = deploymentsUpdated.Select(deployment => this.client.ReplaceNamespacedDeploymentAsync(deployment, deployment.Metadata.Name, this.k8sNamespace));
await Task.WhenAll(updateDeploymentTasks);
this.currentModules = desiredModules;
}
private Option <V1Service> GetServiceFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
var portList = new List <V1ServicePort>();
Option <Dictionary <string, string> > serviceAnnotations = Option.None <Dictionary <string, string> >();
bool onlyExposedPorts = true;
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
// Add annotations from Docker labels. This provides the customer a way to assign annotations to services if they want
// to tie backend services to load balancers via an Ingress Controller.
var annotations = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
annotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
serviceAnnotations = Option.Some(annotations);
}
// Handle ExposedPorts entries
if (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
{
// Entries in the Exposed Port list just tell Docker that this container wants to listen on that port.
// We interpret this as a "ClusterIP" service type listening on that exposed port, backed by this module.
// Users of this Module's exposed port should be able to find the service by connecting to "<module name>:<port>"
this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts)
.ForEach(
exposedList =>
exposedList.ForEach((item) => portList.Add(new V1ServicePort(item.Port, name: $"ExposedPort-{item.Port}-{item.Protocol.ToLower()}", protocol: item.Protocol))));
}
// Handle HostConfig PortBindings entries
if (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings != null)
{
foreach (KeyValuePair <string, IList <DockerModels.PortBinding> > portBinding in moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.PortBindings)
{
string[] portProtocol = portBinding.Key.Split('/');
if (portProtocol.Length == 2)
{
if (int.TryParse(portProtocol[0], out int port) && this.ValidateProtocol(portProtocol[1], out string protocol))
{
// Entries in Docker portMap wants to expose a port on the host (hostPort) and map it to the container's port (port)
// We interpret that as the pod wants the cluster to expose a port on a public IP (hostPort), and target it to the container's port (port)
foreach (DockerModels.PortBinding hostBinding in portBinding.Value)
{
if (int.TryParse(hostBinding.HostPort, out int hostPort))
{
// If a port entry contains the same "port", then remove it and replace with a new ServicePort that contains a target.
var duplicate = portList.SingleOrDefault(a => a.Port == hostPort);
if (duplicate != default(V1ServicePort))
{
portList.Remove(duplicate);
}
portList.Add(new V1ServicePort(hostPort, name: $"HostPort-{port}-{protocol.ToLower()}", protocol: protocol, targetPort: port));
onlyExposedPorts = false;
}
else
{
Events.PortBindingValue(module, portBinding.Key);
}
}
}
}
}
}
}
if (portList.Count > 0)
{
// Selector: by module name and device name, also how we will label this puppy.
var objectMeta = new V1ObjectMeta(annotations: serviceAnnotations.GetOrElse(() => null), labels: labels, name: KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId));
// How we manage this service is dependent on the port mappings user asks for.
// If the user tells us to only use ClusterIP ports, we will always set the type to ClusterIP.
// If all we had were exposed ports, we will assume ClusterIP. Otherwise, we use the given value as the default service type
//
// If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
// This gives the user more control as to how they want this to work.
string serviceType;
if (onlyExposedPorts)
{
serviceType = "ClusterIP";
}
else
{
serviceType = this.defaultMapServiceType;
}
return(Option.Some(new V1Service(metadata: objectMeta, spec: new V1ServiceSpec(type: serviceType, ports: portList, selector: labels))));
}
else
{
return(Option.None <V1Service>());
}
}
private static IList <V1OwnerReference> EnsureOwnerReferences(this V1ObjectMeta meta) => meta.OwnerReferences ??= new List <V1OwnerReference>();
public async Task ManageFileShareSecretAsync(V1Secret secret)
{
byte[] accountKeyData;
byte[] accountNameData;
if (!secret.Data.TryGetValue(AccountKey, out accountKeyData))
{
Console.WriteLine($"Secret {secret.Metadata.Name} doesn't have [{AccountKey}] Data");
return;
}
if (!secret.Data.TryGetValue(AccountName, out accountNameData))
{
Console.WriteLine($"Secret {secret.Metadata.Name} doesn't have [{AccountName}] Data");
return;
}
var pvLabels = new Dictionary <string, string>
{
[Constants.LabelSelectorKey] = Constants.LabelSelectorValue
};
var mountOptions = new List <string>
{
"dir_mode=0777",
"file_mode=0777",
"uid=1000",
"gid=1000",
"mfsymlinks",
"nobrl"
};
V1PersistentVolumeList currentPvs = await k8sClient.ListPersistentVolumeAsync(labelSelector : Constants.LabelSelector);
var existingPvSet = new Set <V1PersistentVolume>(currentPvs.Items
.Where(pv => pv.Spec?.AzureFile?.SecretName == secret.Metadata.Name)
.ToDictionary(pv => pv.Metadata.Name));
var desiredPvs = new ConcurrentDictionary <string, V1PersistentVolume>();
string accountKey = Encoding.UTF8.GetString(accountKeyData);
string accountName = Encoding.UTF8.GetString(accountNameData);
string connectionString = $"DefaultEndpointsProtocol=https;AccountName={accountName};AccountKey={accountKey};EndpointSuffix=core.windows.net";
// Open a FileShare client with secret.
var serviceClient = new ShareServiceClient(connectionString);
var shares = serviceClient.GetSharesAsync(ShareTraits.Metadata, ShareStates.None);
await foreach (var share in shares)
{
// Get all file shares from client that match a trait
if ((share.Properties?.Metadata != null) &&
(share.Properties.Metadata.TryGetValue(Constants.LabelSelectorKey, out string labelValue)) &&
(labelValue == Constants.LabelSelectorValue))
{
// Create a PV from secret and ShareItem
Console.WriteLine($"ShareItem {share.Name} found!");
string name = KubeUtils.SanitizeK8sValue($"{accountName}-{share.Name}");
var metadata = new V1ObjectMeta(name: name, labels: pvLabels);
var accessModes = new List <string> {
AccessMode
};
var azurefile = new V1AzureFilePersistentVolumeSource(secret.Metadata.Name, share.Name, readOnlyProperty: false, secret.Metadata.NamespaceProperty);
var capacity = new Dictionary <string, ResourceQuantity> {
["storage"] = new ResourceQuantity($"{share.Properties.QuotaInGB}Gi")
};
var spec = new V1PersistentVolumeSpec(
accessModes: accessModes,
azureFile: azurefile,
capacity: capacity,
storageClassName: StorageClassName,
mountOptions: mountOptions);
var pv = new V1PersistentVolume(metadata: metadata, spec: spec);
if (!desiredPvs.TryAdd(name, pv))
{
Console.WriteLine($"Duplicate share name {name}");
}
}
}
var desiredPvSet = new Set <V1PersistentVolume>(desiredPvs);
var diff = desiredPvSet.Diff(existingPvSet, PvComparer);
await this.ManagePvs(diff);
}
public static IDictionary <string, string> SafeAnnotations(this V1ObjectMeta metadata)
{
return(metadata.Annotations ?? new Dictionary <string, string>());
}
public static bool ReflectionVersionMatch(this V1ObjectMeta metadata, string version)
{
return(metadata.ReflectionVersion() == version);
}
public KubernetesObjectId(V1ObjectMeta metadata) : this(metadata.NamespaceProperty, metadata.Name)
{
}
public static KubernetesObjectId For(V1ObjectMeta metadata)
{
return(new KubernetesObjectId(metadata));
}
/// <summary>
/// Initializes a new <see cref="Channel"/>
/// </summary>
/// <param name="metadata">The <see cref="Channel"/>'s <see cref="V1ObjectMeta"/></param>
/// <param name="spec">The <see cref="Channel"/>'s <see cref="ChannelSpec"/></param>
public Channel(V1ObjectMeta metadata, ChannelSpec spec)
: this()
{
this.Metadata = metadata;
this.Spec = spec;
}
/// <summary>
/// Initializes a new <see cref="Channel"/>
/// </summary>
/// <param name="metadata">The <see cref="Channel"/>'s <see cref="V1ObjectMeta"/></param>
public Channel(V1ObjectMeta metadata)
: this(metadata, null)
{
}
public async Task <Job> SubmitJob(Job job)
{
var resources = new V1ResourceRequirements
{
Requests = new Dictionary <string, ResourceQuantity>()
{
{ "cpu", new ResourceQuantity("1") },
{ "memory", new ResourceQuantity($"1Gi") }
}
};
var podSpec = new V1PodSpec
{
Containers = new[]
{
new V1Container
{
Name = job.Id,
Image = _config["Pi:Docker:Image"],
Command = new []
{
"/app/Pi.Runtime.NetFx",
"-dp",
job.DecimalPlaces.ToString()
},
Resources = resources
}
},
RestartPolicy = "Never"
};
//for running in AKS with ACI integration:
if (_config.GetValue <bool>("Pi:Processors:Kubernetes:UseAci"))
{
AddAciConfiguration(podSpec);
}
var jobSpec = new V1JobSpec()
{
Template = new V1PodTemplateSpec(spec: podSpec)
};
var jobMetadata = new V1ObjectMeta(name: job.Id)
{
Labels = new Dictionary <string, string>()
{
{ "com.pi", "1" }
}
};
var k8sJob = new V1Job(metadata: jobMetadata, spec: jobSpec);
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("*** Generated YAML: ***");
var yaml = Yaml.SaveToString(k8sJob);
_logger.LogDebug(yaml);
_logger.LogDebug("---");
}
await _kubernetes.CreateNamespacedJobAsync(k8sJob, _namespace);
job.ProcessingId = k8sJob.Metadata.Name;
return(job);
}
/// <summary>
/// Creates a virtual <see cref="V1Service"/> for the specified <see cref="Broker"/>
/// </summary>
/// <param name="broker">The <see cref="Broker"/> to deploy</param>
/// <returns>A new awaitable <see cref="Task"/></returns>
protected virtual async Task CreateBrokerVirtualServiceAsync(Broker broker)
{
VirtualService virtualService;
try
{
this.Logger.LogInformation("Creating a new istio virtual service for the broker with name '{resourceName}'...", broker.Name());
V1ObjectMeta metadata = new V1ObjectMeta(name: $"{broker.Name()}-vs", namespaceProperty: broker.Namespace());
VirtualServiceSpec spec = new VirtualServiceSpec()
{
Hosts = new List <string>()
{
broker.Name()
},
Http = new List <HttpRoute>()
{
new HttpRoute()
{
Headers = new Headers()
{
Request = new HeadersOperations()
{
Add = new Dictionary <string, string>()
{
{ EventingDefaults.Headers.Channel, broker.Spec.Channel }
}
}
},
Route = new List <HttpRouteDestination>()
{
new HttpRouteDestination()
{
Destination = new Destination(broker.Name())
{
Port = new PortSelector(80)
}
}
}
}
}
};
virtualService = new VirtualService(metadata, spec);
await this.KubernetesClient.CreateNamespacedCustomObjectAsync(virtualService, broker.Namespace());
this.Logger.LogInformation("A new istio virtual service for the broker with name '{resourceName}' has been successfully created.", broker.Name());
}
catch (HttpOperationException ex)
{
this.Logger.LogError($"An error occured while creating the istio virtual service for the broker with name '{{resourceName}}': the server responded with a non-success status code '{{statusCode}}'.{Environment.NewLine}Details: {{responseContent}}", broker.Name(), ex.Response.StatusCode, ex.Response.Content);
return;
}
try
{
this.Logger.LogInformation("Updating the status of the broker with name '{resourceName}'...", broker.Name());
broker.Status = new BrokerStatus()
{
Url = $"http://{broker.Name()}.{broker.Namespace()}.svc.cluster.local"
};
await this.KubernetesClient.ReplaceNamespacedCustomObjectStatusAsync(broker, broker.ApiGroup(), broker.ApiGroupVersion(), broker.Namespace(), BrokerDefinition.PLURAL, broker.Name());
this.Logger.LogInformation("The status of the broker with name '{resourceName}' has been successfully updated", broker.Name());
}
catch (HttpOperationException ex)
{
this.Logger.LogError($"An error occured while updating the status of the CRD '{{resourceKind}}' with name '{{resourceName}}': the server responded with a non-success status code '{{statusCode}}'.{Environment.NewLine}Details: {{responseContent}}", broker.Kind, broker.Name(), ex.Response.StatusCode, ex.Response.Content);
}
}
public static bool PodHasMatchingNamespace(this V1Pod pod, V1ObjectMeta metadata)
{
return(pod.Metadata.Namespace() == metadata.Namespace());
}
/// <summary>
/// Initializes a new instance of the <see cref="ArgoWorkflowTemplate" /> class.
/// </summary>
/// <param name="apiVersion">APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources.</param>
/// <param name="kind">Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds.</param>
/// <param name="metadata">metadata.</param>
/// <param name="spec">spec (required).</param>
public ArgoWorkflowTemplate(string apiVersion = default(string), string kind = default(string), V1ObjectMeta metadata = default(V1ObjectMeta), ArgoWorkflowTemplateSpec spec = default(ArgoWorkflowTemplateSpec))
{
// to ensure "spec" is required (not null)
if (spec == null)
{
throw new InvalidDataException("spec is a required property for ArgoWorkflowTemplate and cannot be null");
}
else
{
this.Spec = spec;
}
this.ApiVersion = apiVersion;
this.Kind = kind;
this.Metadata = metadata;
}
public ChallangeResource(V1CustomResourceDefinitionSpec spec, string apiVersion = null, string kind = null, V1ObjectMeta metadata = null, V1CustomResourceDefinitionStatus status = null) : base(spec, apiVersion, kind, metadata, status)
{
}
public static string FortiCertificate(this V1ObjectMeta metadata)
{
return(metadata.SafeAnnotations().TryGetValue(Annotations.Reflection.FortiCertificate, out var raw)
? string.IsNullOrWhiteSpace(raw) ? null : raw
: null);
}
private static YarpIngressOptions HandleAnnotations(YarpIngressContext context, V1ObjectMeta metadata)
{
var options = context.Options;
var annotations = metadata.Annotations;
if (annotations == null)
{
return(options);
}
if (annotations.TryGetValue("yarp.ingress.kubernetes.io/backend-protocol", out var http))
{
options.Https = http.Equals("https", StringComparison.OrdinalIgnoreCase);
}
// metadata to support:
// rewrite target
// auth
// http or https
// default backend
// CORS
// GRPC
// HTTP2
// Conneciton limits
// rate limits
// backend health checks.
return(options);
}
public static string ReflectionAllowedNamespaces(this V1ObjectMeta metadata)
{
return(metadata.SafeAnnotations().TryGetValue(Annotations.Reflection.AllowedNamespaces, out var raw)
? string.IsNullOrWhiteSpace(raw) ? null : raw
: null);
}
private static V1Deployment CreateDeploymentDefinition()
{
Guid podIdentifier = Guid.NewGuid();
var podidentifierstring = podIdentifier.ToString();
string podName = "pod-" + podidentifierstring;
string containerPortName = "containerport";
string containerName = "container";
string deploymentName = "poddeployment-" + podIdentifier;
V1ObjectMeta deplMetadata = new V1ObjectMeta();
deplMetadata.Name = deploymentName;
deplMetadata.Labels = new Dictionary <string, string>();
deplMetadata.Labels.Add("component", "migrationservice");
V1DeploymentSpec deplSpec = new V1DeploymentSpec();
deplSpec.Selector = new V1LabelSelector();
deplSpec.Selector.MatchLabels = new Dictionary <string, string>();
deplSpec.Selector.MatchLabels.Add("app", "bamreplacement");
deplSpec.Replicas = 3; // to be tokenized
deplSpec.Strategy = new V1DeploymentStrategy();
deplSpec.Strategy.Type = "RollingUpdate";
deplSpec.Strategy.RollingUpdate = new V1RollingUpdateDeployment();
deplSpec.Strategy.RollingUpdate.MaxSurge = 1;
deplSpec.Strategy.RollingUpdate.MaxUnavailable = 0;
V1ObjectMeta podMetadata = new V1ObjectMeta();
podMetadata.Name = podName;
podMetadata.Labels = new Dictionary <string, string>();
podMetadata.Labels.Add("app", "bamreplacement");
V1ContainerPort port = new V1ContainerPort();
port.Name = containerPortName;
port.ContainerPort = 8080;
V1EnvVar addr = new V1EnvVar();
addr.Name = "ACTIVITY_PARTITION_NAME";
addr.Value = "bam_EnrichedFPAttributesV2_Instances";
V1Container container = new V1Container();
container.Name = containerName;
// container.Image = "abhishekagg24/stableapp:v1";
//container.Image = "nbsbamreplacementmigrationservice:dev";
container.Image = "bamreplacementwebapp:dev";
container.Ports = new List <V1ContainerPort>();
container.Ports.Add(port);
container.Env = new List <V1EnvVar>();
container.Env.Add(addr);
V1PodSpec podSpec = new V1PodSpec();
podSpec.Containers = new List <V1Container>();
podSpec.Containers.Add(container);
deplSpec.Template = new V1PodTemplateSpec();
deplSpec.Template.Metadata = podMetadata;
deplSpec.Template.Spec = podSpec;
V1Deployment deployment = new V1Deployment();
deployment.ApiVersion = "apps/v1";
deployment.Kind = "Deployment";
deployment.Metadata = deplMetadata;
deployment.Spec = deplSpec;
return(deployment);
}
public static string ReflectionVersion(this V1ObjectMeta metadata)
{
return(metadata.Annotations.TryGetValue(Annotations.Reflection.ReflectedVersion, out var revisionValue)
? revisionValue
: string.Empty);
}
private static V1Pod CreatePodDefinition()
{
Guid podIdentifier = Guid.NewGuid();
var podidentifierstring = podIdentifier.ToString();
string podName = "pod" + podidentifierstring;
string containerPortName = "containerport";
string containerName = "container";
V1ObjectMeta meta = new V1ObjectMeta();
meta.Name = podName;
V1EnvVar addr = new V1EnvVar();
addr.Name = "ACTIVITY_PARTITION_NAME";
addr.Value = "bam_EnrichedFPAttributesV2_Instances";
//V1EnvVar port = new V1EnvVar();
//addr.name("var2");
//addr.value("value2");
//V1ResourceRequirements res = new V1ResourceRequirements();
//Map<String, String> limits = new HashMap<>();
//limits.put("cpu", "300m");
//limits.put("memory", "500Mi");
//res.limits(limits);
V1ContainerPort port = new V1ContainerPort();
port.Name = containerPortName;
port.ContainerPort = 8080;
V1Container container = new V1Container();
container.Name = containerName;
container.Image = "bamreplacementwebapp:dev";
//container.Image = "nbsbamreplacementmigrationservice:dev";
//container.Image = "migrationservice:dev";
container.Ports = new List <V1ContainerPort>();
container.Ports.Add(port);
container.Env = new List <V1EnvVar>();
container.Env.Add(addr);
//container.Args = new List<string>();
//container.Args.Add("bam_EnrichedFPAttributesV2_Instances");
// container.Args.Add("bar");
// container.env(Arrays.asList(addr, port));
//container.resources(res);
V1PodSpec spec = new V1PodSpec();
spec.Containers = new List <V1Container>();
spec.Containers.Add(container);
V1Pod podBody = new V1Pod();
podBody.ApiVersion = "v1";
podBody.Kind = "Pod";
podBody.Metadata = meta;
podBody.Spec = spec;
return(podBody);
}
public async void CrdCommandExecuteWithAuthReplaceObjects()
{
string resourceName = Hostname + Constants.K8sNameDivider + DeviceId.ToLower();
string metaApiVersion = Constants.K8sApi + "/" + Constants.K8sApiVersion;
string secretName = "username-docker.io";
var secretData = new Dictionary <string, byte[]> {
[Constants.K8sPullSecretData] = Encoding.UTF8.GetBytes("Invalid Secret Data")
};
var secretMeta = new V1ObjectMeta(name: secretName, namespaceProperty: Ns);
IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars);
var km1 = new KubernetesModule <DockerConfig>((IModule <DockerConfig>)m1);
KubernetesModule <DockerConfig>[] modules = { km1 };
var token = new CancellationToken();
Option <IRuntimeInfo> runtimeOption = Option.Maybe(Runtime);
var auth = new AuthConfig()
{
Username = "******", Password = "******", ServerAddress = "docker.io"
};
var configProvider = new Mock <ICombinedConfigProvider <CombinedDockerConfig> >();
configProvider.Setup(cp => cp.GetCombinedConfig(km1, Runtime)).Returns(() => new CombinedDockerConfig("test-image:1", Config1.CreateOptions, Option.Maybe(auth)));
var existingSecret = new V1Secret("v1", secretData, type: Constants.K8sPullSecretType, kind: "Secret", metadata: secretMeta);
var existingDeployment = new EdgeDeploymentDefinition <DockerConfig>(metaApiVersion, Constants.K8sCrdKind, new V1ObjectMeta(name: resourceName), new List <KubernetesModule <DockerConfig> >());
bool getSecretCalled = false;
bool putSecretCalled = false;
bool getCrdCalled = false;
bool putCrdCalled = false;
using (var server = new MockKubeApiServer(
resp: string.Empty,
shouldNext: async httpContext =>
{
string pathStr = httpContext.Request.Path.Value;
string method = httpContext.Request.Method;
if (string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase))
{
if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets/{secretName}"))
{
getSecretCalled = true;
await httpContext.Response.Body.WriteAsync(JsonConvert.SerializeObject(existingSecret).ToBody(), token);
}
else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}/{resourceName}"))
{
getCrdCalled = true;
await httpContext.Response.Body.WriteAsync(JsonConvert.SerializeObject(existingDeployment).ToBody(), token);
}
}
else if (string.Equals(method, "PUT", StringComparison.OrdinalIgnoreCase))
{
httpContext.Response.Body = httpContext.Request.Body;
if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets/{secretName}"))
{
putSecretCalled = true;
}
else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}/{resourceName}"))
{
putCrdCalled = true;
}
}
return(false);
}))
{
var client = new Kubernetes(
new KubernetesClientConfiguration
{
Host = server.Uri.ToString()
});
var cmd = new KubernetesCrdCommand <CombinedDockerConfig>(Ns, Hostname, DeviceId, client, modules, runtimeOption, configProvider.Object);
await cmd.ExecuteAsync(token);
Assert.True(getSecretCalled, nameof(getSecretCalled));
Assert.True(putSecretCalled, nameof(putSecretCalled));
Assert.True(getCrdCalled, nameof(getCrdCalled));
Assert.True(putCrdCalled, nameof(putCrdCalled));
}
}
/// <summary>
/// Initializes a new <see cref="Broker"/>
/// </summary>
/// <param name="metadata">The <see cref="Broker"/>'s <see cref="V1ObjectMeta"/></param>
/// <param name="spec">The <see cref="Broker"/>'s <see cref="BrokerSpec"/></param>
public Broker(V1ObjectMeta metadata, BrokerSpec spec)
: this(metadata, spec, null)
{
}
public V1Deployment Build(HealthCheckResource resource)
{
var metadata = new V1ObjectMeta
{
OwnerReferences = new List <V1OwnerReference> {
resource.CreateOwnerReference()
},
Annotations = new Dictionary <string, string>(),
Labels = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
},
Name = $"{resource.Spec.Name}-deploy",
NamespaceProperty = resource.Metadata.NamespaceProperty
};
var spec = new V1DeploymentSpec
{
Selector = new V1LabelSelector
{
MatchLabels = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
}
},
Replicas = 1,
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = new Dictionary <string, string>
{
["app"] = resource.Spec.Name
},
},
Spec = new V1PodSpec
{
Containers = new List <V1Container>
{
new V1Container
{
ImagePullPolicy = resource.Spec.ImagePullPolicy ?? Constants.DefaultPullPolicy,
Name = Constants.PodName,
Image = resource.Spec.Image ?? Constants.ImageName,
Ports = new List <V1ContainerPort>
{
new V1ContainerPort(80)
},
Env = new List <V1EnvVar>
{
new V1EnvVar("ui_path", resource.Spec.UiPath ?? Constants.DefaultUIPath),
new V1EnvVar("enable_push_endpoint", "true"),
new V1EnvVar("push_endpoint_secret", valueFrom: new V1EnvVarSource(secretKeyRef: new V1SecretKeySelector("key", $"{resource.Spec.Name}-secret"))),
new V1EnvVar("Logging__LogLevel__Default", "Debug"),
new V1EnvVar("Logging__LogLevel__Microsoft", "Warning"),
new V1EnvVar("Logging__LogLevel__System", "Warning"),
new V1EnvVar("Logging__LogLevel__HealthChecks", "Information")
}
}
}
}
}
};
foreach (var annotation in resource.Spec.DeploymentAnnotations)
{
_logger.LogInformation("Adding annotation {Annotation} to ui deployment with value {AnnotationValue}", annotation.Name, annotation.Value);
metadata.Annotations.Add(annotation.Name, annotation.Value);
}
var specification = spec.Template.Spec;
var container = specification.Containers.First();
for (int i = 0; i < resource.Spec.Webhooks.Count(); i++)
{
var webhook = resource.Spec.Webhooks[i];
_logger.LogInformation("Adding webhook configuration for webhook {Webhook}", webhook.Name);
container.Env.Add(new V1EnvVar($"HealthChecksUI__Webhooks__{i}__Name", webhook.Name));
container.Env.Add(new V1EnvVar($"HealthChecksUI__Webhooks__{i}__Uri", webhook.Uri));
container.Env.Add(new V1EnvVar($"HealthChecksUI__Webhooks__{i}__Payload", webhook.Payload));
container.Env.Add(new V1EnvVar($"HealthChecksUI__Webhooks__{i}__RestoredPayload", webhook.RestoredPayload));
}
if (resource.HasBrandingConfigured())
{
var volumeName = "healthchecks-volume";
if (specification.Volumes == null)
{
specification.Volumes = new List <V1Volume>();
}
if (container.VolumeMounts == null)
{
container.VolumeMounts = new List <V1VolumeMount>();
}
specification.Volumes.Add(new V1Volume(name: volumeName,
configMap: new V1ConfigMapVolumeSource(name: $"{resource.Spec.Name}-config")));
container.Env.Add(new V1EnvVar("ui_stylesheet", $"{Constants.StylesPath}/{Constants.StyleSheetName}"));
container.VolumeMounts.Add(new V1VolumeMount($"/app/{Constants.StylesPath}", volumeName));
}
return(new V1Deployment(metadata: metadata, spec: spec));
}
/// <summary>
/// Initializes a new <see cref="Broker"/>
/// </summary>
/// <param name="metadata">The <see cref="Broker"/>'s <see cref="V1ObjectMeta"/></param>
public Broker(V1ObjectMeta metadata)
: this(metadata, null, null)
{
}
V1PodTemplateSpec GetPodFromModule(Dictionary <string, string> labels, KubernetesModule <TConfig> module, IModuleIdentity moduleIdentity)
{
if (module is IModule <AgentDocker.CombinedDockerConfig> moduleWithDockerConfig)
{
// pod labels
var podLabels = new Dictionary <string, string>(labels);
// pod annotations
var podAnnotations = new Dictionary <string, string>();
podAnnotations.Add(Constants.K8sEdgeOriginalModuleId, moduleIdentity.ModuleId);
// Convert docker labels to annotations because docker labels don't have the same restrictions as
// Kuberenetes labels.
if (moduleWithDockerConfig.Config.CreateOptions?.Labels != null)
{
foreach (KeyValuePair <string, string> label in moduleWithDockerConfig.Config.CreateOptions?.Labels)
{
podAnnotations.Add(KubeUtils.SanitizeAnnotationKey(label.Key), label.Value);
}
}
// Per container settings:
// exposed ports
Option <List <V1ContainerPort> > exposedPortsOption = (moduleWithDockerConfig.Config?.CreateOptions?.ExposedPorts != null)
? this.GetExposedPorts(moduleWithDockerConfig.Config.CreateOptions.ExposedPorts).Map(
servicePorts =>
servicePorts.Select(tuple => new V1ContainerPort(tuple.Port, protocol: tuple.Protocol)).ToList())
: Option.None <List <V1ContainerPort> >();
// privileged container
Option <V1SecurityContext> securityContext = (moduleWithDockerConfig.Config?.CreateOptions?.HostConfig?.Privileged == true) ? Option.Some(new V1SecurityContext(privileged: true)) : Option.None <V1SecurityContext>();
// Environment Variables.
List <V1EnvVar> env = this.CollectEnv(moduleWithDockerConfig, (KubernetesModuleIdentity)moduleIdentity);
// Bind mounts
(List <V1Volume> volumeList, List <V1VolumeMount> proxyMounts, List <V1VolumeMount> volumeMountList) = this.GetVolumesFromModule(moduleWithDockerConfig).GetOrElse((null, null, null));
// Image
string moduleImage = moduleWithDockerConfig.Config.Image;
var containerList = new List <V1Container>()
{
new V1Container(
KubeUtils.SanitizeDNSValue(moduleIdentity.ModuleId),
env: env,
image: moduleImage,
volumeMounts: volumeMountList,
securityContext: securityContext.GetOrElse(() => null),
ports: exposedPortsOption.GetOrElse(() => null)),
// TODO: Add Proxy container here - configmap for proxy configuration.
new V1Container(
"proxy",
env: env, // TODO: check these for validity for proxy.
image: this.proxyImage,
volumeMounts: proxyMounts)
};
Option <List <V1LocalObjectReference> > imageSecret = moduleWithDockerConfig.Config.AuthConfig.Map(
auth =>
{
var secret = new ImagePullSecret(auth);
var authList = new List <V1LocalObjectReference>
{
new V1LocalObjectReference(secret.Name)
};
return(authList);
});
var modulePodSpec = new V1PodSpec(containerList, volumes: volumeList, imagePullSecrets: imageSecret.GetOrElse(() => null));
var objectMeta = new V1ObjectMeta(labels: podLabels, annotations: podAnnotations);
return(new V1PodTemplateSpec(metadata: objectMeta, spec: modulePodSpec));
}
else
{
Events.InvalidModuleType(module);
}
return(new V1PodTemplateSpec());
}
public Resource(string kind, V1ObjectMeta metadata, Func <string, Task <Rest.HttpOperationResponse <V1Status> > > deleter)
{
Kind = kind;
Metadata = metadata;
Deleter = deleter;
}
public async void CrdCommandExecuteWithAuthReplaceObjects()
{
string secretName = "username-docker.io";
var secretData = new Dictionary <string, byte[]> {
[Constants.K8sPullSecretData] = Encoding.UTF8.GetBytes("Invalid Secret Data")
};
var secretMeta = new V1ObjectMeta(name: secretName, namespaceProperty: Namespace);
IModule dockerModule = new DockerModule("module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars);
ModuleSet currentModules = ModuleSet.Create(dockerModule);
var dockerConfigProvider = new Mock <ICombinedConfigProvider <CombinedDockerConfig> >();
dockerConfigProvider.Setup(cp => cp.GetCombinedConfig(dockerModule, Runtime))
.Returns(() => new CombinedDockerConfig("test-image:1", Config1.CreateOptions, Option.Maybe(DockerAuth)));
var configProvider = new Mock <ICombinedConfigProvider <CombinedKubernetesConfig> >();
configProvider.Setup(cp => cp.GetCombinedConfig(dockerModule, Runtime))
.Returns(() => new CombinedKubernetesConfig("test-image:1", CreatePodParameters.Create(image: "test-image:1"), Option.Maybe(ImagePullSecret)));
var existingSecret = new V1Secret("v1", secretData, type: Constants.K8sPullSecretType, kind: "Secret", metadata: secretMeta);
var existingDeployment = new EdgeDeploymentDefinition(Constants.EdgeDeployment.ApiVersion, Constants.EdgeDeployment.Kind, new V1ObjectMeta(name: ResourceName), new List <KubernetesModule>());
bool getSecretCalled = false;
bool putSecretCalled = false;
bool getCrdCalled = false;
bool putCrdCalled = false;
using (var server = new KubernetesApiServer(
resp: string.Empty,
shouldNext: async httpContext =>
{
string pathStr = httpContext.Request.Path.Value;
string method = httpContext.Request.Method;
if (string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase))
{
if (pathStr.Contains($"api/v1/namespaces/{Namespace}/secrets/{secretName}"))
{
getSecretCalled = true;
await httpContext.Response.Body.WriteAsync(JsonConvert.SerializeObject(existingSecret).ToBody());
}
else if (pathStr.Contains($"namespaces/{Namespace}/{Constants.EdgeDeployment.Plural}/{ResourceName}"))
{
getCrdCalled = true;
await httpContext.Response.Body.WriteAsync(JsonConvert.SerializeObject(existingDeployment).ToBody());
}
}
else if (string.Equals(method, "PUT", StringComparison.OrdinalIgnoreCase))
{
httpContext.Response.Body = httpContext.Request.Body;
if (pathStr.Contains($"api/v1/namespaces/{Namespace}/secrets/{secretName}"))
{
putSecretCalled = true;
}
else if (pathStr.Contains($"namespaces/{Namespace}/{Constants.EdgeDeployment.Plural}/{ResourceName}"))
{
putCrdCalled = true;
}
}
return(false);
}))
{
var client = new Kubernetes(new KubernetesClientConfiguration {
Host = server.Uri
});
var cmd = new EdgeDeploymentCommand(Namespace, ResourceName, client, new[] { dockerModule }, currentModules, Runtime, configProvider.Object);
await cmd.ExecuteAsync(CancellationToken.None);
Assert.True(getSecretCalled, nameof(getSecretCalled));
Assert.True(putSecretCalled, nameof(putSecretCalled));
Assert.True(getCrdCalled, nameof(getCrdCalled));
Assert.True(putCrdCalled, nameof(putCrdCalled));
}
}
/// <summary>
/// Initializes a new <see cref="VirtualService"/>
/// </summary>
/// <param name="metadata">The <see cref="VirtualService"/>'s <see cref="V1ObjectMeta"/></param>
/// <param name="spec">The <see cref="VirtualService"/>'s <see cref="VirtualServiceSpec"/></param>
public VirtualService(V1ObjectMeta metadata, VirtualServiceSpec spec)
: this()
{
this.Metadata = metadata;
this.Spec = spec;
}
