public async Task StartChaosResources(V1Pod pod) { var chaos = await ChaosClient.ListNamespacedAsync(pod.Namespace()); var tasks = chaos.Items .Where(cr => ShouldStartChaos(cr, pod)) .Select(async cr => { string plural = ""; foreach (string pluralName in Enum.GetNames(typeof(GenericChaosClient.ChaosResourcePlurals))) { if (pluralName.Contains(cr.Kind.ToLower())) { plural = pluralName; break; } } await Client.PatchNamespacedCustomObjectWithHttpMessagesAsync( PodChaosResumePatchBody, ChaosClient.Group, ChaosClient.Version, pod.Namespace(), plural, cr.Metadata.Name); using (LogContext.PushProperty("chaosResource", $"{cr.Kind}/{cr.Metadata.Name}")) { Logger.Information($"Started chaos for pod."); } }); await Task.WhenAll(tasks); }
public static void CopyFileToPod(this Kubernetes client, V1Pod pod, string container, string sourceFilePath, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken)) { if (!IsKubectlInstalled()) { throw new Exception($"kubectl is either not installed or is not on path\n{kubectlInstallLink}"); } if (!(File.Exists(sourceFilePath) || Directory.Exists(sourceFilePath))) { throw new FileNotFoundException($"File {sourceFilePath} does not exist"); } Process proc = new Process(); proc.StartInfo.FileName = "kubectl"; proc.StartInfo.Arguments = $"cp {sourceFilePath} {pod.Namespace()}/{pod.Name()}:{destinationFilePath} -c {container}"; proc.StartInfo.RedirectStandardOutput = true; proc.StartInfo.RedirectStandardError = true; proc.Start(); proc.WaitForExit(); if (proc.ExitCode != 0) { string stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); throw new Exception($"Failed to copy {sourceFilePath} to pod {pod.Name()};\nstdout:\n{stdout}\nstderr:\n{stderr}"); } }
public async Task StartChaosResources(V1Pod pod) { var chaos = await ChaosClient.ListNamespacedAsync(pod.Namespace()); var tasks = chaos.Items .Where(cr => ShouldStartChaos(cr, pod)) .Select(async cr => { await Client.PatchNamespacedCustomObjectWithHttpMessagesAsync( PodChaosResumePatchBody, ChaosClient.Group, ChaosClient.Version, pod.Namespace(), cr.Kind.ToLower(), cr.Metadata.Name); using (LogContext.PushProperty("chaosResource", $"{cr.Kind}/{cr.Metadata.Name}")) { Logger.Information($"Started chaos for pod."); } }); await Task.WhenAll(tasks); }
public bool ShouldStartPodChaos(WatchEventType type, V1Pod pod) { if (!string.IsNullOrEmpty(Namespace) && Namespace != pod.Namespace()) { return(false); } if ((type != WatchEventType.Added && type != WatchEventType.Modified) || pod.Status.Phase != "Running") { return(false); } var autoStart = ""; pod.Metadata.Annotations?.TryGetValue("stress/chaos.autoStart", out autoStart); if (autoStart == "false") { return(false); } if (!pod.Metadata.Labels.TryGetValue("chaos", out var chaos) || chaos != "true") { return(false); } if (String.IsNullOrEmpty(pod.TestInstance())) { Logger.Information($"Pod has 'chaos' label but missing or empty {GenericChaosResource.TestInstanceLabelKey} label."); return(false); } var started = ""; pod.Metadata.Annotations?.TryGetValue("stress/chaos.started", out started); if (started == "true") { Logger.Information($"Pod is in chaos.started state."); return(false); } return(true); }
public void HandlePodEvent(WatchEventType type, V1Pod pod) { using (LogContext.PushProperty("namespace", pod.Namespace())) using (LogContext.PushProperty("pod", pod.Name())) { ResumeChaos(type, pod).ContinueWith(t => { if (t.Exception != null) { // TODO: handle watch event re-queue on failure Logger.Error(t.Exception, "Error handling pod event."); } }); DeleteResources(pod).ContinueWith(t => { if (t.Exception != null) { Logger.Error(t.Exception, "Error deleting resources."); } }); } }
public bool ShouldDeleteResources(V1Pod pod) { if (!string.IsNullOrEmpty(Namespace) && Namespace != pod.Namespace()) { return(false); } var initContainers = pod.Spec?.InitContainers; if (initContainers == null || initContainers.Count() == 0) { return(false); } var deployContainers = initContainers.Where(c => c.Name == "init-azure-deployer"); if (deployContainers.Count() == 0) { return(false); } return((pod.Status.Phase == "Succeeded") || (pod.Status.Phase == "Failed")); }
/// <summary> /// Returns the owner identification for a pod. /// </summary> /// <param name="k8s">The Kubernetes client.</param> /// <param name="pod">The pod.</param> /// <returns>The owner ID.</returns> private static async Task <string> GetOwnerIdAsync(IKubernetes k8s, V1Pod pod) { Covenant.Requires <ArgumentNullException>(k8s != null, nameof(k8s)); Covenant.Requires <ArgumentNullException>(pod != null, nameof(pod)); // We're going to favor standard top-level owners, when present. string ownerName = null; string ownerKind = null; foreach (var owner in pod.OwnerReferences()) { switch (owner.Kind) { case "DaemonSet": case "Deployment": case "StatefulSet": ownerName = owner.Name; ownerKind = owner.Kind; break; case "Node": // We'll see this for static pods. [owner.Name] is the node name which isn't terribly useful, // so we'll used the pod name instead, removing the node name part, which will look something // like: // // kube-scheduler-master-0 var nodeNamePos = pod.Metadata.Name.IndexOf(owner.Name) - 1; ownerName = pod.Metadata.Name.Substring(0, nodeNamePos); ownerKind = owner.Kind; break; case "ReplicaSet": // Use the replica set's owner when present. var replicaSet = await k8s.ReadNamespacedReplicaSetAsync(owner.Name, pod.Namespace()); var replicaSetOwner = replicaSet.OwnerReferences().FirstOrDefault(); if (replicaSetOwner != null) { ownerName = replicaSetOwner.Name; ownerKind = replicaSetOwner.Kind; } else { ownerName = owner.Name; ownerKind = owner.Kind; } break; } } var podNamespace = pod.Namespace(); if (!string.IsNullOrEmpty(ownerName)) { return($"{podNamespace}/{ownerName} ({ownerKind})"); } // Default to using the pod name or kind for standalone pods. return($"{podNamespace}/{pod.Name} ({pod.Kind})"); }
public void TestMetadata() { // test getters on null metadata var pod = new V1Pod(); Assert.Null(pod.Annotations()); Assert.Null(pod.ApiGroup()); var(g, v) = pod.ApiGroupAndVersion(); Assert.Null(g); Assert.Null(v); Assert.Null(pod.ApiGroupVersion()); Assert.Null(pod.CreationTimestamp()); Assert.Null(pod.DeletionTimestamp()); Assert.Null(pod.Finalizers()); Assert.Equal(-1, pod.FindOwnerReference(r => true)); Assert.Null(pod.Generation()); Assert.Null(pod.GetAnnotation("x")); Assert.Null(pod.GetController()); Assert.Null(pod.GetLabel("x")); Assert.Null(pod.GetOwnerReference(r => true)); Assert.False(pod.HasFinalizer("x")); Assert.Null(pod.Labels()); Assert.Null(pod.Name()); Assert.Null(pod.Namespace()); Assert.Null(pod.OwnerReferences()); Assert.Null(pod.ResourceVersion()); Assert.Null(pod.Uid()); Assert.Null(pod.Metadata); // test API version stuff pod = new V1Pod() { ApiVersion = "v1" }; Assert.Equal("", pod.ApiGroup()); (g, v) = pod.ApiGroupAndVersion(); Assert.Equal("", g); Assert.Equal("v1", v); Assert.Equal("v1", pod.ApiGroupVersion()); pod.ApiVersion = "abc/v2"; Assert.Equal("abc", pod.ApiGroup()); (g, v) = pod.ApiGroupAndVersion(); Assert.Equal("abc", g); Assert.Equal("v2", v); Assert.Equal("v2", pod.ApiGroupVersion()); // test the Ensure*() functions Assert.NotNull(pod.EnsureMetadata()); Assert.NotNull(pod.Metadata); Assert.NotNull(pod.Metadata.EnsureAnnotations()); Assert.NotNull(pod.Metadata.Annotations); Assert.NotNull(pod.Metadata.EnsureFinalizers()); Assert.NotNull(pod.Metadata.Finalizers); Assert.NotNull(pod.Metadata.EnsureLabels()); Assert.NotNull(pod.Metadata.Labels); // test getters with non-null values DateTime ts = DateTime.UtcNow, ts2 = DateTime.Now; pod.Metadata = new V1ObjectMeta() { CreationTimestamp = ts, DeletionTimestamp = ts2, Generation = 1, Name = "name", NamespaceProperty = "ns", ResourceVersion = "42", Uid = "id" }; Assert.Equal(ts, pod.CreationTimestamp().Value); Assert.Equal(ts2, pod.DeletionTimestamp().Value); Assert.Equal(1, pod.Generation().Value); Assert.Equal("name", pod.Name()); Assert.Equal("ns", pod.Namespace()); Assert.Equal("42", pod.ResourceVersion()); Assert.Equal("id", pod.Uid()); // test annotations and labels pod.SetAnnotation("x", "y"); pod.SetLabel("a", "b"); Assert.Equal(1, pod.Annotations().Count); Assert.Equal(1, pod.Labels().Count); Assert.Equal("y", pod.GetAnnotation("x")); Assert.Equal("y", pod.Metadata.Annotations["x"]); Assert.Null(pod.GetAnnotation("a")); Assert.Equal("b", pod.GetLabel("a")); Assert.Equal("b", pod.Metadata.Labels["a"]); Assert.Null(pod.GetLabel("x")); pod.SetAnnotation("x", null); Assert.Equal(0, pod.Annotations().Count); pod.SetLabel("a", null); Assert.Equal(0, pod.Labels().Count); // test finalizers Assert.False(pod.HasFinalizer("abc")); Assert.True(pod.AddFinalizer("abc")); Assert.True(pod.HasFinalizer("abc")); Assert.False(pod.AddFinalizer("abc")); Assert.False(pod.HasFinalizer("xyz")); Assert.False(pod.RemoveFinalizer("xyz")); Assert.True(pod.RemoveFinalizer("abc")); Assert.False(pod.HasFinalizer("abc")); Assert.False(pod.RemoveFinalizer("abc")); }
public void TestReferences() { // test object references var pod = new V1Pod() { ApiVersion = "v1", Kind = "Pod" }; pod.Metadata = new V1ObjectMeta() { Name = "name", NamespaceProperty = "ns", ResourceVersion = "ver", Uid = "id" }; var objr = new V1ObjectReference() { ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), NamespaceProperty = pod.Namespace(), Uid = pod.Uid() }; Assert.True(objr.Matches(pod)); (pod.ApiVersion, pod.Kind) = (null, null); Assert.False(objr.Matches(pod)); (pod.ApiVersion, pod.Kind) = ("v1", "Pod"); Assert.True(objr.Matches(pod)); pod.Metadata.Name = "nome"; Assert.False(objr.Matches(pod)); // test owner references (pod.ApiVersion, pod.Kind) = ("abc/xyz", "sometimes"); var ownr = new V1OwnerReference() { ApiVersion = "abc/xyz", Kind = "sometimes", Name = pod.Name(), Uid = pod.Uid() }; Assert.True(ownr.Matches(pod)); (pod.ApiVersion, pod.Kind) = (null, null); Assert.False(ownr.Matches(pod)); (ownr.ApiVersion, ownr.Kind) = ("v1", "Pod"); Assert.False(ownr.Matches(pod)); (pod.ApiVersion, pod.Kind) = (ownr.ApiVersion, ownr.Kind); Assert.True(ownr.Matches(pod)); ownr.Name = "nim"; Assert.False(ownr.Matches(pod)); ownr.Name = pod.Name(); var svc = new V1Service(); svc.AddOwnerReference(ownr); Assert.Equal(0, svc.FindOwnerReference(pod)); Assert.Equal(-1, svc.FindOwnerReference(svc)); Assert.Same(ownr, svc.GetOwnerReference(pod)); Assert.Null(svc.GetOwnerReference(svc)); Assert.Null(svc.GetController()); svc.OwnerReferences()[0].Controller = true; Assert.Same(ownr, svc.GetController()); Assert.Same(ownr, svc.RemoveOwnerReference(pod)); Assert.Equal(0, svc.OwnerReferences().Count); svc.AddOwnerReference(new V1OwnerReference() { ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), Uid = pod.Uid(), Controller = true }); svc.AddOwnerReference(new V1OwnerReference() { ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), Uid = pod.Uid(), Controller = false }); svc.AddOwnerReference(new V1OwnerReference() { ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), Uid = pod.Uid() }); Assert.Equal(3, svc.OwnerReferences().Count); Assert.NotNull(svc.RemoveOwnerReference(pod)); Assert.Equal(2, svc.OwnerReferences().Count); Assert.True(svc.RemoveOwnerReferences(pod)); Assert.Equal(0, svc.OwnerReferences().Count); }
public async Task ResumeChaos(WatchEventType type, V1Pod pod) { if (!ShouldStartPodChaos(type, pod)) { Logger.Debug("Skipping pod."); return; } await StartChaosResources(pod); Logger.Information($"Started chaos resources for pod"); await Client.PatchNamespacedPodAsync(PodChaosHandledPatchBody, pod.Name(), pod.Namespace()); Logger.Information($"Annotated pod chaos started"); }
public static string NamespacedName(this V1Pod pod) => $"{pod.Namespace()}/{pod.Name()}";