public async Task StartChaosResources(V1Pod pod)
{
var chaos = await ChaosClient.ListNamespacedAsync(pod.Namespace());
var tasks = chaos.Items
.Where(cr => ShouldStartChaos(cr, pod))
.Select(async cr =>
{
string plural = "";
foreach (string pluralName in Enum.GetNames(typeof(GenericChaosClient.ChaosResourcePlurals)))
{
if (pluralName.Contains(cr.Kind.ToLower()))
{
plural = pluralName;
break;
}
}
await Client.PatchNamespacedCustomObjectWithHttpMessagesAsync(
PodChaosResumePatchBody, ChaosClient.Group, ChaosClient.Version,
pod.Namespace(), plural, cr.Metadata.Name);
using (LogContext.PushProperty("chaosResource", $"{cr.Kind}/{cr.Metadata.Name}"))
{
Logger.Information($"Started chaos for pod.");
}
});
await Task.WhenAll(tasks);
}
public static void CopyFileToPod(this Kubernetes client, V1Pod pod, string container, string sourceFilePath, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken))
{
if (!IsKubectlInstalled())
{
throw new Exception($"kubectl is either not installed or is not on path\n{kubectlInstallLink}");
}
if (!(File.Exists(sourceFilePath) || Directory.Exists(sourceFilePath)))
{
throw new FileNotFoundException($"File {sourceFilePath} does not exist");
}
Process proc = new Process();
proc.StartInfo.FileName = "kubectl";
proc.StartInfo.Arguments = $"cp {sourceFilePath} {pod.Namespace()}/{pod.Name()}:{destinationFilePath} -c {container}";
proc.StartInfo.RedirectStandardOutput = true;
proc.StartInfo.RedirectStandardError = true;
proc.Start();
proc.WaitForExit();
if (proc.ExitCode != 0)
{
string stdout = proc.StandardOutput.ReadToEnd();
string stderr = proc.StandardError.ReadToEnd();
throw new Exception($"Failed to copy {sourceFilePath} to pod {pod.Name()};\nstdout:\n{stdout}\nstderr:\n{stderr}");
}
}
public async Task StartChaosResources(V1Pod pod)
{
var chaos = await ChaosClient.ListNamespacedAsync(pod.Namespace());
var tasks = chaos.Items
.Where(cr => ShouldStartChaos(cr, pod))
.Select(async cr =>
{
await Client.PatchNamespacedCustomObjectWithHttpMessagesAsync(
PodChaosResumePatchBody, ChaosClient.Group, ChaosClient.Version,
pod.Namespace(), cr.Kind.ToLower(), cr.Metadata.Name);
using (LogContext.PushProperty("chaosResource", $"{cr.Kind}/{cr.Metadata.Name}"))
{
Logger.Information($"Started chaos for pod.");
}
});
await Task.WhenAll(tasks);
}
public bool ShouldStartPodChaos(WatchEventType type, V1Pod pod)
{
if (!string.IsNullOrEmpty(Namespace) && Namespace != pod.Namespace())
{
return(false);
}
if ((type != WatchEventType.Added && type != WatchEventType.Modified) ||
pod.Status.Phase != "Running")
{
return(false);
}
var autoStart = "";
pod.Metadata.Annotations?.TryGetValue("stress/chaos.autoStart", out autoStart);
if (autoStart == "false")
{
return(false);
}
if (!pod.Metadata.Labels.TryGetValue("chaos", out var chaos) || chaos != "true")
{
return(false);
}
if (String.IsNullOrEmpty(pod.TestInstance()))
{
Logger.Information($"Pod has 'chaos' label but missing or empty {GenericChaosResource.TestInstanceLabelKey} label.");
return(false);
}
var started = "";
pod.Metadata.Annotations?.TryGetValue("stress/chaos.started", out started);
if (started == "true")
{
Logger.Information($"Pod is in chaos.started state.");
return(false);
}
return(true);
}
public void HandlePodEvent(WatchEventType type, V1Pod pod)
{
using (LogContext.PushProperty("namespace", pod.Namespace()))
using (LogContext.PushProperty("pod", pod.Name()))
{
ResumeChaos(type, pod).ContinueWith(t =>
{
if (t.Exception != null)
{
// TODO: handle watch event re-queue on failure
Logger.Error(t.Exception, "Error handling pod event.");
}
});
DeleteResources(pod).ContinueWith(t =>
{
if (t.Exception != null)
{
Logger.Error(t.Exception, "Error deleting resources.");
}
});
}
}
public bool ShouldDeleteResources(V1Pod pod)
{
if (!string.IsNullOrEmpty(Namespace) && Namespace != pod.Namespace())
{
return(false);
}
var initContainers = pod.Spec?.InitContainers;
if (initContainers == null || initContainers.Count() == 0)
{
return(false);
}
var deployContainers = initContainers.Where(c => c.Name == "init-azure-deployer");
if (deployContainers.Count() == 0)
{
return(false);
}
return((pod.Status.Phase == "Succeeded") || (pod.Status.Phase == "Failed"));
}
/// <summary>
/// Returns the owner identification for a pod.
/// </summary>
/// <param name="k8s">The Kubernetes client.</param>
/// <param name="pod">The pod.</param>
/// <returns>The owner ID.</returns>
private static async Task <string> GetOwnerIdAsync(IKubernetes k8s, V1Pod pod)
{
Covenant.Requires <ArgumentNullException>(k8s != null, nameof(k8s));
Covenant.Requires <ArgumentNullException>(pod != null, nameof(pod));
// We're going to favor standard top-level owners, when present.
string ownerName = null;
string ownerKind = null;
foreach (var owner in pod.OwnerReferences())
{
switch (owner.Kind)
{
case "DaemonSet":
case "Deployment":
case "StatefulSet":
ownerName = owner.Name;
ownerKind = owner.Kind;
break;
case "Node":
// We'll see this for static pods. [owner.Name] is the node name which isn't terribly useful,
// so we'll used the pod name instead, removing the node name part, which will look something
// like:
//
// kube-scheduler-master-0
var nodeNamePos = pod.Metadata.Name.IndexOf(owner.Name) - 1;
ownerName = pod.Metadata.Name.Substring(0, nodeNamePos);
ownerKind = owner.Kind;
break;
case "ReplicaSet":
// Use the replica set's owner when present.
var replicaSet = await k8s.ReadNamespacedReplicaSetAsync(owner.Name, pod.Namespace());
var replicaSetOwner = replicaSet.OwnerReferences().FirstOrDefault();
if (replicaSetOwner != null)
{
ownerName = replicaSetOwner.Name;
ownerKind = replicaSetOwner.Kind;
}
else
{
ownerName = owner.Name;
ownerKind = owner.Kind;
}
break;
}
}
var podNamespace = pod.Namespace();
if (!string.IsNullOrEmpty(ownerName))
{
return($"{podNamespace}/{ownerName} ({ownerKind})");
}
// Default to using the pod name or kind for standalone pods.
return($"{podNamespace}/{pod.Name} ({pod.Kind})");
}
public void TestMetadata()
{
// test getters on null metadata
var pod = new V1Pod();
Assert.Null(pod.Annotations());
Assert.Null(pod.ApiGroup());
var(g, v) = pod.ApiGroupAndVersion();
Assert.Null(g);
Assert.Null(v);
Assert.Null(pod.ApiGroupVersion());
Assert.Null(pod.CreationTimestamp());
Assert.Null(pod.DeletionTimestamp());
Assert.Null(pod.Finalizers());
Assert.Equal(-1, pod.FindOwnerReference(r => true));
Assert.Null(pod.Generation());
Assert.Null(pod.GetAnnotation("x"));
Assert.Null(pod.GetController());
Assert.Null(pod.GetLabel("x"));
Assert.Null(pod.GetOwnerReference(r => true));
Assert.False(pod.HasFinalizer("x"));
Assert.Null(pod.Labels());
Assert.Null(pod.Name());
Assert.Null(pod.Namespace());
Assert.Null(pod.OwnerReferences());
Assert.Null(pod.ResourceVersion());
Assert.Null(pod.Uid());
Assert.Null(pod.Metadata);
// test API version stuff
pod = new V1Pod()
{
ApiVersion = "v1"
};
Assert.Equal("", pod.ApiGroup());
(g, v) = pod.ApiGroupAndVersion();
Assert.Equal("", g);
Assert.Equal("v1", v);
Assert.Equal("v1", pod.ApiGroupVersion());
pod.ApiVersion = "abc/v2";
Assert.Equal("abc", pod.ApiGroup());
(g, v) = pod.ApiGroupAndVersion();
Assert.Equal("abc", g);
Assert.Equal("v2", v);
Assert.Equal("v2", pod.ApiGroupVersion());
// test the Ensure*() functions
Assert.NotNull(pod.EnsureMetadata());
Assert.NotNull(pod.Metadata);
Assert.NotNull(pod.Metadata.EnsureAnnotations());
Assert.NotNull(pod.Metadata.Annotations);
Assert.NotNull(pod.Metadata.EnsureFinalizers());
Assert.NotNull(pod.Metadata.Finalizers);
Assert.NotNull(pod.Metadata.EnsureLabels());
Assert.NotNull(pod.Metadata.Labels);
// test getters with non-null values
DateTime ts = DateTime.UtcNow, ts2 = DateTime.Now;
pod.Metadata = new V1ObjectMeta()
{
CreationTimestamp = ts,
DeletionTimestamp = ts2,
Generation = 1,
Name = "name",
NamespaceProperty = "ns",
ResourceVersion = "42",
Uid = "id"
};
Assert.Equal(ts, pod.CreationTimestamp().Value);
Assert.Equal(ts2, pod.DeletionTimestamp().Value);
Assert.Equal(1, pod.Generation().Value);
Assert.Equal("name", pod.Name());
Assert.Equal("ns", pod.Namespace());
Assert.Equal("42", pod.ResourceVersion());
Assert.Equal("id", pod.Uid());
// test annotations and labels
pod.SetAnnotation("x", "y");
pod.SetLabel("a", "b");
Assert.Equal(1, pod.Annotations().Count);
Assert.Equal(1, pod.Labels().Count);
Assert.Equal("y", pod.GetAnnotation("x"));
Assert.Equal("y", pod.Metadata.Annotations["x"]);
Assert.Null(pod.GetAnnotation("a"));
Assert.Equal("b", pod.GetLabel("a"));
Assert.Equal("b", pod.Metadata.Labels["a"]);
Assert.Null(pod.GetLabel("x"));
pod.SetAnnotation("x", null);
Assert.Equal(0, pod.Annotations().Count);
pod.SetLabel("a", null);
Assert.Equal(0, pod.Labels().Count);
// test finalizers
Assert.False(pod.HasFinalizer("abc"));
Assert.True(pod.AddFinalizer("abc"));
Assert.True(pod.HasFinalizer("abc"));
Assert.False(pod.AddFinalizer("abc"));
Assert.False(pod.HasFinalizer("xyz"));
Assert.False(pod.RemoveFinalizer("xyz"));
Assert.True(pod.RemoveFinalizer("abc"));
Assert.False(pod.HasFinalizer("abc"));
Assert.False(pod.RemoveFinalizer("abc"));
}
public void TestReferences()
{
// test object references
var pod = new V1Pod()
{
ApiVersion = "v1", Kind = "Pod"
};
pod.Metadata = new V1ObjectMeta()
{
Name = "name", NamespaceProperty = "ns", ResourceVersion = "ver", Uid = "id"
};
var objr = new V1ObjectReference()
{
ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), NamespaceProperty = pod.Namespace(), Uid = pod.Uid()
};
Assert.True(objr.Matches(pod));
(pod.ApiVersion, pod.Kind) = (null, null);
Assert.False(objr.Matches(pod));
(pod.ApiVersion, pod.Kind) = ("v1", "Pod");
Assert.True(objr.Matches(pod));
pod.Metadata.Name = "nome";
Assert.False(objr.Matches(pod));
// test owner references
(pod.ApiVersion, pod.Kind) = ("abc/xyz", "sometimes");
var ownr = new V1OwnerReference()
{
ApiVersion = "abc/xyz", Kind = "sometimes", Name = pod.Name(), Uid = pod.Uid()
};
Assert.True(ownr.Matches(pod));
(pod.ApiVersion, pod.Kind) = (null, null);
Assert.False(ownr.Matches(pod));
(ownr.ApiVersion, ownr.Kind) = ("v1", "Pod");
Assert.False(ownr.Matches(pod));
(pod.ApiVersion, pod.Kind) = (ownr.ApiVersion, ownr.Kind);
Assert.True(ownr.Matches(pod));
ownr.Name = "nim";
Assert.False(ownr.Matches(pod));
ownr.Name = pod.Name();
var svc = new V1Service();
svc.AddOwnerReference(ownr);
Assert.Equal(0, svc.FindOwnerReference(pod));
Assert.Equal(-1, svc.FindOwnerReference(svc));
Assert.Same(ownr, svc.GetOwnerReference(pod));
Assert.Null(svc.GetOwnerReference(svc));
Assert.Null(svc.GetController());
svc.OwnerReferences()[0].Controller = true;
Assert.Same(ownr, svc.GetController());
Assert.Same(ownr, svc.RemoveOwnerReference(pod));
Assert.Equal(0, svc.OwnerReferences().Count);
svc.AddOwnerReference(new V1OwnerReference()
{
ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), Uid = pod.Uid(), Controller = true
});
svc.AddOwnerReference(new V1OwnerReference()
{
ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), Uid = pod.Uid(), Controller = false
});
svc.AddOwnerReference(new V1OwnerReference()
{
ApiVersion = pod.ApiVersion, Kind = pod.Kind, Name = pod.Name(), Uid = pod.Uid()
});
Assert.Equal(3, svc.OwnerReferences().Count);
Assert.NotNull(svc.RemoveOwnerReference(pod));
Assert.Equal(2, svc.OwnerReferences().Count);
Assert.True(svc.RemoveOwnerReferences(pod));
Assert.Equal(0, svc.OwnerReferences().Count);
}
public async Task ResumeChaos(WatchEventType type, V1Pod pod)
{
if (!ShouldStartPodChaos(type, pod))
{
Logger.Debug("Skipping pod.");
return;
}
await StartChaosResources(pod);
Logger.Information($"Started chaos resources for pod");
await Client.PatchNamespacedPodAsync(PodChaosHandledPatchBody, pod.Name(), pod.Namespace());
Logger.Information($"Annotated pod chaos started");
}
public static string NamespacedName(this V1Pod pod) => $"{pod.Namespace()}/{pod.Name()}";
