Exemple #1
0
        public async Task ConvertsPodsToModules()
        {
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            foreach (V1Pod pod in BuildPodList().Values)
            {
                runtimeInfo.CreateOrUpdateAddPodInfo(pod);
            }

            var modules = (await runtimeInfo.GetModules(CancellationToken.None)).ToList();

            Assert.Equal(3, modules.Count);
            foreach (var module in modules)
            {
                Assert.Contains("Started", module.Description);
                Assert.Equal(ModuleStatus.Running, module.ModuleStatus);
                Assert.Equal(new DateTime(2019, 6, 12), module.StartTime.GetOrElse(DateTime.MinValue).Date);
                Assert.Equal("docker", module.Type);
                if (module is ModuleRuntimeInfo <DockerReportedConfig> config)
                {
                    Assert.NotEqual("unknown:unknown", config.Config.Image);
                }
            }
        }
Exemple #2
0
        public async Task ReturnsModuleRuntimeInfoWhenPodsAreUpdated()
        {
            V1Pod edgeagent1 = BuildPodList()["edgeagent"];

            edgeagent1.Metadata.Name = "edgeagent_123";
            edgeagent1.Status.ContainerStatuses
            .First(c => c.Name == "edgeagent").State.Running.StartedAt = new DateTime(2019, 10, 28);
            V1Pod edgeagent2 = BuildPodList()["edgeagent"];

            edgeagent2.Metadata.Name = "edgeAgent_456";
            edgeagent2.Status.ContainerStatuses
            .First(c => c.Name == "edgeagent").State.Running.StartedAt = new DateTime(2019, 10, 29);
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            runtimeInfo.CreateOrUpdateAddPodInfo(edgeagent1);
            runtimeInfo.CreateOrUpdateAddPodInfo(edgeagent2);
            runtimeInfo.RemovePodInfo(edgeagent1);

            var modules = await runtimeInfo.GetModules(CancellationToken.None);

            var info = modules.Single();

            Assert.NotNull(info);
            Assert.Equal(info.StartTime, Option.Some(new DateTime(2019, 10, 29)));
        }
        public async Task ReturnModuleStatusWithPodConditionsWhenThereAreNoContainers()
        {
            var   client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var   moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var   runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);
            V1Pod pod           = CreatePodWithPodParametersOnly("Pending", string.Empty, string.Empty);

            pod.Status.Conditions = new List <V1PodCondition>()
            {
                new V1PodCondition
                {
                    LastTransitionTime = new DateTime(2020, 02, 05, 10, 10, 10),
                    Message            = "Ready",
                    Reason             = "Scheduling",
                },
                new V1PodCondition
                {
                    LastTransitionTime = new DateTime(2020, 02, 05, 10, 10, 15),
                    Message            = "persistentvolumeclaim module-a-pvc not found",
                    Reason             = "Unschedulable",
                }
            };
            runtimeInfo.CreateOrUpdateAddPodInfo(pod);
            string expectedDescription = "Module Failed with container status Unknown More Info: persistentvolumeclaim module-a-pvc not found K8s reason: Unschedulable";

            ModuleRuntimeInfo info = (await runtimeInfo.GetModules(CancellationToken.None)).Single();

            Assert.Equal(ModuleStatus.Failed, info.ModuleStatus);
            Assert.Equal(expectedDescription, info.Description);
        }
Exemple #4
0
        public async void PodWatchSuccessTest()
        {
            AsyncCountdownEvent   requestHandled = new AsyncCountdownEvent(3);
            AsyncManualResetEvent serverShutdown = new AsyncManualResetEvent();

            var podWatchData = await File.ReadAllTextAsync("podwatch.txt");

            var addedAgent  = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added);
            var addedHub    = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added, 1);
            var addedSensor = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added, 3);

            using (var server = new MockKubeApiServer(
                       async httpContext =>
            {
                await MockKubeApiServer.WriteStreamLine(httpContext, addedAgent);
                await MockKubeApiServer.WriteStreamLine(httpContext, addedHub);
                await MockKubeApiServer.WriteStreamLine(httpContext, addedSensor);
                return(false);
            }))
            {
                var client = new Kubernetes(
                    new KubernetesClientConfiguration
                {
                    Host = server.Uri.ToString()
                });

                var k8sRuntimeInfo = new KubernetesRuntimeInfoProvider(PodwatchNamespace, client);

                k8sRuntimeInfo.PropertyChanged += (sender, args) =>
                {
                    Assert.Equal("Modules", args.PropertyName);
                    requestHandled.Signal();
                };

                k8sRuntimeInfo.Start();

                await Task.WhenAny(requestHandled.WaitAsync(), Task.Delay(TestTimeout));

                var runtimeModules = await k8sRuntimeInfo.GetModules(CancellationToken.None);

                var moduleRuntimeInfos = runtimeModules as ModuleRuntimeInfo[] ?? runtimeModules.ToArray();
                Assert.True(moduleRuntimeInfos.Count() == 3);
                Dictionary <string, int> uniqueModules = new Dictionary <string, int>();
                foreach (var i in moduleRuntimeInfos)
                {
                    uniqueModules[i.Name] = 1;
                    Assert.Contains("Started", i.Description);
                    Assert.Equal(ModuleStatus.Running, i.ModuleStatus);
                    Assert.Equal(new DateTime(2019, 6, 12), i.StartTime.GetOrElse(DateTime.MinValue).Date);
                    Assert.Equal("docker", i.Type);
                    if (i is ModuleRuntimeInfo <DockerReportedConfig> d)
                    {
                        Assert.NotEqual("unknown:unknown", d.Config.Image);
                    }
                }

                Assert.Equal(3, uniqueModules.Count);
            }
        }
Exemple #5
0
        public async void GetModuleLogsTest()
        {
            var client         = new Mock <IKubernetes>(MockBehavior.Strict);
            var k8sRuntimeInfo = new KubernetesRuntimeInfoProvider(PodwatchNamespace, client.Object);
            var result         = await k8sRuntimeInfo.GetModuleLogs("module", true, Option.None <int>(), Option.None <int>(), CancellationToken.None);

            Assert.True(result.Length == 0);
        }
Exemple #6
0
        public KubernetesEnvironmentOperatorTest(KubernetesClusterFixture fixture)
        {
            string deviceNamespace = $"device-{Guid.NewGuid()}";

            this.client = new KubernetesClient(deviceNamespace, fixture.Client);

            this.runtimeInfoProvider = new KubernetesRuntimeInfoProvider(deviceNamespace, fixture.Client, new DummyModuleManager());
            this.environmentOperator = new KubernetesEnvironmentOperator(deviceNamespace, this.runtimeInfoProvider, fixture.Client);
        }
Exemple #7
0
        public async Task ReturnsEmptyModulesWhenNoDataAvailable()
        {
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            var modules = await runtimeInfo.GetModules(CancellationToken.None);

            Assert.Empty(modules);
        }
        public async Task ReturnsLastKnowModuleState()
        {
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            foreach ((string podName, var pod) in BuildPodList())
            {
                runtimeInfo.CreateOrUpdateAddPodInfo(podName, pod);
            }

            Dictionary <string, V1Pod> modified = BuildPodList();

            modified["edgeagent"].Status.ContainerStatuses[0].State.Running    = null;
            modified["edgeagent"].Status.ContainerStatuses[0].State.Terminated = new V1ContainerStateTerminated(139, finishedAt: DateTime.Parse("2019-06-12T16:13:07Z"), startedAt: DateTime.Parse("2019-06-12T16:11:22Z"));
            modified["edgehub"].Status.ContainerStatuses[0].State.Running      = null;
            modified["edgehub"].Status.ContainerStatuses[1].State.Waiting      = new V1ContainerStateWaiting("waiting", "reason");
            modified["simulatedtemperaturesensor"].Status.ContainerStatuses[1].State.Running = null;
            foreach ((string podName, var pod) in modified)
            {
                runtimeInfo.CreateOrUpdateAddPodInfo(podName, pod);
            }

            var modules = (await runtimeInfo.GetModules(CancellationToken.None)).ToList();

            Assert.Equal(3, modules.Count);
            foreach (var i in modules)
            {
                if (!string.Equals("edgeAgent", i.Name))
                {
                    Assert.Equal(ModuleStatus.Unknown, i.ModuleStatus);
                }
                else
                {
                    Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
                }

                if (string.Equals("edgeHub", i.Name))
                {
                    Assert.Equal(Option.None <DateTime>(), i.ExitTime);
                    Assert.Equal(Option.None <DateTime>(), i.StartTime);
                }
                else
                {
                    Assert.Equal(new DateTime(2019, 6, 12), i.StartTime.OrDefault().Date);
                    Assert.Equal(new DateTime(2019, 6, 12), i.ExitTime.OrDefault().Date);
                }

                if (i is ModuleRuntimeInfo <DockerReportedConfig> d)
                {
                    Assert.NotEqual("unknown:unknown", d.Config.Image);
                }
            }
        }
Exemple #9
0
        public async Task ReturnModuleStatusWhenPodIsAbnormal(V1Pod pod, string description, ModuleStatus status)
        {
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            runtimeInfo.CreateOrUpdateAddPodInfo(pod);

            ModuleRuntimeInfo info = (await runtimeInfo.GetModules(CancellationToken.None)).Single();

            Assert.Equal(status, info.ModuleStatus);
            Assert.Equal(description, info.Description);
        }
        public async void GetModuleLogsTest()
        {
            var response = new HttpOperationResponse <Stream>();

            response.Request = new System.Net.Http.HttpRequestMessage();
            response.Body    = new MemoryStream();

            var client = new Mock <IKubernetes>(MockBehavior.Strict);

            client.Setup(kc => kc.ReadNamespacedPodLogWithHttpMessagesAsync(It.IsAny <string>(), It.IsAny <string>(), null, true, null, null, null, null, null, null, null, It.IsAny <CancellationToken>())).ReturnsAsync(() => response);
            var k8sRuntimeInfo = new KubernetesRuntimeInfoProvider(PodwatchNamespace, client.Object);
            var result         = await k8sRuntimeInfo.GetModuleLogs("module", true, Option.None <int>(), Option.None <int>(), CancellationToken.None);

            Assert.True(result.Length == 0);
        }
        public async Task ReturnModuleStatusWithPodConditionsIsEmpty()
        {
            var   client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var   moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var   runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);
            V1Pod pod           = CreatePodWithPodParametersOnly("Pending", string.Empty, string.Empty);

            pod.Status.Conditions = null;
            runtimeInfo.CreateOrUpdateAddPodInfo(pod);
            string expectedDescription = "Module Failed with Unknown pod status";

            ModuleRuntimeInfo info = (await runtimeInfo.GetModules(CancellationToken.None)).Single();

            Assert.Equal(ModuleStatus.Failed, info.ModuleStatus);
            Assert.Equal(expectedDescription, info.Description);
        }
Exemple #12
0
        public async void GetModuleLogsTest()
        {
            var logs     = Encoding.UTF8.GetBytes("some logs");
            var response = new HttpOperationResponse <Stream> {
                Request = new System.Net.Http.HttpRequestMessage(), Body = new MemoryStream(logs)
            };
            var client = new Mock <IKubernetes>(MockBehavior.Strict);

            client.Setup(kc => kc.ReadNamespacedPodLogWithHttpMessagesAsync(It.IsAny <string>(), It.IsAny <string>(), null, true, null, null, null, null, null, null, null, It.IsAny <CancellationToken>())).ReturnsAsync(() => response);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            var result = await runtimeInfo.GetModuleLogs("module", true, Option.None <int>(), Option.None <int>(), CancellationToken.None);

            Assert.True(result.Length == logs.Length);
        }
Exemple #13
0
        public async Task ReturnsModulesWhenModuleInfoAdded()
        {
            V1Pod edgeagent     = BuildPodList()["edgeagent"];
            var   client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var   moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var   runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            runtimeInfo.CreateOrUpdateAddPodInfo(edgeagent);

            var modules = await runtimeInfo.GetModules(CancellationToken.None);

            var info = modules.Single();

            Assert.NotNull(info);
            Assert.Equal("edgeAgent", info.Name);
        }
        public async Task ReturnsRestModulesWhenSomeModulesInfoRemoved()
        {
            Dictionary <string, V1Pod> pods = BuildPodList();
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            runtimeInfo.CreateOrUpdateAddPodInfo("edgeagent", pods["edgeagent"]);
            runtimeInfo.CreateOrUpdateAddPodInfo("edgehub", pods["edgehub"]);
            runtimeInfo.RemovePodInfo("edgeagent");

            var modules = await runtimeInfo.GetModules(CancellationToken.None);

            var info = modules.Single();

            Assert.NotNull(info);
            Assert.Equal("edgeHub", info.Name);
        }
Exemple #15
0
        public async void GetSystemInfoTest(V1NodeList k8SNodes, SystemInfo expectedInfo)
        {
            var response = new HttpOperationResponse <V1NodeList>();

            response.Body = k8SNodes;
            var client = new Mock <IKubernetes>(MockBehavior.Strict); // Mock.Of<IKubernetes>(kc => kc.ListNodeAsync() == Task.FromResult(k8SNodes));

            client.Setup(
                kc =>
                kc.ListNodeWithHttpMessagesAsync(null, null, null, null, null, null, null, null, null, It.IsAny <CancellationToken>())).ReturnsAsync(() => response);
            var k8sRuntimeInfo = new KubernetesRuntimeInfoProvider(PodwatchNamespace, client.Object);

            var result = await k8sRuntimeInfo.GetSystemInfo();

            Assert.Equal(expectedInfo.Architecture, result.Architecture);
            Assert.Equal(expectedInfo.OperatingSystemType, result.OperatingSystemType);
            Assert.Equal(expectedInfo.Version, result.Version);
            client.VerifyAll();
        }
Exemple #16
0
        public async Task ReturnsLastKnowModuleState()
        {
            var client        = new Mock <IKubernetes>(MockBehavior.Strict);
            var moduleManager = new Mock <IModuleManager>(MockBehavior.Strict);
            var runtimeInfo   = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);

            foreach (V1Pod pod in BuildPodList().Values)
            {
                runtimeInfo.CreateOrUpdateAddPodInfo(pod);
            }

            Dictionary <string, V1Pod> modified = BuildPodList();

            DateTime agentStartTime = new DateTime(2019, 6, 11);

            modified["edgeagent"].Status.Phase     = "Running";
            modified["edgeagent"].Status.StartTime = agentStartTime;

            string pendingDescription = "0/1 node available";

            modified["edgehub"].Status.Phase  = "Pending";
            modified["edgehub"].Status.Reason = pendingDescription;

            string finishedDescription = "Pod finished";

            modified["simulatedtemperaturesensor"].Status.Phase  = "Succeeded";
            modified["simulatedtemperaturesensor"].Status.Reason = finishedDescription;
            modified["simulatedtemperaturesensor"].Status.ContainerStatuses[1].State.Running    = null;
            modified["simulatedtemperaturesensor"].Status.ContainerStatuses[1].State.Terminated = new V1ContainerStateTerminated(139, finishedAt: DateTime.Parse("2019-06-12T16:13:07Z"), startedAt: DateTime.Parse("2019-06-12T16:11:22Z"));

            foreach (V1Pod pod in modified.Values)
            {
                runtimeInfo.CreateOrUpdateAddPodInfo(pod);
            }

            var modules = (await runtimeInfo.GetModules(CancellationToken.None)).ToList();

            Assert.Equal(3, modules.Count);

            // Normal operation statuses
            foreach (var i in modules)
            {
                if (string.Equals("edgeAgent", i.Name))
                {
                    Assert.Equal(ModuleStatus.Running, i.ModuleStatus);
                    Assert.Equal($"Started at {agentStartTime.ToString()}", i.Description);
                }
                else if (string.Equals("edgeHub", i.Name))
                {
                    Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
                    Assert.Equal(pendingDescription, i.Description);
                    Assert.Equal(Option.None <DateTime>(), i.ExitTime);
                }
                else if (string.Equals("SimulatedTemperatureSensor", i.Name))
                {
                    Assert.Equal(ModuleStatus.Stopped, i.ModuleStatus);
                    Assert.Equal(finishedDescription, i.Description);
                    Assert.Equal(new DateTime(2019, 6, 12), i.StartTime.OrDefault().Date);
                    Assert.Equal(new DateTime(2019, 6, 12), i.ExitTime.OrDefault().Date);
                }
                else
                {
                    Assert.True(false, $"Missing module {i.Name} in validation");
                }

                if (i is ModuleRuntimeInfo <DockerReportedConfig> d)
                {
                    Assert.NotEqual("unknown:unknown", d.Config.Image);
                }
            }

            string unknownDescription = "Could not reach pod";

            modified["edgeagent"].Status.Phase  = "Unknown";
            modified["edgeagent"].Status.Reason = unknownDescription;

            modified["edgehub"].Status = null;

            foreach (V1Pod pod in modified.Values)
            {
                runtimeInfo.CreateOrUpdateAddPodInfo(pod);
            }

            var abnormalModules = (await runtimeInfo.GetModules(CancellationToken.None)).ToList();

            Assert.Equal(3, modules.Count);

            // Abnormal operation statuses
            foreach (var i in abnormalModules)
            {
                if (string.Equals("edgeAgent", i.Name))
                {
                    Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
                    Assert.Equal(unknownDescription, i.Description);
                }
                else if (string.Equals("edgeHub", i.Name))
                {
                    Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
                    Assert.Equal("Unable to get pod status", i.Description);
                }
                else if (string.Equals("SimulatedTemperatureSensor", i.Name))
                {
                }
                else
                {
                    Assert.True(false, $"Missing module {i.Name} in validation");
                }
            }
        }
Exemple #17
0
        public async void PodWatchDelete()
        {
            AsyncCountdownEvent   requestHandled = new AsyncCountdownEvent(5);
            AsyncManualResetEvent serverShutdown = new AsyncManualResetEvent();

            var podWatchData = await File.ReadAllTextAsync("podwatch.txt");

            var addedAgent  = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added);
            var addedHub    = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added, 1);
            var addedSensor = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added, 3);
            var v1PodList   = JsonConvert.DeserializeObject <V1PodList>(podWatchData);

            V1Pod modHubPod = v1PodList.Items[1];
            var   modHub    = BuildPodStreamLine(modHubPod, WatchEventType.Deleted);

            V1Pod tempSensorPod = v1PodList.Items[3];
            var   modTempSensor = BuildPodStreamLine(tempSensorPod, WatchEventType.Deleted);

            using (var server = new MockKubeApiServer(
                       async httpContext =>
            {
                await MockKubeApiServer.WriteStreamLine(httpContext, addedAgent);
                await MockKubeApiServer.WriteStreamLine(httpContext, addedHub);
                await MockKubeApiServer.WriteStreamLine(httpContext, addedSensor);
                await MockKubeApiServer.WriteStreamLine(httpContext, modTempSensor);
                await MockKubeApiServer.WriteStreamLine(httpContext, modHub);
                return(false);
            }))
            {
                var client = new Kubernetes(
                    new KubernetesClientConfiguration
                {
                    Host = server.Uri.ToString()
                });

                var k8sRuntimeInfo = new KubernetesRuntimeInfoProvider(PodwatchNamespace, client);

                k8sRuntimeInfo.PropertyChanged += (sender, args) =>
                {
                    Assert.Equal("Modules", args.PropertyName);
                    requestHandled.Signal();
                };

                k8sRuntimeInfo.Start();

                await Task.WhenAny(requestHandled.WaitAsync(), Task.Delay(TestTimeout));

                var runtimeModules = await k8sRuntimeInfo.GetModules(CancellationToken.None);

                var moduleRuntimeInfos = runtimeModules as ModuleRuntimeInfo[] ?? runtimeModules.ToArray();

                Assert.Single(moduleRuntimeInfos);
                foreach (var i in moduleRuntimeInfos)
                {
                    Assert.Equal("edgeAgent", i.Name);
                    Assert.Equal(ModuleStatus.Running, i.ModuleStatus);
                    Assert.Equal(new DateTime(2019, 6, 12), i.StartTime.GetOrElse(DateTime.MinValue).Date);
                    Assert.Equal("docker", i.Type);
                    if (i is ModuleRuntimeInfo <DockerReportedConfig> d)
                    {
                        Assert.NotEqual("unknown:unknown", d.Config.Image);
                    }
                }
            }
        }
Exemple #18
0
        public async void PodWatchMods()
        {
            AsyncCountdownEvent   requestHandled = new AsyncCountdownEvent(6);
            AsyncManualResetEvent serverShutdown = new AsyncManualResetEvent();

            var podWatchData = await File.ReadAllTextAsync("podwatch.txt");

            var   addedAgent  = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added);
            var   addedHub    = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added, 1);
            var   addedSensor = BuildWatchEventStreamLine(podWatchData, WatchEventType.Added, 3);
            var   v1PodList   = JsonConvert.DeserializeObject <V1PodList>(podWatchData);
            V1Pod modAgentPod = v1PodList.Items[0];

            modAgentPod.Status.ContainerStatuses[0].State.Running    = null;
            modAgentPod.Status.ContainerStatuses[0].State.Terminated = new V1ContainerStateTerminated(139, finishedAt: DateTime.Parse("2019-06-12T16:13:07Z"), startedAt: DateTime.Parse("2019-06-12T16:11:22Z"));
            var modAgent = BuildPodStreamLine(modAgentPod, WatchEventType.Modified);

            V1Pod modHubPod = v1PodList.Items[1];

            modHubPod.Status.ContainerStatuses[0].State.Running = null;
            modHubPod.Status.ContainerStatuses[1].State.Waiting = new V1ContainerStateWaiting("waiting", "reason");
            var modHub = BuildPodStreamLine(modHubPod, WatchEventType.Modified);

            V1Pod tempSensorPod = v1PodList.Items[3]; // temp sensor has a "LastState"

            tempSensorPod.Status.ContainerStatuses[1].State.Running = null;
            var modTempSensor = BuildPodStreamLine(tempSensorPod, WatchEventType.Modified);

            using (var server = new MockKubeApiServer(
                       async httpContext =>
            {
                await MockKubeApiServer.WriteStreamLine(httpContext, addedAgent);
                await MockKubeApiServer.WriteStreamLine(httpContext, addedHub);
                await MockKubeApiServer.WriteStreamLine(httpContext, addedSensor);
                await MockKubeApiServer.WriteStreamLine(httpContext, modTempSensor);
                await MockKubeApiServer.WriteStreamLine(httpContext, modHub);
                await MockKubeApiServer.WriteStreamLine(httpContext, modAgent);
                return(false);
            }))
            {
                var client = new Kubernetes(
                    new KubernetesClientConfiguration
                {
                    Host = server.Uri.ToString()
                });

                var k8sRuntimeInfo = new KubernetesRuntimeInfoProvider(PodwatchNamespace, client);

                k8sRuntimeInfo.PropertyChanged += (sender, args) =>
                {
                    Assert.Equal("Modules", args.PropertyName);
                    requestHandled.Signal();
                };

                k8sRuntimeInfo.Start();

                await Task.WhenAny(requestHandled.WaitAsync(), Task.Delay(TestTimeout));

                var runtimeModules = await k8sRuntimeInfo.GetModules(CancellationToken.None);

                var moduleRuntimeInfos = runtimeModules as ModuleRuntimeInfo[] ?? runtimeModules.ToArray();

                Assert.True(moduleRuntimeInfos.Count() == 3);
                foreach (var i in moduleRuntimeInfos)
                {
                    if (!string.Equals("edgeAgent", i.Name))
                    {
                        Assert.Equal(ModuleStatus.Unknown, i.ModuleStatus);
                    }
                    else
                    {
                        Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
                    }

                    if (string.Equals("edgeHub", i.Name))
                    {
                        Assert.Equal(Option.None <DateTime>(), i.ExitTime);
                        Assert.Equal(Option.None <DateTime>(), i.StartTime);
                    }
                    else
                    {
                        Assert.Equal(new DateTime(2019, 6, 12), i.StartTime.GetOrElse(DateTime.MinValue).Date);
                        Assert.Equal(new DateTime(2019, 6, 12), i.ExitTime.GetOrElse(DateTime.MinValue).Date);
                    }

                    if (i is ModuleRuntimeInfo <DockerReportedConfig> d)
                    {
                        Assert.NotEqual("unknown:unknown", d.Config.Image);
                    }
                }
            }
        }