public async Task UpdateSuccess_ShallArchiveAndDelete() { _kubernetesClient .Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>())) .Returns(Task.FromResult(new HttpOperationResponse <object> { Response = new HttpResponseMessage() })); _kubernetesClient .Setup(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <string>())) .Returns(Task.FromResult(new HttpOperationResponse <object> { Response = new HttpResponseMessage() })); var inferenceRequest = new InferenceRequest(); inferenceRequest.JobId = Guid.NewGuid().ToString(); inferenceRequest.PayloadId = Guid.NewGuid().ToString(); inferenceRequest.TransactionId = Guid.NewGuid().ToString(); var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object); await store.Update(inferenceRequest, InferenceRequestStatus.Success); _logger.VerifyLoggingMessageBeginsWith($"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once()); _logger.VerifyLoggingMessageBeginsWith($"Inference request deleted. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once()); _kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once()); _kubernetesClient.Verify(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), inferenceRequest.JobId), Times.Once()); }
public async Task UpdateSuccess_ShallRetryDeleteOnFailure() { _kubernetesClient .Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>())) .Returns(Task.FromResult(new HttpOperationResponse <object> { Response = new HttpResponseMessage() })); _kubernetesClient .Setup(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <string>())) .Throws(new HttpOperationException("error message") { Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.Conflict), "error content") }); var inferenceRequest = new InferenceRequest(); inferenceRequest.JobId = Guid.NewGuid().ToString(); inferenceRequest.PayloadId = Guid.NewGuid().ToString(); inferenceRequest.TransactionId = Guid.NewGuid().ToString(); var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object); await Assert.ThrowsAsync <HttpOperationException>(async() => await store.Update(inferenceRequest, InferenceRequestStatus.Success)); _logger.VerifyLoggingMessageBeginsWith($"Failed to delete inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD.", LogLevel.Warning, Times.Exactly(3)); _logger.VerifyLoggingMessageBeginsWith($"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once()); _logger.VerifyLoggingMessageBeginsWith($"Inference request deleted. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Never()); _kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once()); _kubernetesClient.Verify(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), inferenceRequest.JobId), Times.Exactly(4)); }
public void CancellationTokenShallCancelTheService() { var cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.Cancel(); var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object); store.StartAsync(cancellationTokenSource.Token); store.StopAsync(cancellationTokenSource.Token); Thread.Sleep(100); _logger.VerifyLogging($"Inference Request Store Hosted Service is running.", LogLevel.Information, Times.Once()); _logger.VerifyLogging($"Inference Request Store Hosted Service is stopping.", LogLevel.Information, Times.Once()); }
public async Task Add_ShallRetryOnFailure() { _kubernetesClient .Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>())) .Throws(new HttpOperationException("error message") { Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.Conflict), "error content") }); var inferenceRequest = new InferenceRequest(); inferenceRequest.JobId = Guid.NewGuid().ToString(); inferenceRequest.PayloadId = Guid.NewGuid().ToString(); inferenceRequest.TransactionId = Guid.NewGuid().ToString(); var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object); await Assert.ThrowsAsync <HttpOperationException>(async() => await store.Add(inferenceRequest)); _logger.VerifyLoggingMessageBeginsWith($"Failed to add new inference request with JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD", LogLevel.Warning, Times.Exactly(3)); _kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Exactly(4)); }
public async Task Add_ShallAddItemToCrd() { _kubernetesClient .Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>())) .Returns(Task.FromResult(new HttpOperationResponse <object> { Response = new HttpResponseMessage() })); var inferenceRequest = new InferenceRequest(); inferenceRequest.JobId = Guid.NewGuid().ToString(); inferenceRequest.PayloadId = Guid.NewGuid().ToString(); inferenceRequest.TransactionId = Guid.NewGuid().ToString(); var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object); await store.Add(inferenceRequest); _logger.VerifyLoggingMessageBeginsWith($"Failed to add new inference request with JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD", LogLevel.Warning, Times.Never()); _kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once()); _logger.VerifyLoggingMessageBeginsWith($"Inference request saved. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD", LogLevel.Warning, Times.Never()); }
public async Task Take_ShallReturnAJobReadFromCrd() { var cancellationSource = new CancellationTokenSource(); var list = new InferenceRequestCustomResourceList(); list.Items = new List <InferenceRequestCustomResource>(); list.Items.Add(new InferenceRequestCustomResource { Spec = new InferenceRequest { State = InferenceRequestState.InProcess }, Metadata = new V1ObjectMeta { Name = Guid.NewGuid().ToString() } }); list.Items.Add(new InferenceRequestCustomResource { Spec = new InferenceRequest { State = InferenceRequestState.InProcess }, Metadata = new V1ObjectMeta { Name = Guid.NewGuid().ToString() } }); list.Items.Add(new InferenceRequestCustomResource { Spec = new InferenceRequest { State = InferenceRequestState.Queued }, Metadata = new V1ObjectMeta { Name = Guid.NewGuid().ToString() } }); _kubernetesClient .SetupSequence(p => p.ListNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>())) .Returns( Task.FromResult(new HttpOperationResponse <object> { Body = new object(), Response = new HttpResponseMessage { Content = new StringContent(JsonConvert.SerializeObject(list)) } })) .Returns(() => { cancellationSource.Cancel(); Thread.Sleep(100); throw new HttpOperationException("exception"); }); _kubernetesClient .Setup(p => p.PatchNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <CustomResource>(), It.IsAny <string>())) .Returns(Task.FromResult(new HttpOperationResponse <object> { Body = new object(), Response = new HttpResponseMessage(HttpStatusCode.OK) })); var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object); await store.StartAsync(cancellationSource.Token); var expectedItem = list.Items.Last(); var item = await store.Take(cancellationSource.Token); Assert.Equal(expectedItem.Spec.JobId, item.JobId); _logger.VerifyLogging($"Inference request added to queue {item.JobId}", LogLevel.Debug, Times.AtLeastOnce()); _logger.VerifyLogging($"Inference Request Store Hosted Service is running.", LogLevel.Information, Times.Once()); await store.StopAsync(cancellationSource.Token); _logger.VerifyLogging($"Inference Request Store Hosted Service is stopping.", LogLevel.Information, Times.Once()); _kubernetesClient.Verify( p => p.PatchNamespacedCustomObjectWithHttpMessagesAsync( It.IsAny <CustomResourceDefinition>(), It.IsAny <InferenceRequestCustomResource>(), expectedItem.Spec.JobId), Times.Once()); }