public async Task UpdateSuccess_ShallArchiveAndDelete()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
_kubernetesClient
.Setup(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <string>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await store.Update(inferenceRequest, InferenceRequestStatus.Success);
_logger.VerifyLoggingMessageBeginsWith($"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once());
_logger.VerifyLoggingMessageBeginsWith($"Inference request deleted. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once());
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once());
_kubernetesClient.Verify(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), inferenceRequest.JobId), Times.Once());
}
public async Task UpdateSuccess_ShallRetryDeleteOnFailure()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
_kubernetesClient
.Setup(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <string>()))
.Throws(new HttpOperationException("error message")
{
Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.Conflict), "error content")
});
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await Assert.ThrowsAsync <HttpOperationException>(async() => await store.Update(inferenceRequest, InferenceRequestStatus.Success));
_logger.VerifyLoggingMessageBeginsWith($"Failed to delete inference request JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD.", LogLevel.Warning, Times.Exactly(3));
_logger.VerifyLoggingMessageBeginsWith($"Inference request archived. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Once());
_logger.VerifyLoggingMessageBeginsWith($"Inference request deleted. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId}", LogLevel.Information, Times.Never());
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once());
_kubernetesClient.Verify(p => p.DeleteNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), inferenceRequest.JobId), Times.Exactly(4));
}
public void CancellationTokenShallCancelTheService()
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.Cancel();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
store.StartAsync(cancellationTokenSource.Token);
store.StopAsync(cancellationTokenSource.Token);
Thread.Sleep(100);
_logger.VerifyLogging($"Inference Request Store Hosted Service is running.", LogLevel.Information, Times.Once());
_logger.VerifyLogging($"Inference Request Store Hosted Service is stopping.", LogLevel.Information, Times.Once());
}
public async Task Add_ShallRetryOnFailure()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Throws(new HttpOperationException("error message")
{
Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.Conflict), "error content")
});
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await Assert.ThrowsAsync <HttpOperationException>(async() => await store.Add(inferenceRequest));
_logger.VerifyLoggingMessageBeginsWith($"Failed to add new inference request with JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD", LogLevel.Warning, Times.Exactly(3));
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Exactly(4));
}
public async Task Add_ShallAddItemToCrd()
{
_kubernetesClient
.Setup(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Response = new HttpResponseMessage()
}));
var inferenceRequest = new InferenceRequest();
inferenceRequest.JobId = Guid.NewGuid().ToString();
inferenceRequest.PayloadId = Guid.NewGuid().ToString();
inferenceRequest.TransactionId = Guid.NewGuid().ToString();
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await store.Add(inferenceRequest);
_logger.VerifyLoggingMessageBeginsWith($"Failed to add new inference request with JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD", LogLevel.Warning, Times.Never());
_kubernetesClient.Verify(p => p.CreateNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <object>()), Times.Once());
_logger.VerifyLoggingMessageBeginsWith($"Inference request saved. JobId={inferenceRequest.JobId}, TransactionId={inferenceRequest.TransactionId} in CRD", LogLevel.Warning, Times.Never());
}
public async Task Take_ShallReturnAJobReadFromCrd()
{
var cancellationSource = new CancellationTokenSource();
var list = new InferenceRequestCustomResourceList();
list.Items = new List <InferenceRequestCustomResource>();
list.Items.Add(new InferenceRequestCustomResource
{
Spec = new InferenceRequest
{
State = InferenceRequestState.InProcess
},
Metadata = new V1ObjectMeta {
Name = Guid.NewGuid().ToString()
}
});
list.Items.Add(new InferenceRequestCustomResource
{
Spec = new InferenceRequest
{
State = InferenceRequestState.InProcess
},
Metadata = new V1ObjectMeta {
Name = Guid.NewGuid().ToString()
}
});
list.Items.Add(new InferenceRequestCustomResource
{
Spec = new InferenceRequest
{
State = InferenceRequestState.Queued
},
Metadata = new V1ObjectMeta {
Name = Guid.NewGuid().ToString()
}
});
_kubernetesClient
.SetupSequence(p => p.ListNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>()))
.Returns(
Task.FromResult(new HttpOperationResponse <object>
{
Body = new object(),
Response = new HttpResponseMessage {
Content = new StringContent(JsonConvert.SerializeObject(list))
}
}))
.Returns(() =>
{
cancellationSource.Cancel();
Thread.Sleep(100);
throw new HttpOperationException("exception");
});
_kubernetesClient
.Setup(p => p.PatchNamespacedCustomObjectWithHttpMessagesAsync(It.IsAny <CustomResourceDefinition>(), It.IsAny <CustomResource>(), It.IsAny <string>()))
.Returns(Task.FromResult(new HttpOperationResponse <object>
{
Body = new object(),
Response = new HttpResponseMessage(HttpStatusCode.OK)
}));
var store = new InferenceRequestStore(_loggerFactory.Object, _configuration, _kubernetesClient.Object);
await store.StartAsync(cancellationSource.Token);
var expectedItem = list.Items.Last();
var item = await store.Take(cancellationSource.Token);
Assert.Equal(expectedItem.Spec.JobId, item.JobId);
_logger.VerifyLogging($"Inference request added to queue {item.JobId}", LogLevel.Debug, Times.AtLeastOnce());
_logger.VerifyLogging($"Inference Request Store Hosted Service is running.", LogLevel.Information, Times.Once());
await store.StopAsync(cancellationSource.Token);
_logger.VerifyLogging($"Inference Request Store Hosted Service is stopping.", LogLevel.Information, Times.Once());
_kubernetesClient.Verify(
p => p.PatchNamespacedCustomObjectWithHttpMessagesAsync(
It.IsAny <CustomResourceDefinition>(),
It.IsAny <InferenceRequestCustomResource>(),
expectedItem.Spec.JobId), Times.Once());
}
