public async Task GetNextSubtaskAsync_ShouldNot_ReturnSubtask_When_AllAreDone()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Return_next_subtask_when_all_are_done");
using (var dbContext = new TestDbContext(dbContextOptions))
{
SeedDistributedTask(dbContext, 1);
dbContext.Subtasks.Add(new Subtask()
{
Id = 1,
DistributedTaskId = 1,
Status = SubtaskStatus.Done
});
dbContext.Subtasks.Add(new Subtask()
{
Id = 2,
DistributedTaskId = 1,
Status = SubtaskStatus.Done
});
await dbContext.SaveChangesAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var getNextSubtaskToComputeService = new GetNextSubtaskToComputeService(dbContext);
var subtask = await getNextSubtaskToComputeService.GetNextSubtaskAsync();
Assert.Null(subtask);
}
}
public async Task GetNextSubtaskAsync_ShouldNot_ReturnSubtask_When_DistributedTaskIsNotInProgress()
{
var dbContextOptions =
DbContextOptionsFactory.CreateOptions("Return_next_subtask_when_distributed_task_not_in_progress");
using (var dbContext = new TestDbContext(dbContextOptions))
{
dbContext.DistributedTasks.Add(new DistributedTask()
{
Id = 1,
Status = DistributedTaskStatus.Error,
});
dbContext.Subtasks.Add(new Subtask()
{
Id = 1,
DistributedTaskId = 1,
Status = SubtaskStatus.Done
});
await dbContext.SaveChangesAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var getNextSubtaskToComputeService = new GetNextSubtaskToComputeService(dbContext);
var subtask = await getNextSubtaskToComputeService.GetNextSubtaskAsync();
Assert.Null(subtask);
}
}
public async Task GetNextSubtaskAsync_Should_ReturnASubtask_BasedOnDistributedTaskPriority()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Return_next_subtask_base_on_priority");
using (var dbContext = new TestDbContext(dbContextOptions))
{
SeedDistributedTask(dbContext, 1);
dbContext.Subtasks.Add(new Subtask()
{
Id = 1,
DistributedTaskId = 1,
Status = SubtaskStatus.WaitingForExecution
});
SeedDistributedTask(dbContext, 2, priority: 5);
dbContext.Subtasks.Add(new Subtask()
{
Id = 2,
DistributedTaskId = 2,
Status = SubtaskStatus.WaitingForExecution
});
await dbContext.SaveChangesAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var getNextSubtaskToComputeService = new GetNextSubtaskToComputeService(dbContext);
var subtask = await getNextSubtaskToComputeService.GetNextSubtaskAsync();
Assert.AreEqual(2, subtask.Id);
}
}
public async Task FailSubtaskInProgressAsync_Should_MarkSubtaskInProgressAsError()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Mark_subtask_as_error");
using (var dbContext = new TestDbContext(dbContextOptions))
{
CreateMockData(dbContext);
await dbContext.SaveChangesAsync();
}
var subtaskInProgressCleanupServiceMock = new Mock <ISubtasksInProgressCleanupService>();
subtaskInProgressCleanupServiceMock.Setup(service => service.RemoveSubtasksInProgress(It.IsAny <int>()));
using (var dbContext = new TestDbContext(dbContextOptions))
{
var computationFailService =
new ComputationFailService(dbContext, subtaskInProgressCleanupServiceMock.Object);
await computationFailService.FailSubtaskInProgressAsync(1, new[] { "Some error" });
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var foundSubtaskInProgress = await dbContext.SubtasksInProgress
.Include(subtaskInProgress => subtaskInProgress.Subtask)
.FirstAsync(subtaskInProgress => subtaskInProgress.Id == 1);
Assert.AreEqual(SubtaskInProgressStatus.Error, foundSubtaskInProgress.Status);
Assert.AreEqual(SubtaskStatus.Executing, foundSubtaskInProgress.Subtask.Status);
}
}
public async Task CancelSubtaskInProgressAsync_Should_SetSubtaskInProgressStatusToCancelled()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Cancel_subtask_in_progress");
using (var dbContext = new TestDbContext(dbContextOptions))
{
await dbContext.SubtasksInProgress.AddAsync(new SubtaskInProgress()
{
Id = 1,
Status = SubtaskInProgressStatus.Executing
});
await dbContext.SaveChangesAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var computationCancelService = new ComputationCancelService(dbContext);
await computationCancelService.CancelSubtaskInProgressAsync(1);
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var subtaskInProgress = await dbContext.SubtasksInProgress.FindAsync(1);
Assert.AreEqual(SubtaskInProgressStatus.Cancelled, subtaskInProgress.Status);
}
}
public async Task CleanAsync_ShouldNot_ModifySubtasks_When_NodeIsAlive()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Not_modify_subtasks_when_node_is_alive");
using (var dbContext = new TestDbContext(dbContextOptions))
{
var distributedNodeId = Guid.NewGuid();
dbContext.DistributedNodes.Add(new DistributedNode()
{
Id = distributedNodeId,
LastKeepAliveTime = DateTime.Now.Subtract(TimeSpan.FromMinutes(5)),
SubtasksInProgress = new List <SubtaskInProgress>()
{
new SubtaskInProgress()
{
Id = 1,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Executing
},
new SubtaskInProgress()
{
Id = 2,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Done
},
new SubtaskInProgress()
{
Id = 3,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Executing
}
}
});
await dbContext.SaveChangesAsync();
}
var computationCancelServiceMock = new Mock <IComputationCancelService>();
computationCancelServiceMock.Setup(service => service.CancelSubtaskInProgressWithoutSavingAsync(It.IsAny <int>()))
.Returns(Task.CompletedTask);
using (var dbContext = new TestDbContext(dbContextOptions))
{
var cleaner = new DistributedNodesCleaner(dbContext, computationCancelServiceMock.Object,
TimeSpan.FromMinutes(60));
await cleaner.CleanAsync();
}
computationCancelServiceMock.VerifyNoOtherCalls();
using (var dbContext = new TestDbContext(dbContextOptions))
{
var distributedNodesCount = await dbContext.DistributedNodes.CountAsync();
Assert.AreEqual(1, distributedNodesCount);
}
}
public async Task CleanAsync_Should_RemoveNode_When_StaleAndNoDoneTasks()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Remove_node_when_stale_and_has_no_done_tasks");
using (var dbContext = new TestDbContext(dbContextOptions))
{
var distributedNodeId = Guid.NewGuid();
dbContext.DistributedNodes.Add(new DistributedNode()
{
Id = distributedNodeId,
LastKeepAliveTime = DateTime.Now.Subtract(TimeSpan.FromMinutes(5)),
SubtasksInProgress = new List <SubtaskInProgress>()
{
new SubtaskInProgress()
{
Id = 1,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Executing
},
new SubtaskInProgress()
{
Id = 2,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Error
},
new SubtaskInProgress()
{
Id = 3,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Cancelled
}
}
});
await dbContext.SaveChangesAsync();
}
var computationCancelServiceMock = new Mock <IComputationCancelService>();
computationCancelServiceMock.Setup(service => service.CancelSubtaskInProgressWithoutSavingAsync(It.IsAny <int>()))
.Returns(Task.CompletedTask);
using (var dbContext = new TestDbContext(dbContextOptions))
{
var cleaner = new DistributedNodesCleaner(dbContext, computationCancelServiceMock.Object,
TimeSpan.FromSeconds(5));
await cleaner.CleanAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var distributedNodesCount = await dbContext.DistributedNodes.CountAsync();
Assert.AreEqual(0, distributedNodesCount);
}
}
public async Task CompleteSubtaskInProgressAsync_Should_MarkDistributedTaskAsDone_When_NoOtherSubtasksLeft()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Mark_distributed_task_as_done");
using (var dbContext = new TestDbContext(dbContextOptions))
{
await CreateMockData(dbContext);
await dbContext.SaveChangesAsync();
}
var problemPluginFacadeMock = new Mock <IProblemPluginFacade>();
problemPluginFacadeMock.Setup(facade => facade.JoinSubtaskResults(It.IsAny <IEnumerable <byte[]> >()))
.Returns(new byte[] { 1, 2, 3 });
var problemPluginFacadeProviderMock = new Mock <IProblemPluginFacadeProvider>();
problemPluginFacadeProviderMock.Setup(provider =>
provider.Provide(
It.Is <DistributedTaskDefinition>(taskDefinition => taskDefinition.Name == "Task definition")))
.Returns(() => problemPluginFacadeMock.Object);
var subtaskInProgressCleanupServiceMock = new Mock <ISubtasksInProgressCleanupService>();
subtaskInProgressCleanupServiceMock.Setup(service => service.RemoveSubtasksInProgress(It.IsAny <int>()));
using (var dbContext = new TestDbContext(dbContextOptions))
{
var computationCompleteService =
new ComputationCompleteService(dbContext, problemPluginFacadeProviderMock.Object,
subtaskInProgressCleanupServiceMock.Object);
using (var stream = new MemoryStream(new byte[] { 4, 5, 6 }))
{
await computationCompleteService.CompleteSubtaskInProgressAsync(1, stream);
}
}
problemPluginFacadeProviderMock.Verify(provider => provider.Provide(It.IsAny <DistributedTaskDefinition>()),
Times.Once);
problemPluginFacadeMock.Verify(facade => facade.JoinSubtaskResults(It.IsAny <IEnumerable <byte[]> >()),
Times.Once);
using (var dbContext = new TestDbContext(dbContextOptions))
{
var foundSubtask = await dbContext.Subtasks
.Include(subtask => subtask.DistributedTask)
.Include(subtask => subtask.SubtasksInProgress)
.FirstAsync(subtask => subtask.Id == 1);
Assert.AreEqual(SubtaskStatus.Done, foundSubtask.Status);
Assert.AreEqual(DistributedTaskStatus.Done, foundSubtask.DistributedTask.Status);
}
}
public async Task CompleteSubtaskInProgressAsync_Should_MarkSubtaskAsDone_When_TrustLevelReached()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Mark_subtask_as_done");
using (var dbContext = new TestDbContext(dbContextOptions))
{
await CreateMockData(dbContext);
dbContext.Subtasks.Add(new Subtask()
{
DistributedTaskId = 1,
Id = 2,
SequenceNumber = 1,
});
await dbContext.SaveChangesAsync();
}
var problemPluginFacadeProviderMock = new Mock <IProblemPluginFacadeProvider>();
var subtaskInProgressCleanupServiceMock = new Mock <ISubtasksInProgressCleanupService>();
subtaskInProgressCleanupServiceMock.Setup(service => service.RemoveSubtasksInProgress(It.IsAny <int>()));
using (var dbContext = new TestDbContext(dbContextOptions))
{
if (subtaskInProgressCleanupServiceMock.Object != null)
{
var computationCompleteService =
new ComputationCompleteService(dbContext, problemPluginFacadeProviderMock.Object,
subtaskInProgressCleanupServiceMock.Object);
using (var stream = new MemoryStream(new byte[] { 4, 5, 6 }))
{
await computationCompleteService.CompleteSubtaskInProgressAsync(1, stream);
}
}
}
subtaskInProgressCleanupServiceMock.Verify(service => service.RemoveSubtasksInProgress(1), Times.Once);
using (var dbContext = new TestDbContext(dbContextOptions))
{
var foundSubtaskInProgress = await dbContext.SubtasksInProgress
.Include(subtaskInProgress => subtaskInProgress.Subtask)
.ThenInclude(subtask => subtask.DistributedTask)
.FirstAsync(subtaskInProgress => subtaskInProgress.Id == 1);
Assert.AreEqual(SubtaskInProgressStatus.Done, foundSubtaskInProgress.Status);
Assert.AreEqual(SubtaskStatus.Done, foundSubtaskInProgress.Subtask.Status);
Assert.AreEqual(DistributedTaskStatus.InProgress,
foundSubtaskInProgress.Subtask.DistributedTask.Status);
}
}
public async Task GetNextSubtaskAsync_ShouldNot_ReturnSubtask_When_SubtasksInProgressAreCancelled()
{
var dbContextOptions =
DbContextOptionsFactory.CreateOptions("Return_next_subtask_when_subtasks_in_progress_are_cancelled");
using (var dbContext = new TestDbContext(dbContextOptions))
{
SeedDistributedTask(dbContext, 1);
dbContext.Subtasks.Add(new Subtask()
{
Id = 1,
DistributedTaskId = 1,
Status = SubtaskStatus.Executing,
SubtasksInProgress = new List <SubtaskInProgress>()
{
new SubtaskInProgress()
{
Status = SubtaskInProgressStatus.Cancelled,
Id = 1,
Node = new DistributedNode()
{
TrustLevel = 5
}
},
new SubtaskInProgress()
{
Status = SubtaskInProgressStatus.Cancelled,
Id = 2,
Node = new DistributedNode()
{
TrustLevel = 5
}
}
}
});
await dbContext.SaveChangesAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var getNextSubtaskToComputeService = new GetNextSubtaskToComputeService(dbContext);
var subtask = await getNextSubtaskToComputeService.GetNextSubtaskAsync();
Assert.NotNull(subtask);
Assert.AreEqual(1, subtask.Id);
}
}
public async Task CompleteSubtaskInProgressAsync_ShouldNot_MarkSubtaskAsDone_When_TrustLevelNotReached()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Not_mark_subtask_as_done");
using (var dbContext = new TestDbContext(dbContextOptions))
{
await CreateMockData(dbContext);
var distributedTask = await dbContext.DistributedTasks.FindAsync(1);
distributedTask.TrustLevelToComplete = 5;
await dbContext.SaveChangesAsync();
}
var problemPluginFacadeProviderMock = new Mock <IProblemPluginFacadeProvider>();
var subtaskInProgressCleanupServiceMock = new Mock <ISubtasksInProgressCleanupService>();
using (var dbContext = new TestDbContext(dbContextOptions))
{
var computationCompleteService =
new ComputationCompleteService(dbContext, problemPluginFacadeProviderMock.Object,
subtaskInProgressCleanupServiceMock.Object);
using (var stream = new MemoryStream(new byte[] { 4, 5, 6 }))
{
await computationCompleteService.CompleteSubtaskInProgressAsync(1, stream);
}
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var foundSubtaskInProgress = await dbContext.SubtasksInProgress
.Include(subtaskInProgress => subtaskInProgress.Subtask)
.ThenInclude(subtask => subtask.DistributedTask)
.FirstAsync(subtaskInProgress => subtaskInProgress.Id == 1);
Assert.AreEqual(SubtaskInProgressStatus.Done, foundSubtaskInProgress.Status);
Assert.AreEqual(SubtaskStatus.Executing, foundSubtaskInProgress.Subtask.Status);
Assert.AreEqual(DistributedTaskStatus.InProgress,
foundSubtaskInProgress.Subtask.DistributedTask.Status);
}
}
FailSubtaskInProgressAsync_Should_MarkDistributedTaskAsError_WhenMultipleSubtasksInProgressFailed()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Mark_distributed task_as_error");
using (var dbContext = new TestDbContext(dbContextOptions))
{
CreateMockData(dbContext);
dbContext.SubtasksInProgress.Add(new SubtaskInProgress()
{
Id = 2,
SubtaskId = 1,
Status = SubtaskInProgressStatus.Executing
});
await dbContext.SaveChangesAsync();
}
var subtaskInProgressCleanupServiceMock = new Mock <ISubtasksInProgressCleanupService>();
subtaskInProgressCleanupServiceMock.Setup(service => service.RemoveSubtasksInProgress(It.IsAny <int>()));
using (var dbContext = new TestDbContext(dbContextOptions))
{
var computationFailService =
new ComputationFailService(dbContext, subtaskInProgressCleanupServiceMock.Object);
await computationFailService.FailSubtaskInProgressAsync(2, new[] { "Some error" });
}
subtaskInProgressCleanupServiceMock.Verify(service => service.RemoveSubtasksInProgress(1), Times.Once);
using (var dbContext = new TestDbContext(dbContextOptions))
{
var foundSubtaskInProgress = await dbContext.SubtasksInProgress
.Include(subtaskInProgress => subtaskInProgress.Subtask)
.ThenInclude(subtask => subtask.DistributedTask)
.FirstAsync(subtaskInProgress => subtaskInProgress.Id == 1);
Assert.AreEqual(SubtaskInProgressStatus.Error, foundSubtaskInProgress.Status);
Assert.AreEqual(SubtaskStatus.Error, foundSubtaskInProgress.Subtask.Status);
Assert.AreEqual(DistributedTaskStatus.Error, foundSubtaskInProgress.Subtask.DistributedTask.Status);
}
}
public async Task GetNextSubtaskAsync_ShouldNot_ReturnSubtask_When_TrustLevelAlreadyReached()
{
var dbContextOptions =
DbContextOptionsFactory.CreateOptions("Return_next_subtask_when_trust_level_already_reached");
using (var dbContext = new TestDbContext(dbContextOptions))
{
SeedDistributedTask(dbContext, 1, trustLevelToComplete: 5);
dbContext.Subtasks.Add(new Subtask()
{
Id = 1,
DistributedTaskId = 1,
Status = SubtaskStatus.Executing,
SubtasksInProgress = new List <SubtaskInProgress>()
{
new SubtaskInProgress()
{
Status = SubtaskInProgressStatus.Executing,
Id = 1,
Node = new DistributedNode()
{
TrustLevel = 5
}
}
}
});
await dbContext.SaveChangesAsync();
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var getNextSubtaskToComputeService = new GetNextSubtaskToComputeService(dbContext);
var subtask = await getNextSubtaskToComputeService.GetNextSubtaskAsync();
Assert.Null(subtask);
}
}
public async Task CleanAsync_Should_MarkSubtasksInProgressAsCancelled()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Mark_subtasks_as_cancelled");
using (var dbContext = new TestDbContext(dbContextOptions))
{
var distributedNodeId = Guid.NewGuid();
dbContext.DistributedNodes.Add(new DistributedNode()
{
Id = distributedNodeId,
LastKeepAliveTime = DateTime.Now.Subtract(TimeSpan.FromMinutes(5)),
SubtasksInProgress = new List <SubtaskInProgress>()
{
new SubtaskInProgress()
{
Id = 1,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Executing
},
new SubtaskInProgress()
{
Id = 2,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Done
},
new SubtaskInProgress()
{
Id = 3,
NodeId = distributedNodeId,
Status = SubtaskInProgressStatus.Executing
}
}
});
dbContext.DistributedNodes.Add(new DistributedNode()
{
Id = Guid.NewGuid(),
LastKeepAliveTime = DateTime.Now.Subtract(TimeSpan.FromMinutes(5)),
SubtasksInProgress = new List <SubtaskInProgress>()
{
new SubtaskInProgress()
{
Id = 4,
Status = SubtaskInProgressStatus.Done
},
}
});
await dbContext.SaveChangesAsync();
}
var computationCancelServiceMock = new Mock <IComputationCancelService>();
computationCancelServiceMock.Setup(service => service.CancelSubtaskInProgressWithoutSavingAsync(1))
.Returns(Task.CompletedTask)
.Verifiable();
computationCancelServiceMock.Setup(service => service.CancelSubtaskInProgressWithoutSavingAsync(3))
.Returns(Task.CompletedTask)
.Verifiable();
using (var dbContext = new TestDbContext(dbContextOptions))
{
var cleaner = new DistributedNodesCleaner(dbContext, computationCancelServiceMock.Object,
TimeSpan.FromSeconds(1));
await cleaner.CleanAsync();
}
computationCancelServiceMock.Verify(service => service.CancelSubtaskInProgressWithoutSavingAsync(1));
computationCancelServiceMock.Verify(service => service.CancelSubtaskInProgressWithoutSavingAsync(3));
computationCancelServiceMock.VerifyNoOtherCalls();
using (var dbContext = new TestDbContext(dbContextOptions))
{
var distributedNodesCount = await dbContext.DistributedNodes.CountAsync();
Assert.AreEqual(2, distributedNodesCount);
}
}
public async Task CompleteSubtaskInProgressAsync_Should_SelectMostTrustedResult_When_TrustLevelReached()
{
var dbContextOptions = DbContextOptionsFactory.CreateOptions("Select_most_trusted_result");
using (var dbContext = new TestDbContext(dbContextOptions))
{
await CreateMockData(dbContext);
dbContext.Subtasks.Add(new Subtask()
{
DistributedTaskId = 1,
Id = 2,
SequenceNumber = 1,
});
dbContext.SubtasksInProgress.AddRange(new SubtaskInProgress()
{
Id = 2,
SubtaskId = 1,
Node = new DistributedNode()
{
TrustLevel = 5,
},
Result = new byte[] { 1, 2, 3 },
Status = SubtaskInProgressStatus.Done,
}, new SubtaskInProgress()
{
Id = 3,
SubtaskId = 1,
Node = new DistributedNode()
{
TrustLevel = 3,
},
Result = new byte[] { 4, 5, 6 },
Status = SubtaskInProgressStatus.Done,
});
await dbContext.SaveChangesAsync();
}
var problemPluginFacadeProviderMock = new Mock <IProblemPluginFacadeProvider>();
var subtaskInProgressCleanupServiceMock = new Mock <ISubtasksInProgressCleanupService>();
subtaskInProgressCleanupServiceMock.Setup(service => service.RemoveSubtasksInProgress(It.IsAny <int>()));
using (var dbContext = new TestDbContext(dbContextOptions))
{
var computationCompleteService =
new ComputationCompleteService(dbContext, problemPluginFacadeProviderMock.Object,
subtaskInProgressCleanupServiceMock.Object);
using (var stream = new MemoryStream(new byte[] { 4, 5, 6 }))
{
await computationCompleteService.CompleteSubtaskInProgressAsync(1, stream);
}
}
using (var dbContext = new TestDbContext(dbContextOptions))
{
var foundSubtask = await dbContext.Subtasks
.FirstAsync(subtask => subtask.Id == 1);
Assert.AreEqual(new byte[] { 1, 2, 3 }, foundSubtask.Result);
}
}
public DataServiceContext() : base(DbContextOptionsFactory.CreateOptions(ServiceConfiguration.DefaultEnvironment, ServiceConfiguration.ConnectionName, ServiceConfiguration.ServiceSchema))
{
ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
ChangeTracker.AutoDetectChangesEnabled = false;
}
