private async Task ProcessNew(File file, EofMessage eofMessage)
{
var tempPath = await BuildFile(file, eofMessage);
if (tempPath is null)
{
return;
}
var filePath = file.GetFullPath();
if (System.IO.File.Exists(filePath) && FileLock.IsLocked(filePath, checkWriteAccess: true))
{
WorkerLog.Instance.Warning($"File '{filePath}' is locked for writing. unable overwrite file");
}
else
{
WorkerLog.Instance.Information($"Decompressing file '{tempPath}'");
await file.DecompressFrom(tempPath);
WorkerLog.Instance.Information($"File decompressed to '{filePath}'");
await MarkFileAsCompleted(file);
}
DeleteTempPath(tempPath);
}
public async Task SendEofMessage_ShouldCallTransmissionLinkSendEofMessage()
{
Recipient transmittedRecipient = null;
EofMessage transmittedEofMessage = null;
var transmissionLinkMock = new Mock <ITransmissionLink>();
transmissionLinkMock.Setup(x => x.SendEofMessage(It.IsAny <Recipient>(), It.IsAny <EofMessage>()))
.Callback <Recipient, EofMessage>((x, y) =>
{
transmittedRecipient = x;
transmittedEofMessage = y;
});
var transmissionRepositoryMock = new Mock <ITransmissionRepository>();
transmissionRepositoryMock.Setup(x => x.AddFailedFileTransmission(It.IsAny <Guid>(), It.IsAny <Guid>()));
var recipient = Helper.GetRecipientModel("testRecipient");
var eofMesage = Helper.GetEofMessageModel();
var transmissionService = new TransmissionService(transmissionLinkMock.Object, transmissionRepositoryMock.Object);
await transmissionService.SendEofMessage(recipient, eofMesage);
Assert.AreSame(recipient, transmittedRecipient);
Assert.AreSame(eofMesage, transmittedEofMessage);
}
public async Task <Result> AddReceivedEofMessage(EofMessage eofMessage)
{
var db = _database.Get();
using (db)
return(await db.Execute(_insertStatementAddReceivedEofMessage, new { eofMessage.Id, eofMessage.FileId, eofMessage.AmountOfChunks }));
}
private async Task <string> BuildFile(File file, EofMessage eofMessage)
{
try
{
WorkerLog.Instance.Information($"Building {(file.IsNew() ? "" : "delta")} file '{file.Name}'");
var temppath = System.IO.Path.Combine(System.IO.Path.GetTempPath(), System.IO.Path.GetTempFileName());
using var sw = System.IO.File.OpenWrite(temppath);
var toTake = 10;
var toSkip = 0;
while (toSkip < eofMessage.AmountOfChunks)
{
//best way is to get the chunks in chunks. If a file exists out of 1000 * 1Mb files and load that into memory, you are gonna have a bad time
var chunks = await _fileChunkRepository.GetFileChunkData(eofMessage.FileId, toSkip, toTake, _database);
foreach (var chunk in chunks.Data)
{
await sw.WriteAsync(chunk, 0, chunk.Length);
}
toSkip = toTake + 1;
toTake += 10;
}
WorkerLog.Instance.Information($"File built to '{temppath}'");
return(temppath);
}
catch (Exception ex)
{
WorkerLog.Instance.Error(ex, $"Error occured while assembling {(file.IsNew() ? "" : "delta")} file '{file.Name}' to tempfile");
return(null);
}
}
public async Task ProcessFile_NewFile_QueryToGetChunkShouldGetCorrectParameters()
{
var configMock = CreateConfigurationMock();
var domainFile = Helper.GetFileModel();
var eofMessage = new EofMessage(Guid.Empty, 15);
var databaseMock = new Mock <IDatabase>();
var toSkipSum = 0;
var toTakeSum = 0;
var fileChunkRepositoryMock = new Mock <IFileChunkRepository>();
fileChunkRepositoryMock.Setup(x => x.GetFileChunkData(It.IsAny <Guid>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <IDatabase>()))
.Returns((Guid x, int y, int z, IDatabase d) =>
{
toSkipSum += y;
toTakeSum += z;
return(Task.FromResult(Result <ICollection <byte[]> > .Success(new List <byte[]>())));
});
fileChunkRepositoryMock.Setup(x => x.DeleteChunksForFile(It.IsAny <Guid>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result.Success()));
var fileRepositoryMock = new Mock <IFileRepository>();
fileRepositoryMock.Setup(x => x.GetAllVersionsOfFile(It.IsAny <File>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result <ICollection <File> > .Success(new List <File>())));
fileRepositoryMock.Setup(x => x.UpdateFileAsAssembled(It.IsAny <Guid>(), It.IsAny <DateTime>(), It.IsAny <byte[]>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result.Success()));
var fileAssemblerService = new FileAssemblerService(databaseMock.Object, fileRepositoryMock.Object, fileChunkRepositoryMock.Object);
await fileAssemblerService.ProcessFile(domainFile, eofMessage);
Assert.AreEqual(11, toSkipSum);
}
public async Task ProcessFile_ChangedFile_ApplyDeltaChouldBeCalled()
{
var configMock = CreateConfigurationMock();
var domainFile = Helper.GetFileModel();
domainFile.Update(new System.IO.FileInfo(System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "DropLocation", "dummy.txt")));
var eofMessage = new EofMessage(Guid.Empty, 15);
var databaseMock = new Mock <IDatabase>();
var fileChunkRepositoryMock = new Mock <IFileChunkRepository>();
fileChunkRepositoryMock.Setup(x => x.GetFileChunkData(It.IsAny <Guid>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result <ICollection <byte[]> > .Success(new List <byte[]>())));
fileChunkRepositoryMock.Setup(x => x.DeleteChunksForFile(It.IsAny <Guid>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result.Success()));
var fileRepositoryMock = new Mock <IFileRepository>();
fileRepositoryMock.Setup(x => x.GetAllVersionsOfFile(It.IsAny <File>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result <ICollection <File> > .Success(new List <File>())));
fileRepositoryMock.Setup(x => x.UpdateFileAsAssembled(It.IsAny <Guid>(), It.IsAny <DateTime>(), It.IsAny <byte[]>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result.Success()));
var fileAssemblerService = new FileAssemblerService(databaseMock.Object, fileRepositoryMock.Object, fileChunkRepositoryMock.Object);
await fileAssemblerService.ProcessFile(domainFile, eofMessage);
Assert.IsFalse(domainFile.IsNew());
}
public async Task ProcessFile_NewFile_FinalizationShouldBeCalled()
{
var configMock = CreateConfigurationMock();
var domainFile = Helper.GetFileModel();
var eofMessage = new EofMessage(Guid.Empty, 15);
var databaseMock = new Mock <IDatabase>();
var fileChunkRepositoryMock = new Mock <IFileChunkRepository>();
fileChunkRepositoryMock.Setup(x => x.GetFileChunkData(It.IsAny <Guid>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result <ICollection <byte[]> > .Success(new List <byte[]>())));
var fileChunkDeleteCalled = false;
fileChunkRepositoryMock.Setup(x => x.DeleteChunksForFile(It.IsAny <Guid>(), It.IsAny <IDatabase>()))
.Callback(() => fileChunkDeleteCalled = true)
.Returns(() => Task.FromResult(Result.Success()));
var fileRepositoryMock = new Mock <IFileRepository>();
fileRepositoryMock.Setup(x => x.GetAllVersionsOfFile(It.IsAny <File>(), It.IsAny <IDatabase>()))
.Returns(() => Task.FromResult(Result <ICollection <File> > .Success(new List <File>())));
var updateFileCalled = false;
fileRepositoryMock.Setup(x => x.UpdateFileAsAssembled(It.IsAny <Guid>(), It.IsAny <DateTime>(), It.IsAny <byte[]>(), It.IsAny <IDatabase>()))
.Callback(() => updateFileCalled = true)
.Returns(() => Task.FromResult(Result.Success()));
var fileAssemblerService = new FileAssemblerService(databaseMock.Object, fileRepositoryMock.Object, fileChunkRepositoryMock.Object);
await fileAssemblerService.ProcessFile(domainFile, eofMessage);
Assert.IsTrue(updateFileCalled);
Assert.IsTrue(fileChunkDeleteCalled);
}
public async Task SendEofMessage(Recipient recipient, EofMessage message)
{
var httpClient = _httpClientFactory.CreateClient("default");
var endpoint = $"{recipient.Address}/api/file/{message.FileId}/eot";
var eotModel = new EofMessageTransmissionModel {
AmountOfChunks = message.AmountOfChunks
};
var response = await httpClient.PostAsync(endpoint, eotModel, throwErrorOnResponseNok : true);
response.EnsureSuccessStatusCode();
}
public async Task SendEofMessage(Recipient recipient, EofMessage message)
{
var client = new Transmitter.TransmitterClient(_channelStore.Get(recipient.Address));
var result = await client.SendEofMessageAsync(new EofMessageTransmissionRequest
{
FileId = message.FileId.ToString(),
AmountOfChunks = message.AmountOfChunks
});
if (!result.IsSuccessful)
{
throw new InvalidOperationException($"'{nameof(SendEofMessage)}' received a negative 'IsSuccessful' from '{recipient.Name}'");
}
}
public async Task <bool> SendEofMessage(Recipient recipient, EofMessage message)
{
var canContinue = true;
try
{
WorkerLog.Instance.Information($"Sending Eot message to {recipient.Name}");
await _transmissionLink.SendEofMessage(recipient, message);
}
catch (System.Exception ex)
{
WorkerLog.Instance.Error(ex, $"Failed to send Eof message to '{recipient.Name}'. Adding file to failed transmissions and retrying later");
var result = await _transmissionRepository.AddFailedEofMessageTransmission(message.Id, recipient.Id);
canContinue = result.WasSuccessful;
}
return(canContinue);
}
private async Task ProcessChange(File file, EofMessage eofMessage)
{
var tempPath = await BuildFile(file, eofMessage);
if (tempPath is null)
{
return;
}
var filePath = file.GetFullPath();
if (System.IO.File.Exists(filePath) && !FileLock.IsLocked(filePath, checkWriteAccess: true))
{
WorkerLog.Instance.Information($"Applying delta to {filePath}");
file.ApplyDelta(System.IO.File.ReadAllBytes(tempPath));
await MarkFileAsCompleted(file);
}
else
{
WorkerLog.Instance.Warning($"File '{filePath}' does not exist or is locked for writing. unable to apply delta");
}
DeleteTempPath(tempPath);
}
public async Task <EofMessage> ProcessFile()
{
var path = _file.GetFullPath();
EofMessage eofMessage = null;
if (!string.IsNullOrWhiteSpace(path) && System.IO.File.Exists(path) && !FileLock.IsLocked(path))
{
try
{
if (_file.IsNew())
{
eofMessage = await ProcessNewFile();
}
else
{
eofMessage = await ProcessChangedFile();
}
}
catch (Exception ex)
{
WorkerLog.Instance.Error(ex, $"Unexpected error occured while disassembling file '{_file.Path}'");
}
}
else
{
WorkerLog.Instance.Warning($"File '{path}' does not exist or is locked. File will not be processed");
}
if (eofMessage is null)
{
await _fileRepository.UpdateFileAsFailed(_file.Id);
}
return(eofMessage);
}
public async Task <bool> ProcessFile(File file, EofMessage eofMessage)
{
try
{
//this class has alot of access to database. Just open 1 connection instead of every call creating its own
using (_database)
{
if (file.IsNew())
{
await ProcessNew(file, eofMessage);
}
else
{
await ProcessChange(file, eofMessage);
}
}
return(true);
}
catch (Exception ex)
{
WorkerLog.Instance.Error(ex, $"Unexpected error occured while assembling the file '{file.Name}'");
return(false);
}
}
