public async Task PreProcessAsync_WhenModifiedCsv_PutObjectStreamMustBeCalled()
{
var data = new FilePreProcessorData()
{
Minutes = 10,
S3Locations = new string[] { "S3://test-bucket/location/" }
};
var s3ObjectModels = new List <S3ObjectModel>
{
new S3ObjectModel {
BucketName = "test-bucket", Key = "location/file-1.csv", LastModified = DateTime.UtcNow
}
};
A.CallTo(() => _s3ClientService.ListObjectsAsync(A <string> .That.Matches(m => m.Equals(data.S3Locations[0]))))
.Returns(Task.FromResult(s3ObjectModels));
var s3ObjectStream = new MemoryStream();
A.CallTo(() => _s3ClientService.GetObjectStream(A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].BucketName)), A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].Key)), A <string> .That.Matches(m => m == null)))
.Returns(Task.FromResult(s3ObjectStream));
var newLinesReplaced = 4;
A.CallTo(() => _csvService.ReplaceNewLines(A <string> .That.Matches(m => m.Equals($"{ s3ObjectModels[0].BucketName}/{s3ObjectModels[0].Key}")), A <MemoryStream> .That.Matches(m => m.Equals(s3ObjectStream)), A <MemoryStream> .Ignored))
.Returns(newLinesReplaced);
var result = await _filePreProcessorService.PreProcessAsync(data);
A.CallTo(() => _s3ClientService.PutObjectStream(A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].BucketName)), A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].Key)), A <Stream> .Ignored))
.MustHaveHappenedOnceExactly();
result.Should().BeTrue();
}
public async Task <bool> PreProcessAsync(FilePreProcessorData data)
{
var requestJson = data != null?JsonConvert.SerializeObject(data) : "";
_logger.LogDebug($"Request: '{requestJson}'");
if (data == null)
{
return(true);
}
if (data.Minutes <= 0)
{
return(true);
}
if ((data.S3Locations?.Any() ?? false) == false)
{
return(true);
}
var lastModified = DateTime.UtcNow.AddMinutes(data.Minutes * -1);
foreach (var location in data.S3Locations)
{
var s3Objects = await _s3ClientService.ListObjectsAsync(location);
var s3ObjectFiles = s3Objects
.Where(w => !w.Key.EndsWith("/") && w.LastModified >= lastModified)
.ToList();
_logger.LogDebug($"PreProcessing '{s3ObjectFiles.Count}' files in location '{location}' newer than '{lastModified}'");
foreach (var s3ObjectFile in s3ObjectFiles)
{
_logger.LogDebug($"PreProcessing file: '{s3ObjectFile.BucketName}/{s3ObjectFile.Key}' (LastModified:'{s3ObjectFile.LastModified}', Size:'{s3ObjectFile.Size}')");
var s3ObjectStream = await _s3ClientService.GetObjectStream(s3ObjectFile.BucketName, s3ObjectFile.Key);
var memoryStream = new MemoryStream();
var newLinesReplaced = _csvService.ReplaceNewLines($"{s3ObjectFile.BucketName}/{s3ObjectFile.Key}", s3ObjectStream, memoryStream);
var memoryStreamLength = memoryStream.Length;
_logger.LogDebug($"PreProcessing file: '{s3ObjectFile.BucketName}/{s3ObjectFile.Key}'. New Lines: '{newLinesReplaced}' replaced. (LastModified:'{s3ObjectFile.LastModified}', Size:'{s3ObjectFile.Size}')");
if (newLinesReplaced <= 0)
{
continue;
}
var result = await _s3ClientService.PutObjectStream(s3ObjectFile.BucketName, s3ObjectFile.Key, memoryStream);
_logger.LogDebug($"PreProcessing file: '{s3ObjectFile.BucketName}/{s3ObjectFile.Key}'. New Lines: '{newLinesReplaced}' replaced, New Size: '{memoryStreamLength}'. (LastModified:'{s3ObjectFile.LastModified}', Size:'{s3ObjectFile.Size}')");
}
}
return(await Task.FromResult(true));
}