Пример #1
0
        public async Task PreProcessAsync_WhenModifiedCsv_PutObjectStreamMustNotBeCalled()
        {
            var data = new FilePreProcessorData()
            {
                Minutes     = 10,
                S3Locations = new string[] { "S3://test-bucket/location/" }
            };

            var s3ObjectModels = new List <S3ObjectModel>
            {
                new S3ObjectModel {
                    BucketName = "test-bucket", Key = "location/file-1.csv", LastModified = DateTime.UtcNow
                }
            };

            A.CallTo(() => _s3ClientService.ListObjectsAsync(A <string> .That.Matches(m => m.Equals(data.S3Locations[0]))))
            .Returns(Task.FromResult(s3ObjectModels));

            var s3ObjectStream = new MemoryStream();

            A.CallTo(() => _s3ClientService.GetObjectStream(A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].BucketName)), A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].Key)), A <string> .That.Matches(m => m == null)))
            .Returns(Task.FromResult(s3ObjectStream));

            var newLinesReplaced = 0;

            A.CallTo(() => _csvService.ReplaceNewLines(A <string> .That.Matches(m => m.Equals($"{ s3ObjectModels[0].BucketName}/{s3ObjectModels[0].Key}")), A <MemoryStream> .That.Matches(m => m.Equals(s3ObjectStream)), A <MemoryStream> .Ignored))
            .Returns(newLinesReplaced);

            var result = await _filePreProcessorService.PreProcessAsync(data);

            A.CallTo(() => _s3ClientService.PutObjectStream(A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].BucketName)), A <string> .That.Matches(m => m.Equals(s3ObjectModels[0].Key)), A <Stream> .Ignored))
            .MustNotHaveHappened();

            result.Should().BeTrue();
        }
Пример #2
0
        public async Task <bool> PreProcessAsync(FilePreProcessorData data)
        {
            var requestJson = data != null?JsonConvert.SerializeObject(data) : "";

            _logger.LogDebug($"Request: '{requestJson}'");

            if (data == null)
            {
                return(true);
            }

            if (data.Minutes <= 0)
            {
                return(true);
            }

            if ((data.S3Locations?.Any() ?? false) == false)
            {
                return(true);
            }

            var lastModified = DateTime.UtcNow.AddMinutes(data.Minutes * -1);

            foreach (var location in data.S3Locations)
            {
                var s3Objects = await _s3ClientService.ListObjectsAsync(location);

                var s3ObjectFiles = s3Objects
                                    .Where(w => !w.Key.EndsWith("/") && w.LastModified >= lastModified)
                                    .ToList();

                _logger.LogDebug($"PreProcessing '{s3ObjectFiles.Count}' files in location '{location}' newer than '{lastModified}'");

                foreach (var s3ObjectFile in s3ObjectFiles)
                {
                    _logger.LogDebug($"PreProcessing file: '{s3ObjectFile.BucketName}/{s3ObjectFile.Key}' (LastModified:'{s3ObjectFile.LastModified}', Size:'{s3ObjectFile.Size}')");

                    var s3ObjectStream = await _s3ClientService.GetObjectStream(s3ObjectFile.BucketName, s3ObjectFile.Key);

                    var memoryStream     = new MemoryStream();
                    var newLinesReplaced = _csvService.ReplaceNewLines($"{s3ObjectFile.BucketName}/{s3ObjectFile.Key}", s3ObjectStream, memoryStream);

                    var memoryStreamLength = memoryStream.Length;

                    _logger.LogDebug($"PreProcessing file: '{s3ObjectFile.BucketName}/{s3ObjectFile.Key}'. New Lines: '{newLinesReplaced}' replaced. (LastModified:'{s3ObjectFile.LastModified}', Size:'{s3ObjectFile.Size}')");

                    if (newLinesReplaced <= 0)
                    {
                        continue;
                    }

                    var result = await _s3ClientService.PutObjectStream(s3ObjectFile.BucketName, s3ObjectFile.Key, memoryStream);

                    _logger.LogDebug($"PreProcessing file: '{s3ObjectFile.BucketName}/{s3ObjectFile.Key}'. New Lines: '{newLinesReplaced}' replaced, New Size: '{memoryStreamLength}'. (LastModified:'{s3ObjectFile.LastModified}', Size:'{s3ObjectFile.Size}')");
                }
            }

            return(await Task.FromResult(true));
        }