/// <summary>
/// Retrieves and zip objects from a s3 bucket and zip them up, uploading the zip file back into s3
/// </summary>
/// <param name="s3FolderName">Name of the S3 bucket</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <bool> ZipBucket(string s3FolderName, string s3ZipFileName, CancellationToken cancellationToken)
{
#pragma warning disable IDE0063 // Use simple 'using' statement
using (var objectZipper = new ObjectZipper(config.TempZipPath, config.ZlibCompressionLevel, _logger))
#pragma warning restore IDE0063 // Use simple 'using' statement
{
//Create temp zip file in zip directory
objectZipper.CreateZip();
List <Task> fetchObjectsTasks = new List <Task>();
ConcurrentBag <IAsyncEnumerable <Models.S3Object> > cb = new ConcurrentBag <IAsyncEnumerable <Models.S3Object> >();
// Create tasks for downloading objects
await foreach (var keys in _s3ClientProxy.ListObjectsAsStream(config.S3BucketName, s3FolderName, cancellationToken))
{
var filterKeys = new List <string>();
if (filterOutFiles == null)
{
filterKeys = keys;
}
else
{
filterKeys = keys.Where(filterOutFiles).ToList();
}
fetchObjectsTasks.Add(Task.Run(() => cb.Add(_s3ClientProxy.FetchObjectsAsStream(config.S3BucketName, s3FolderName, filterKeys, cancellationToken))));
}
Task.WaitAll(fetchObjectsTasks.ToArray(), cancellationToken);
// Download and zip the objects
List <Task> consumeTasks = new List <Task>();
while (!cb.IsEmpty)
{
consumeTasks.Add(Task.Run(async() =>
{
if (cb.TryTake(out IAsyncEnumerable <S3Object> s3Object))
{
await foreach (var obj in s3Object)
{
objectZipper.ZipObject(obj.Key, obj.Content);
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
}));
}
Task.WaitAll(consumeTasks.ToArray(), cancellationToken);
await _s3ClientProxy.UploadZipAsync(config.TempZipPath, config.S3BucketName, s3ZipFileName);
return(true);
}
}
public async Task ShouldGet2ObjectsFromFetchObjectsAsStream()
{
List <string> lsKeys = new List <string>()
{
"test.txt", "test2.txt"
};
List <Models.S3Object> result = new List <Models.S3Object>();
await foreach (var obj in s3Proxy.FetchObjectsAsStream("", "", lsKeys, new CancellationToken()))
{
result.Add(obj);
}
Assert.AreEqual(2, result.Count);
for (int i = 0; i < result.Count - 1; i++)
{
Assert.AreEqual(lsKeys[i], result[i].Key);
Assert.NotNull(result[i].Content);
}
}