async Task Fetch()
{
var isEmpty = false;
while (!isEmpty)
{
LogInfo("Downloading from " + Account.S3Bucket);
var request = new Amazon.S3.Model.ListObjectsV2Request {
BucketName = Account.S3Bucket
};
var response = await S3Client.ListObjectsV2Async(request);
LogInfo($"Downloaded {response.S3Objects.Count} items from " + Account.S3Bucket);
foreach (var item in response.S3Objects)
{
await Fetch(item);
}
Log.For(this).Info($"Downloaded {response.S3Objects.Count} items from " + Account.S3Bucket);
isEmpty = response.NextContinuationToken.IsEmpty();
if (isEmpty)
{
Log.For("Downloaded all the objects from " + Account.S3Bucket);
break;
}
}
}
private IEnumerable <(string ContentName, long?Size)> InternalGetContents(string prefix)
{
var request = new Amazon.S3.Model.ListObjectsV2Request {
BucketName = _bucketName, Prefix = MakeKey(prefix)
};
var containerDepth = _directoryName.Split('/').Length - 1;
while (true)
{
var response = _amazonS3.ListObjectsV2Async(request).Result;
var contents =
response.S3Objects
.Select(x => (ContentName: string.Join("/", x.Key.Split('/').Skip(1 + containerDepth)), Size: x.Size))
.Where(x => !string.IsNullOrEmpty(x.ContentName));
foreach (var content in contents)
{
yield return(content);
}
if (!response.IsTruncated)
{
break;
}
request.ContinuationToken = response.NextContinuationToken;
}
}
private async Task <List <Amazon.S3.Model.S3Object> > ListObjectsAsync(string bucket, string prefix, CancellationToken ct = default)
{
var request = new Amazon.S3.Model.ListObjectsV2Request
{
BucketName = bucket,
Prefix = prefix,
};
var objects = new List <Amazon.S3.Model.S3Object>();
Amazon.S3.Model.ListObjectsV2Response response;
do
{
response = await _client.ListObjectsV2Async(request, ct);
objects.AddRange(response.S3Objects);
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
return(objects);
}
