private async Task StorePendingObjects(PendingBucketObjectKeyCollection pendingObjectKeyCollection, CancellationToken cancellationToken)
{
foreach (var pendingObjectKey in pendingObjectKeyCollection)
{
pendingObjectKey.Status = (int)BucketObjectStatus.Detected;
pendingObjectKey.CurrentStatusErrorCount = 0;
pendingObjectKey.CurrentStatusLastError = null;
}
await this.repository.SaveItemBatchInternalAsync(pendingObjectKeyCollection, cancellationToken);
}
protected async Task <PendingBucketObjectKeyCollection> GetPendingObjectKeyCollectionInternal(string lastObjectMarkerKey, StockMarket sourceBucketType, int maxNumberOfPendingObject, CancellationToken cancellationToken)
{
var pendingObjectKeyCollection = new PendingBucketObjectKeyCollection();
pendingObjectKeyCollection.LastMarker = await this.repository.GetApplicationProperty(lastObjectMarkerKey, cancellationToken);
var bucketName = this.GetBucketName(sourceBucketType);
var listObjectResponse = default(ListObjectsResponse);
var maxLoop = 100;
var currentLoop = 0;
do
{
var listObjectRequest = new ListObjectsRequest
{
BucketName = bucketName,
MaxKeys = maxNumberOfPendingObject,
Marker = pendingObjectKeyCollection.LastMarker
};
listObjectResponse = await amazonS3.ListObjectsAsync(listObjectRequest, cancellationToken);
var bucketObjectStatusCollection = await this.GetBucketObjectStatusCollection(sourceBucketType, listObjectResponse.S3Objects, cancellationToken);
pendingObjectKeyCollection.AddRange(bucketObjectStatusCollection.Where(e => e.Status == (int)BucketObjectStatus.Unknown));
pendingObjectKeyCollection.IsTruncated = listObjectResponse.IsTruncated;
if (listObjectResponse.IsTruncated)
{
pendingObjectKeyCollection.LastMarker = listObjectResponse.NextMarker;
}
currentLoop += 1;
} while (
pendingObjectKeyCollection.Count < maxNumberOfPendingObject &&
listObjectResponse.S3Objects.Any() &&
listObjectResponse.IsTruncated &&
currentLoop < maxLoop);
return(pendingObjectKeyCollection);
}
