/// <summary>
/// Performs a number of async copy object operations in parallel and then a smaller
/// number of delete operations after the copies complete
/// </summary>
/// <param name="client"></param>
/// <param name="request"></param>
/// <returns></returns>
private static async Task <BulkCopyResponse> MoveWithBulkDeleteAsync(this IAmazonS3 client, BulkMoveRequest request)
{
List <FailedCopyRequest> failures = new List <FailedCopyRequest>();
// This method checks for same source/destination problems and will throw an exception
BulkCopyResponse responses = await client.CoreBulkCopyAsync(request, false);
// Max keys in a request is 1000
// Make sure that we don't delete objects that
// were moved to that same location, this shouldn't
// happen because of the same logic in the copy operation
// but make sure
foreach (IEnumerable <KeyValuePair <CopyObjectRequest, CopyObjectResponse> > responseSet in
responses.SuccessfulOperations.Where(x => !(x.Key.SourceKey == x.Key.DestinationKey &&
x.Key.SourceBucket != null &&
x.Key.SourceBucket.Equals(x.Key.DestinationBucket, StringComparison.OrdinalIgnoreCase))).Chunk(1000))
{
DeleteObjectsRequest delete = new DeleteObjectsRequest()
{
BucketName = request.Requests.First().SourceBucket,
Objects = responseSet.Select(x => new KeyVersion()
{
Key = x.Key.SourceKey, VersionId = x.Key.SourceVersionId
}).ToList()
};
try
{
DeleteObjectsResponse response = await client.DeleteObjectsAsync(delete);
// Find the delete errors and create a new failed copy request
// object for each one
List <FailedCopyRequest> deleteFailures = response.DeleteErrors
.Select(x => new FailedCopyRequest(
responseSet.First(y => y.Key.SourceKey == x.Key).Key,
new AmazonS3Exception(x.Message)
{
ErrorCode = x.Code
},
FailureMode.DELETE)
).ToList();
// Remove any items that were successful in the copy
// but failed to delete from the successful responses
// list and indicate they failed during delete
foreach (FailedCopyRequest failure in deleteFailures)
{
responses.SuccessfulOperations.Remove(failure.Request);
}
foreach (FailedCopyRequest fail in deleteFailures)
{
responses.FailedRequests.Add(fail);
}
}
catch (Exception e)
{
// Remove all the copy responses from the success
// group and make them failures when an exception occurs
foreach (KeyValuePair <CopyObjectRequest, CopyObjectResponse> item in responseSet)
{
responses.SuccessfulOperations.Remove(item.Key);
responses.FailedRequests.Add(new FailedCopyRequest(item.Key, e, FailureMode.DELETE));
}
}
}
return(responses);
}
