/// <summary>
/// Initiates a parallel copy operation for all of the included requests. The source objects
/// are either deleted in a Bulk-type operation or individually as each copy completes.
/// </summary>
/// <param name="client">The S3 client to use</param>
/// <param name="requests">The copy requests to process</param>
/// <param name="partSize">The size of the part to use for a multipart copy.</param>
/// <param name="preferMultipart">If set to true, the method will use a multipart copy as long as the part size is less than the object size for any object, even
/// those under 5 GiB.</param>
/// <param name="useBulkDelete">If set to true, the objects will be deleted using request Bulkes of 1000 keys per
/// request. If set to false, each object will be deleted individually after it is copied</param>
/// <returns>The copy object responses.</returns>
public static async Task <BulkCopyResponse> BulkMoveAsync(this IAmazonS3 client, IEnumerable <CopyObjectRequest> requests, long partSize, bool useBulkDelete = true, bool preferMultipart = false)
{
BulkMoveRequest request = new BulkMoveRequest(requests)
{
PartSize = partSize,
BulkDelete = useBulkDelete,
PreferMultipart = preferMultipart
};
return(await BulkMoveAsync(client, request));
}
/// <summary>
/// Performs a number of async copy object operations in parallel and then a smaller
/// number of delete operations after the copies complete
/// </summary>
/// <param name="client"></param>
/// <param name="request"></param>
/// <returns></returns>
private static async Task <BulkCopyResponse> MoveWithBulkDeleteAsync(this IAmazonS3 client, BulkMoveRequest request)
{
List <FailedCopyRequest> failures = new List <FailedCopyRequest>();
// This method checks for same source/destination problems and will throw an exception
BulkCopyResponse responses = await client.CoreBulkCopyAsync(request, false);
// Max keys in a request is 1000
// Make sure that we don't delete objects that
// were moved to that same location, this shouldn't
// happen because of the same logic in the copy operation
// but make sure
foreach (IEnumerable <KeyValuePair <CopyObjectRequest, CopyObjectResponse> > responseSet in
responses.SuccessfulOperations.Where(x => !(x.Key.SourceKey == x.Key.DestinationKey &&
x.Key.SourceBucket != null &&
x.Key.SourceBucket.Equals(x.Key.DestinationBucket, StringComparison.OrdinalIgnoreCase))).Chunk(1000))
{
DeleteObjectsRequest delete = new DeleteObjectsRequest()
{
BucketName = request.Requests.First().SourceBucket,
Objects = responseSet.Select(x => new KeyVersion()
{
Key = x.Key.SourceKey, VersionId = x.Key.SourceVersionId
}).ToList()
};
try
{
DeleteObjectsResponse response = await client.DeleteObjectsAsync(delete);
// Find the delete errors and create a new failed copy request
// object for each one
List <FailedCopyRequest> deleteFailures = response.DeleteErrors
.Select(x => new FailedCopyRequest(
responseSet.First(y => y.Key.SourceKey == x.Key).Key,
new AmazonS3Exception(x.Message)
{
ErrorCode = x.Code
},
FailureMode.DELETE)
).ToList();
// Remove any items that were successful in the copy
// but failed to delete from the successful responses
// list and indicate they failed during delete
foreach (FailedCopyRequest failure in deleteFailures)
{
responses.SuccessfulOperations.Remove(failure.Request);
}
foreach (FailedCopyRequest fail in deleteFailures)
{
responses.FailedRequests.Add(fail);
}
}
catch (Exception e)
{
// Remove all the copy responses from the success
// group and make them failures when an exception occurs
foreach (KeyValuePair <CopyObjectRequest, CopyObjectResponse> item in responseSet)
{
responses.SuccessfulOperations.Remove(item.Key);
responses.FailedRequests.Add(new FailedCopyRequest(item.Key, e, FailureMode.DELETE));
}
}
}
return(responses);
}
/// <summary>
/// A simple wrapper to identify whether deletes will be Bulked or not, all
/// public methods should call this
/// </summary>
/// <param name="client"></param>
/// <param name="request"></param>
/// <returns></returns>
private static async Task <BulkCopyResponse> MoveBulkAsnyc(this IAmazonS3 client, BulkMoveRequest request)
{
if (request.BulkDelete)
{
return(await MoveWithBulkDeleteAsync(client, request));
}
else
{
return(await CoreBulkCopyAsync(client, request, true));
}
}
/// <summary>
/// Initiates a parallel copy operation for all of the included requests.
/// </summary>
/// <param name="client">The Amazon S3 Client to use</param>
/// <param name="request">The Bulk move request to process</param>
/// <returns>The copy object responses.</returns>
public static async Task <BulkCopyResponse> BulkMoveAsync(this IAmazonS3 client, BulkMoveRequest request)
{
return(await MoveBulkAsnyc(client, request));
}
public async Task BulkMoveItems()
{
// ARRANGE
string bucket = sourceBucket;
//string bucket = "mhaken-9fc61dff-1893-42cc-a82f-8024960b158e";
Stopwatch sw = new Stopwatch();
int count = 10000;
try
{
sw.Start();
await CreateAndFillBucket(count);
sw.Stop();
Debug.Print($"Finished creating and filling bucket {bucket} in {sw.Elapsed}.");
sw.Reset();
ListObjectsV2Request list = new ListObjectsV2Request()
{
BucketName = bucket
};
sw.Start();
IEnumerable <S3Object> objects = (await client.ListAllObjectsAsync(list)).SelectMany(x => x.S3Objects);
// ACT
BulkMoveRequest request = new BulkMoveRequest(objects.Select(x => new CopyObjectRequest()
{
SourceBucket = x.BucketName,
DestinationBucket = x.BucketName,
SourceKey = x.Key,
DestinationKey = "moved_" + x.Key,
StorageClass = S3StorageClass.OneZoneInfrequentAccess,
}))
{
PreferMultipart = true
};
BulkCopyResponse response = await client.BulkMoveAsync(request);
sw.Stop();
File.WriteAllText("results.txt", $"Successfully moved {count} items in {sw.Elapsed}.");
// ASSERT
Assert.Equal(objects.Count(), response.SuccessfulOperations.Count);
}
catch (Exception e)
{
}
finally
{
await client.EmptyBucketAsync(bucket);
await client.DeleteBucketAsync(bucket);
}
}
