public async Task CopyObjectTest()
{
// ARRANGE
CopyObjectRequest request = new CopyObjectRequest()
{
DestinationBucket = "mhaken",
SourceBucket = "mhaken-lambda",
SourceKey = "AWSAthenaUserMetrics/athena-metrics-636765132762278062.zip",
DestinationKey = "test/file.txt",
};
GetObjectMetadataRequest meta = new GetObjectMetadataRequest()
{
BucketName = request.SourceBucket,
Key = request.SourceKey
};
GetObjectMetadataResponse Meta = await client.GetObjectMetadataAsync(meta);
// ACT
CopyObjectResponse Response = await client.CopyOrMoveObjectAsync(request, 16777216, false);
// ASSERT
Assert.Equal(HttpStatusCode.OK, Response.HttpStatusCode);
Assert.Equal(Meta.ETag, Response.ETag);
}
private static async Task <CopyObjectResponse> Copy(string sourceBucket, string sourceKey, string destinationBucket, string prefixPattern, ILambdaContext context)
{
// The S3 key prefixes are separated with a forward slash
string[] Parts = sourceKey.Split("/");
string DestinationKey = String.Format(prefixPattern, Parts);
string DestinationUri = $"s3://{destinationBucket}/{DestinationKey}";
context.LogInfo($"Using destination: {DestinationUri}");
GetObjectTaggingRequest TagRequest = new GetObjectTaggingRequest()
{
BucketName = sourceBucket,
Key = sourceKey
};
GetObjectTaggingResponse TagResponse = await _S3Client.GetObjectTaggingAsync(TagRequest);
CopyObjectRequest CopyRequest = new CopyObjectRequest()
{
DestinationBucket = destinationBucket,
SourceBucket = sourceBucket,
SourceKey = sourceKey,
DestinationKey = DestinationKey,
TagSet = TagResponse.Tagging
};
CopyObjectResponse Response = await _S3Client.CopyOrMoveObjectAsync(CopyRequest, true);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully moved s3://{sourceBucket}/{sourceKey} to {DestinationUri}.");
}
else
{
context.LogError($"Unsuccessful copy of s3://{sourceBucket}/{sourceKey} to {DestinationUri} : ${(int)Response.HttpStatusCode}");
}
return(Response);
}
/// <summary>
/// Performs a number of async copy object operations in parallel, all public methods should
/// call this.
/// </summary>
/// <param name="client"></param>
/// <param name="request"></param>
/// <returns></returns>
private static async Task <BulkCopyResponse> CoreBulkCopyAsync(this IAmazonS3 client, BulkCopyRequest request, bool deleteSource)
{
ParameterTests.NonNull(request, "request");
ParameterTests.OutOfRange(request.PartSize >= Constants.MINIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size must be at least {Constants.MINIMUM_MULTIPART_PART_SIZE} bytes.");
ParameterTests.OutOfRange(request.PartSize <= Constants.MAXIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size cannot exceed {Constants.MAXIMUM_MULTIPART_PART_SIZE} bytes.");
// Make sure there are not requests that have the same source and destination
IEnumerable <CopyObjectRequest> errors = request.Requests
.Where(x => x.SourceKey == x.DestinationKey && x.SourceBucket != null && x.SourceBucket.Equals(x.DestinationBucket, StringComparison.OrdinalIgnoreCase));
if (errors.Any())
{
throw new SourceDestinationSameException($"The Bulk copy/move operation contained requests that had the same source and destination and could cause the accidential loss of data.", errors);
}
List <CopyObjectRequestResponse> responses = new List <CopyObjectRequestResponse>();
List <FailedCopyRequest> failures = new List <FailedCopyRequest>();
// Don't copy objects that have the same source and destination
// object keys are case sensitive, but bucket names are not, they
// are all supposed to be lower case
IEnumerable <CopyObjectRequest> filtered = request.Requests.Where(x => !(x.SourceKey == x.DestinationKey && x.SourceBucket != null && x.SourceBucket.Equals(x.DestinationBucket, StringComparison.OrdinalIgnoreCase)));
int counter = 0;
//IEnumerable<>
foreach (List <CopyObjectRequest> chunk in filtered.Chunk(request.MaxConcurrency))
{
Debug.WriteLine($"Processing request chunk {++counter}.");
List <Task <CopyObjectRequestResponse> > insideLoop = new List <Task <CopyObjectRequestResponse> >();
foreach (CopyObjectRequest req in chunk)
{
try
{
if (request.PreferMultipart)
{
insideLoop.Add(client.CopyOrMoveObjectAsync(req, request.PartSize, deleteSource, preferMultipartLogic));
}
else
{
insideLoop.Add(client.CopyOrMoveObjectAsync(req, request.PartSize, deleteSource, standardMultipartLogic));
}
}
catch (Exception e)
{
failures.Add(new FailedCopyRequest(req, e, FailureMode.COPY));
}
}
try
{
IEnumerable <CopyObjectRequestResponse> responseChunk = await Task.WhenAll(insideLoop);
responses.AddRange(responseChunk);
}
catch (Exception e)
{
failures.Add(new FailedCopyRequest(null, e, FailureMode.COPY));
}
}
try
{
var dict = responses.ToDictionary(x => x.Request, x => x.Response);
return(new BulkCopyResponse(dict, failures));
}
catch (Exception e)
{
return(null);
}
}
