public async Task CopyObjectTest() { // ARRANGE CopyObjectRequest request = new CopyObjectRequest() { DestinationBucket = "mhaken", SourceBucket = "mhaken-lambda", SourceKey = "AWSAthenaUserMetrics/athena-metrics-636765132762278062.zip", DestinationKey = "test/file.txt", }; GetObjectMetadataRequest meta = new GetObjectMetadataRequest() { BucketName = request.SourceBucket, Key = request.SourceKey }; GetObjectMetadataResponse Meta = await client.GetObjectMetadataAsync(meta); // ACT CopyObjectResponse Response = await client.CopyOrMoveObjectAsync(request, 16777216, false); // ASSERT Assert.Equal(HttpStatusCode.OK, Response.HttpStatusCode); Assert.Equal(Meta.ETag, Response.ETag); }
private static async Task <CopyObjectResponse> Copy(string sourceBucket, string sourceKey, string destinationBucket, string prefixPattern, ILambdaContext context) { // The S3 key prefixes are separated with a forward slash string[] Parts = sourceKey.Split("/"); string DestinationKey = String.Format(prefixPattern, Parts); string DestinationUri = $"s3://{destinationBucket}/{DestinationKey}"; context.LogInfo($"Using destination: {DestinationUri}"); GetObjectTaggingRequest TagRequest = new GetObjectTaggingRequest() { BucketName = sourceBucket, Key = sourceKey }; GetObjectTaggingResponse TagResponse = await _S3Client.GetObjectTaggingAsync(TagRequest); CopyObjectRequest CopyRequest = new CopyObjectRequest() { DestinationBucket = destinationBucket, SourceBucket = sourceBucket, SourceKey = sourceKey, DestinationKey = DestinationKey, TagSet = TagResponse.Tagging }; CopyObjectResponse Response = await _S3Client.CopyOrMoveObjectAsync(CopyRequest, true); if (Response.HttpStatusCode == HttpStatusCode.OK) { context.LogInfo($"Successfully moved s3://{sourceBucket}/{sourceKey} to {DestinationUri}."); } else { context.LogError($"Unsuccessful copy of s3://{sourceBucket}/{sourceKey} to {DestinationUri} : ${(int)Response.HttpStatusCode}"); } return(Response); }
/// <summary> /// Performs a number of async copy object operations in parallel, all public methods should /// call this. /// </summary> /// <param name="client"></param> /// <param name="request"></param> /// <returns></returns> private static async Task <BulkCopyResponse> CoreBulkCopyAsync(this IAmazonS3 client, BulkCopyRequest request, bool deleteSource) { ParameterTests.NonNull(request, "request"); ParameterTests.OutOfRange(request.PartSize >= Constants.MINIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size must be at least {Constants.MINIMUM_MULTIPART_PART_SIZE} bytes."); ParameterTests.OutOfRange(request.PartSize <= Constants.MAXIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size cannot exceed {Constants.MAXIMUM_MULTIPART_PART_SIZE} bytes."); // Make sure there are not requests that have the same source and destination IEnumerable <CopyObjectRequest> errors = request.Requests .Where(x => x.SourceKey == x.DestinationKey && x.SourceBucket != null && x.SourceBucket.Equals(x.DestinationBucket, StringComparison.OrdinalIgnoreCase)); if (errors.Any()) { throw new SourceDestinationSameException($"The Bulk copy/move operation contained requests that had the same source and destination and could cause the accidential loss of data.", errors); } List <CopyObjectRequestResponse> responses = new List <CopyObjectRequestResponse>(); List <FailedCopyRequest> failures = new List <FailedCopyRequest>(); // Don't copy objects that have the same source and destination // object keys are case sensitive, but bucket names are not, they // are all supposed to be lower case IEnumerable <CopyObjectRequest> filtered = request.Requests.Where(x => !(x.SourceKey == x.DestinationKey && x.SourceBucket != null && x.SourceBucket.Equals(x.DestinationBucket, StringComparison.OrdinalIgnoreCase))); int counter = 0; //IEnumerable<> foreach (List <CopyObjectRequest> chunk in filtered.Chunk(request.MaxConcurrency)) { Debug.WriteLine($"Processing request chunk {++counter}."); List <Task <CopyObjectRequestResponse> > insideLoop = new List <Task <CopyObjectRequestResponse> >(); foreach (CopyObjectRequest req in chunk) { try { if (request.PreferMultipart) { insideLoop.Add(client.CopyOrMoveObjectAsync(req, request.PartSize, deleteSource, preferMultipartLogic)); } else { insideLoop.Add(client.CopyOrMoveObjectAsync(req, request.PartSize, deleteSource, standardMultipartLogic)); } } catch (Exception e) { failures.Add(new FailedCopyRequest(req, e, FailureMode.COPY)); } } try { IEnumerable <CopyObjectRequestResponse> responseChunk = await Task.WhenAll(insideLoop); responses.AddRange(responseChunk); } catch (Exception e) { failures.Add(new FailedCopyRequest(null, e, FailureMode.COPY)); } } try { var dict = responses.ToDictionary(x => x.Request, x => x.Response); return(new BulkCopyResponse(dict, failures)); } catch (Exception e) { return(null); } }