/// <summary> /// Start the asynchronous request for copying a part. /// </summary> /// <param name="request">Parameters in a request for copying a part</param> /// <param name="callback">Asynchronous request callback function</param> /// <param name="state">Asynchronous request status object</param> /// <returns>Response to the asynchronous request</returns> public IAsyncResult BeginCopyPart(CopyPartRequest request, AsyncCallback callback, object state) { return(this.BeginDoRequest <CopyPartRequest>(request, delegate() { if (request.ObjectKey == null) { throw new ObsException(Constants.InvalidObjectKeyMessage, ErrorType.Sender, Constants.InvalidObjectKey, ""); } if (string.IsNullOrEmpty(request.UploadId)) { throw new ObsException(Constants.InvalidUploadIdMessage, ErrorType.Sender, Constants.InvalidUploadId, ""); } if (request.PartNumber <= 0) { throw new ObsException(Constants.InvalidPartNumberMessage, ErrorType.Sender, Constants.InvalidPartNumber, ""); } if (string.IsNullOrEmpty(request.SourceBucketName)) { throw new ObsException(Constants.InvalidSourceBucketNameMessage, ErrorType.Sender, Constants.InvalidBucketName, ""); } if (request.SourceObjectKey == null) { throw new ObsException(Constants.InvalidSourceObjectKeyMessage, ErrorType.Sender, Constants.InvalidObjectKey, ""); } }, callback, state)); }
public void TestConvertRequest() { // ARRANGE CopyObjectRequest copyRequest = new CopyObjectRequest() { DestinationBucket = "destination-bucket", SourceBucket = "source-bucket", SourceKey = "source/key/test.txt", DestinationKey = "destination/key/test.txt", TagSet = new List <Tag>() { new Tag() { Key = "tag1", Value = "myval" } } }; MethodInfo dynMethod = typeof(AmazonS3ExtensionMethods).GetMethod("ConvertTo", BindingFlags.NonPublic | BindingFlags.Static); MethodInfo genericMethod = dynMethod.MakeGenericMethod(typeof(CopyPartRequest)); // ACT CopyPartRequest Request = (CopyPartRequest)genericMethod.Invoke(null, new object[] { copyRequest }); // ASSERT Assert.NotNull(Request); Assert.Equal(copyRequest.DestinationBucket, Request.DestinationBucket); Assert.Equal(copyRequest.SourceBucket, Request.SourceBucket); Assert.Equal(copyRequest.DestinationKey, Request.DestinationKey); Assert.Equal(copyRequest.SourceKey, Request.SourceKey); Assert.Equal(copyRequest.SourceVersionId, Request.SourceVersionId); }
/// <summary> /// Copy a part. /// </summary> /// <param name="request">Parameters in a request for copying a part</param> /// <returns> Response to a part copy request</returns> public CopyPartResponse CopyPart(CopyPartRequest request) { CopyPartResponse response = this.DoRequest <CopyPartRequest, CopyPartResponse>(request, delegate() { if (request.ObjectKey == null) { throw new ObsException(Constants.InvalidObjectKeyMessage, ErrorType.Sender, Constants.InvalidObjectKey, ""); } if (string.IsNullOrEmpty(request.UploadId)) { throw new ObsException(Constants.InvalidUploadIdMessage, ErrorType.Sender, Constants.InvalidUploadId, ""); } if (request.PartNumber <= 0) { throw new ObsException(Constants.InvalidPartNumberMessage, ErrorType.Sender, Constants.InvalidPartNumber, ""); } if (string.IsNullOrEmpty(request.SourceBucketName)) { throw new ObsException(Constants.InvalidSourceBucketNameMessage, ErrorType.Sender, Constants.InvalidBucketName, ""); } if (request.SourceObjectKey == null) { throw new ObsException(Constants.InvalidSourceObjectKeyMessage, ErrorType.Sender, Constants.InvalidObjectKey, ""); } }); response.PartNumber = request.PartNumber; return(response); }
static void Main(string[] args) { IAmazonS3 s3Client = new AmazonS3Client(Amazon.RegionEndpoint.USEast1); // List to store upload part responses. List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>(); List<CopyPartResponse> copyResponses = new List<CopyPartResponse>(); InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest { BucketName = targetBucket, Key = targetObjectKey }; InitiateMultipartUploadResponse initResponse = s3Client.InitiateMultipartUpload(initiateRequest); String uploadId = initResponse.UploadId; try { // Get object size. GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest { BucketName = sourceBucket, Key = sourceObjectKey }; GetObjectMetadataResponse metadataResponse = s3Client.GetObjectMetadata(metadataRequest); long objectSize = metadataResponse.ContentLength; // in bytes // Copy parts. long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB long bytePosition = 0; for (int i = 1; bytePosition < objectSize; i++) { CopyPartRequest copyRequest = new CopyPartRequest { DestinationBucket = targetBucket, DestinationKey = targetObjectKey, SourceBucket = sourceBucket, SourceKey = sourceObjectKey, UploadId = uploadId, FirstByte = bytePosition, LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1, PartNumber = i }; copyResponses.Add(s3Client.CopyPart(copyRequest)); bytePosition += partSize; } CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = targetBucket, Key = targetObjectKey, UploadId = initResponse.UploadId }; completeRequest.AddPartETags(copyResponses); CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest); } catch (Exception e) { Console.WriteLine(e.Message); } }
static void CopyPart() { try { ByteRange range = new ByteRange(10, 20); CopyPartRequest request = new CopyPartRequest() { SourceBucketName = bucketName, SourceObjectKey = objectName, SourceVersionId = versionId, ObjectKey = destobjectName, PartNumber = 1, UploadId = uploadId, ByteRange = range }; CopyPartResponse response = client.CopyPart(request); Console.WriteLine("Copy part response: {0}", response.StatusCode); Console.WriteLine("ETag: {0}", response.ETag); etag = response.ETag; } catch (ObsException ex) { Console.WriteLine("Exception errorcode: {0}, when copy part.", ex.ErrorCode); Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage); } }
/// <summary> /// 结束对复制段的异步请求。 /// </summary> /// <param name="ar">异步请求的响应结果。</param> /// <returns>复制段的响应结果。</returns> public CopyPartResponse EndCopyPart(IAsyncResult ar) { CopyPartResponse response = this.EndDoRequest <CopyPartRequest, CopyPartResponse>(ar); HttpObsAsyncResult result = ar as HttpObsAsyncResult; CopyPartRequest request = result.AdditionalState as CopyPartRequest; response.PartNumber = request.PartNumber; return(response); }
private async Task <CompleteMultipartUploadResponse> MultipartCopy(string sourceKey, string destinationKey, long objectSize, InitiateMultipartUploadRequest initiateRequest) { var copyResponses = new List <CopyPartResponse>(); var partSize = 5 * (long)Math.Pow(2, 20); // Part size is 5 MB. // Initiate the upload. var initResponse = await _s3Client.InitiateMultipartUploadAsync(initiateRequest); long bytePosition = 0; for (int i = 1; bytePosition < objectSize; i++) { var copyRequest = new CopyPartRequest { DestinationBucket = _bucket, DestinationKey = destinationKey, SourceBucket = _bucket, SourceKey = sourceKey, UploadId = initResponse.UploadId, FirstByte = bytePosition, LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1, PartNumber = i, }; copyResponses.Add(await _s3Client.CopyPartAsync(copyRequest)); bytePosition += partSize; } // Set up to complete the copy. var completeRequest = new CompleteMultipartUploadRequest { BucketName = _bucket, Key = destinationKey, UploadId = initResponse.UploadId }; completeRequest.AddPartETags(copyResponses); // Complete the copy. return(await _s3Client.CompleteMultipartUploadAsync(completeRequest)); }
public Task <CopyPartResponse> CopyPartAsync(CopyPartRequest request, CancellationToken cancellationToken = default) { throw new NotImplementedException(); }
private void TestCopyPart(string bucketName, string key, string keyId) { string dstKey = "dstObject"; string srcKey = key; string srcVersionID; string srcETag; DateTime srcTimeStamp; string uploadID = null; try { //Get the srcObjectTimestamp GetObjectMetadataResponse gomr = Client.GetObjectMetadata(new GetObjectMetadataRequest { BucketName = bucketName, Key = srcKey }); srcTimeStamp = gomr.LastModified; srcVersionID = gomr.VersionId; srcETag = gomr.ETag; //Start the multipart upload InitiateMultipartUploadResponse imur = Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest { BucketName = bucketName, Key = dstKey, ServerSideEncryptionMethod = ServerSideEncryptionMethod.AWSKMS, ServerSideEncryptionKeyManagementServiceKeyId = keyId }); Assert.AreEqual(ServerSideEncryptionMethod.AWSKMS, imur.ServerSideEncryptionMethod); var usedKeyId = imur.ServerSideEncryptionKeyManagementServiceKeyId; VerifyKeyId(keyId, usedKeyId); uploadID = imur.UploadId; CopyPartRequest request = new CopyPartRequest { DestinationBucket = bucketName, DestinationKey = dstKey, SourceBucket = bucketName, SourceKey = srcKey, UploadId = uploadID, PartNumber = 1, }; CopyPartResponse response = Client.CopyPart(request); Assert.AreEqual(ServerSideEncryptionMethod.AWSKMS, response.ServerSideEncryptionMethod); usedKeyId = response.ServerSideEncryptionKeyManagementServiceKeyId; VerifyKeyId(keyId, usedKeyId); //ETag Assert.IsNotNull(response.ETag); Assert.IsTrue((response.ETag != null) && (response.ETag.Length > 0)); //LastModified Assert.IsNotNull(response.LastModified); Assert.AreNotEqual(DateTime.MinValue, response.LastModified); //PartNumber Assert.IsTrue(response.PartNumber == 1); var completeResponse = Client.CompleteMultipartUpload(new CompleteMultipartUploadRequest { BucketName = bucketName, Key = dstKey, UploadId = uploadID, PartETags = new List <PartETag>() { new PartETag { ETag = response.ETag, PartNumber = response.PartNumber } } }); Assert.AreEqual(ServerSideEncryptionMethod.AWSKMS, completeResponse.ServerSideEncryptionMethod); usedKeyId = completeResponse.ServerSideEncryptionKeyManagementServiceKeyId; VerifyKeyId(keyId, usedKeyId); } finally { //abort the multipart upload if (uploadID != null) { Client.AbortMultipartUpload(new AbortMultipartUploadRequest { BucketName = bucketName, Key = dstKey, UploadId = uploadID }); } } }
/// <summary> /// Provides the actual implementation to move or copy an S3 object /// </summary> /// <param name="client"></param> /// <param name="request"></param> /// <param name="partSize"></param> /// <param name="deleteSource"></param> /// <returns></returns> private static async Task <CopyObjectRequestResponse> CopyOrMoveObjectAsync(this IAmazonS3 client, CopyObjectRequest request, long partSize, bool deleteSource, Func <long, long, bool> useMulitpart) { /// Handle operation cancelled exceptions ExponentialBackoffAndRetryClient backoffClient = new ExponentialBackoffAndRetryClient(4, 100, 1000) { ExceptionHandlingLogic = (ex) => { if (ex is OperationCanceledException) { return(true); } else { return(false); } } }; try { ParameterTests.NonNull(request, "request"); ParameterTests.OutOfRange(partSize >= Constants.MINIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size must be at least {Constants.MINIMUM_MULTIPART_PART_SIZE} bytes."); ParameterTests.OutOfRange(partSize <= Constants.MAXIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size cannot exceed {Constants.MAXIMUM_MULTIPART_PART_SIZE} bytes."); if (request.SourceKey == request.DestinationKey && request.SourceBucket != null && request.SourceBucket.Equals(request.DestinationBucket, StringComparison.OrdinalIgnoreCase)) { throw new SourceDestinationSameException("The source and destination of the copy operation cannot be the same.", new CopyObjectRequest[] { request }); } // Get the size of the object. GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest { BucketName = request.SourceBucket, Key = request.SourceKey }; long objectSize; GetObjectMetadataResponse metadataResponse; try { metadataResponse = await backoffClient.RunAsync(() => client.GetObjectMetadataAsync(metadataRequest)); objectSize = metadataResponse.ContentLength; // Length in bytes. } catch (Exception e) { throw e; } CopyObjectResponse response = null; if (UseMultipart(objectSize, partSize)) { // If it takes more than a 5 GiB part to make 10000 or less parts, than this operation // isn't supported for an object this size if (objectSize / partSize > Constants.MAXIMUM_PARTS) { throw new NotSupportedException($"The object size, {objectSize}, cannot be broken into fewer than {Constants.MAXIMUM_PARTS} parts using a part size of {partSize} bytes."); } List <Task <CopyPartResponse> > copyResponses = new List <Task <CopyPartResponse> >(); // This property has a nullable backing private field that when set to // anything non-null causes the x-amz-object-lock-retain-until-date // header to be sent which in turn results in an exception being thrown // that the Bucket is missing ObjectLockConfiguration InitiateMultipartUploadRequest initiateRequest = request.ConvertTo <InitiateMultipartUploadRequest>("ObjectLockRetainUntilDate"); initiateRequest.BucketName = request.DestinationBucket; initiateRequest.Key = request.DestinationKey; InitiateMultipartUploadResponse initiateResponse = await backoffClient.RunAsync(() => client.InitiateMultipartUploadAsync(initiateRequest)); try { long bytePosition = 0; int counter = 1; // Launch all of the copy parts while (bytePosition < objectSize) { CopyPartRequest copyRequest = request.ConvertTo <CopyPartRequest>("ObjectLockRetainUntilDate"); copyRequest.UploadId = initiateResponse.UploadId; copyRequest.FirstByte = bytePosition; // If we're on the last part, the last byte is the object size minus 1, otherwise the last byte is the part size minus one // added to the current byte position copyRequest.LastByte = ((bytePosition + partSize - 1) >= objectSize) ? objectSize - 1 : bytePosition + partSize - 1; copyRequest.PartNumber = counter++; copyResponses.Add(backoffClient.RunAsync(() => client.CopyPartAsync(copyRequest))); bytePosition += partSize; } IEnumerable <CopyPartResponse> responses = (await Task.WhenAll(copyResponses)).OrderBy(x => x.PartNumber); // Set up to complete the copy. CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = request.DestinationBucket, Key = request.DestinationKey, UploadId = initiateResponse.UploadId }; completeRequest.AddPartETags(responses); // Complete the copy. CompleteMultipartUploadResponse completeUploadResponse = await backoffClient.RunAsync(() => client.CompleteMultipartUploadAsync(completeRequest)); response = completeUploadResponse.CopyProperties <CopyObjectResponse>(); response.SourceVersionId = metadataResponse.VersionId; } catch (AmazonS3Exception e) { AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest() { BucketName = request.DestinationBucket, Key = request.DestinationKey, UploadId = initiateResponse.UploadId }; await backoffClient.RunAsync(() => client.AbortMultipartUploadAsync(abortRequest)); throw e; } } else { response = await backoffClient.RunAsync(() => client.CopyObjectAsync(request)); } if (response.HttpStatusCode != HttpStatusCode.OK) { throw new AmazonS3Exception($"Could not copy object from s3://{request.SourceBucket}/{request.SourceKey} to s3://{request.DestinationBucket}/{request.DestinationKey}. Received response : {(int)response.HttpStatusCode}"); } else { // We already checked to make sure the source and destination weren't the same // and it's safe to delete the source object if (deleteSource) { DeleteObjectRequest deleteRequest = new DeleteObjectRequest() { BucketName = request.SourceBucket, Key = request.SourceKey }; DeleteObjectResponse deleteResponse = await backoffClient.RunAsync(() => client.DeleteObjectAsync(deleteRequest)); if (deleteResponse.HttpStatusCode != HttpStatusCode.NoContent) { throw new AmazonS3Exception($"Could not delete s3://{request.SourceBucket}/{request.SourceKey}. Received response : {(int)deleteResponse.HttpStatusCode}"); } } return(new CopyObjectRequestResponse(request, response)); } } catch (Exception e) { return(null); } }
public CopyPartResponse CopyPart(CopyPartRequest request) { throw new NotImplementedException(); }
/// <summary> /// This method uses the passed client object to perform a multi-part /// copy operation. /// </summary> /// <param name="client">An Amazon S3 client object that will be used /// to perform the copy.</param> public static async Task MPUCopyObjectAsync(AmazonS3Client client) { // Create a list to store the upload part responses. var uploadResponses = new List <UploadPartResponse>(); var copyResponses = new List <CopyPartResponse>(); // Setup information required to initiate the multipart upload. var initiateRequest = new InitiateMultipartUploadRequest { BucketName = TargetBucket, Key = TargetObjectKey, }; // Initiate the upload. InitiateMultipartUploadResponse initResponse = await client.InitiateMultipartUploadAsync(initiateRequest); // Save the upload ID. string uploadId = initResponse.UploadId; try { // Get the size of the object. var metadataRequest = new GetObjectMetadataRequest { BucketName = SourceBucket, Key = SourceObjectKey, }; GetObjectMetadataResponse metadataResponse = await client.GetObjectMetadataAsync(metadataRequest); var objectSize = metadataResponse.ContentLength; // Length in bytes. // Copy the parts. var partSize = 5 * (long)Math.Pow(2, 20); // Part size is 5 MB. long bytePosition = 0; for (int i = 1; bytePosition < objectSize; i++) { var copyRequest = new CopyPartRequest { DestinationBucket = TargetBucket, DestinationKey = TargetObjectKey, SourceBucket = SourceBucket, SourceKey = SourceObjectKey, UploadId = uploadId, FirstByte = bytePosition, LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1, PartNumber = i, }; copyResponses.Add(await client.CopyPartAsync(copyRequest)); bytePosition += partSize; } // Set up to complete the copy. var completeRequest = new CompleteMultipartUploadRequest { BucketName = TargetBucket, Key = TargetObjectKey, UploadId = initResponse.UploadId, }; completeRequest.AddPartETags(copyResponses); // Complete the copy. CompleteMultipartUploadResponse completeUploadResponse = await client.CompleteMultipartUploadAsync(completeRequest); } catch (AmazonS3Exception e) { Console.WriteLine($"Error encountered on server. Message:'{e.Message}' when writing an object"); } catch (Exception e) { Console.WriteLine($"Unknown encountered on server. Message:'{e.Message}' when writing an object"); } }
private static async Task CopyObjectAsync(IAmazonS3 s3Client, string base64Key) { List <CopyPartResponse> uploadResponses = new List <CopyPartResponse>(); // 1. Initialize. InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest { BucketName = existingBucketName, Key = targetKeyName, ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256, ServerSideEncryptionCustomerProvidedKey = base64Key, }; InitiateMultipartUploadResponse initResponse = await s3Client.InitiateMultipartUploadAsync(initiateRequest); // 2. Upload Parts. long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB long firstByte = 0; long lastByte = partSize; try { // First find source object size. Because object is stored encrypted with // customer provided key you need to provide encryption information in your request. GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest() { BucketName = existingBucketName, Key = sourceKeyName, ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256, ServerSideEncryptionCustomerProvidedKey = base64Key // " * **source object encryption key ***" }; GetObjectMetadataResponse getObjectMetadataResponse = await s3Client.GetObjectMetadataAsync(getObjectMetadataRequest); long filePosition = 0; for (int i = 1; filePosition < getObjectMetadataResponse.ContentLength; i++) { CopyPartRequest copyPartRequest = new CopyPartRequest { UploadId = initResponse.UploadId, // Source. SourceBucket = existingBucketName, SourceKey = sourceKeyName, // Source object is stored using SSE-C. Provide encryption information. CopySourceServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256, CopySourceServerSideEncryptionCustomerProvidedKey = base64Key, //"***source object encryption key ***", FirstByte = firstByte, // If the last part is smaller then our normal part size then use the remaining size. LastByte = lastByte > getObjectMetadataResponse.ContentLength ? getObjectMetadataResponse.ContentLength - 1 : lastByte, // Target. DestinationBucket = existingBucketName, DestinationKey = targetKeyName, PartNumber = i, // Encryption information for the target object. ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256, ServerSideEncryptionCustomerProvidedKey = base64Key }; uploadResponses.Add(await s3Client.CopyPartAsync(copyPartRequest)); filePosition += partSize; firstByte += partSize; lastByte += partSize; } // Step 3: complete. CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = existingBucketName, Key = targetKeyName, UploadId = initResponse.UploadId, }; completeRequest.AddPartETags(uploadResponses); CompleteMultipartUploadResponse completeUploadResponse = await s3Client.CompleteMultipartUploadAsync(completeRequest); } catch (Exception exception) { Console.WriteLine("Exception occurred: {0}", exception.Message); AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest { BucketName = existingBucketName, Key = targetKeyName, UploadId = initResponse.UploadId }; s3Client.AbortMultipartUpload(abortMPURequest); } }
public void CopyPartAsync(CopyPartRequest request, AmazonServiceCallback <CopyPartRequest, CopyPartResponse> callback, AsyncOptions options = null) { throw new System.NotImplementedException(); }
/// <summary> /// Creates a new file part by copying from an existing file and storing it as a part of a large file which has already been started. /// </summary> /// <param name="sourceFileId">The unique identifier of the source file being copied.</param> /// <param name="largeFileId">The unique identifier of the large file the part will belong to.</param> /// <param name="partNumber">The part number of the file.</param> /// <exception cref="AuthenticationException">Thrown when authentication fails.</exception> /// <exception cref="CapExceededExecption">Thrown when a cap is exceeded or an account in bad standing.</exception> /// <exception cref="ApiException">Thrown when an error occurs during client operation.</exception> async Task <IApiResults <CopyPartResponse> > IStorageParts.CopyAsync(string sourceFileId, string largeFileId, int partNumber) { var request = new CopyPartRequest(sourceFileId, largeFileId, partNumber); return(await _client.CopyPartAsync(request, _cancellationToken)); }
/// <summary> /// Creates a new file part by copying from an existing file and storing it as a part of a large file which has already been started. /// </summary> /// <param name="request">The <see cref="CopyPartRequest"/> to send.</param> /// <exception cref="AuthenticationException">Thrown when authentication fails.</exception> /// <exception cref="CapExceededExecption">Thrown when a cap is exceeded or an account in bad standing.</exception> /// <exception cref="ApiException">Thrown when an error occurs during client operation.</exception> async Task <IApiResults <CopyPartResponse> > IStorageParts.CopyAsync(CopyPartRequest request) { return(await _client.CopyPartAsync(request, _cancellationToken)); }