/// <summary> /// Use the S3 protocol to upload the local file at filePath to the server at uploadTarget. /// </summary> /// <param name="filePath"></param> /// <param name="uploadTarget"></param> public static void TranslocateFile(string uploadTarget, string filePath) { // Create an S3 client. AmazonS3Client s3Client = Common.CreateS3Client( uploadTarget); // Initiate an upload to the S3 server. InitiateMultipartUploadResponse initMultiPartUploadResponse = Common.OpenUpload( s3Client, uploadTarget, filePath); string uploadId = initMultiPartUploadResponse.UploadId; // Upload the parts to the S3 server. long partSize = 5 * Common.MiB; List <UploadPartResponse> uploadPartResponses = Common.UploadParts( s3Client, uploadTarget, filePath, uploadId, partSize); // Complete the upload to the S3 server. CompleteMultipartUploadResponse completeMultipartUploadResponse = Common.CloseUpload( s3Client, uploadTarget, filePath, uploadId, uploadPartResponses); }
/// <summary>Puts data from stream to Amazon S3 storage.</summary> /// <param name="obj">Object info.</param> /// <param name="stream">Stream to upload.</param> public void PutDataFromStreamToObject(IS3ObjectInfo obj, System.IO.Stream stream) { if (obj.IsLocked) { throw new Exception($"[IS3ObjectInfoProvider.PutDataFromStreamToObject]: Couldn't upload object {obj.Key} because it is used by another process."); } obj.Lock(); string bucketName = obj.BucketName; long length = stream.Length; if (length > RECOMMENDED_SIZE_FOR_MULTIPART_UPLOAD) { CompleteMultipartUploadResponse response = this.MultiPartUploader.UploadFromStream(obj.Key, bucketName, stream); this.SetS3ObjectMetadaFromResponse(obj, response, length); } else { PutObjectRequest putRequest = CreatePutRequest(obj.Key, bucketName); putRequest.InputStream = stream; PutObjectResponse response = this.S3Client.PutObject(putRequest); this.SetS3ObjectMetadaFromResponse(obj, response, length); } FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), "PutStreamToObject", "Custom Amazon"); obj.UnLock(); RemoveRequestCache(obj.Key); }
static void Main(string[] args) { IAmazonS3 s3Client = new AmazonS3Client(Amazon.RegionEndpoint.USEast1); // List to store upload part responses. List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>(); List<CopyPartResponse> copyResponses = new List<CopyPartResponse>(); InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest { BucketName = targetBucket, Key = targetObjectKey }; InitiateMultipartUploadResponse initResponse = s3Client.InitiateMultipartUpload(initiateRequest); String uploadId = initResponse.UploadId; try { // Get object size. GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest { BucketName = sourceBucket, Key = sourceObjectKey }; GetObjectMetadataResponse metadataResponse = s3Client.GetObjectMetadata(metadataRequest); long objectSize = metadataResponse.ContentLength; // in bytes // Copy parts. long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB long bytePosition = 0; for (int i = 1; bytePosition < objectSize; i++) { CopyPartRequest copyRequest = new CopyPartRequest { DestinationBucket = targetBucket, DestinationKey = targetObjectKey, SourceBucket = sourceBucket, SourceKey = sourceObjectKey, UploadId = uploadId, FirstByte = bytePosition, LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1, PartNumber = i }; copyResponses.Add(s3Client.CopyPart(copyRequest)); bytePosition += partSize; } CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = targetBucket, Key = targetObjectKey, UploadId = initResponse.UploadId }; completeRequest.AddPartETags(copyResponses); CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest); } catch (Exception e) { Console.WriteLine(e.Message); } }
public async Task ManualSinglePartUpload() { string resourceName = nameof(ManualSinglePartUpload); InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(BucketName, resourceName, request => { request.SseAlgorithm = SseAlgorithm.Aes256; request.StorageClass = StorageClass.StandardIa; }).ConfigureAwait(false); Assert.True(initResp.IsSuccess); Assert.Equal(BucketName, initResp.Bucket); Assert.Equal(resourceName, initResp.Key); Assert.NotNull(initResp.UploadId); byte[] file = new byte[1024 * 1024 * 5]; file[0] = (byte)'a'; UploadPartResponse uploadResp = await ObjectClient.UploadPartAsync(BucketName, resourceName, 1, initResp.UploadId, new MemoryStream(file)).ConfigureAwait(false); Assert.True(uploadResp.IsSuccess); Assert.NotNull(uploadResp.ETag); Assert.Equal(SseAlgorithm.Aes256, uploadResp.SseAlgorithm); Assert.Equal(StorageClass.StandardIa, uploadResp.StorageClass); CompleteMultipartUploadResponse completeResp = await ObjectClient.CompleteMultipartUploadAsync(BucketName, resourceName, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.True(completeResp.IsSuccess); Assert.NotNull(completeResp.ETag); GetObjectResponse resp = await ObjectClient.GetObjectAsync(BucketName, resourceName).ConfigureAwait(false); Assert.True(resp.IsSuccess); Assert.Equal(file, await resp.Content.AsDataAsync().ConfigureAwait(false)); }
public async Task MultipartCustomerEncryption(S3Provider _, string bucket, ISimpleClient client) { string objectKey = nameof(MultipartCustomerEncryption); byte[] key = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }; byte[] keyMd5 = CryptoHelper.Md5Hash(key); CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r => { r.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256; r.SseCustomerKey = key; r.SseCustomerKeyMd5 = keyMd5; }).ConfigureAwait(false); Assert.Equal(200, initResp.StatusCode); await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]); UploadPartResponse uploadResp1 = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r => { r.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256; r.SseCustomerKey = key; r.SseCustomerKeyMd5 = keyMd5; }).ConfigureAwait(false); Assert.Equal(200, uploadResp1.StatusCode); Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm); Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5); CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp1 }).ConfigureAwait(false); Assert.Equal(200, completeResp.StatusCode); Assert.Equal(SseCustomerAlgorithm.Aes256, completeResp.SseCustomerAlgorithm); }
public async Task MultipartWithEncryption(SseAlgorithm algorithm) { string objectKey = nameof(MultipartWithEncryption); CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false); Assert.Equal(algorithm, createResp.SseAlgorithm); if (algorithm == SseAlgorithm.AwsKms) { Assert.NotNull(createResp.SseKmsKeyId); } byte[] file = new byte[1024 * 1024 * 5]; UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, createResp.UploadId, new MemoryStream(file)).ConfigureAwait(false); Assert.Equal(algorithm, uploadResp.SseAlgorithm); if (algorithm == SseAlgorithm.AwsKms) { Assert.NotNull(uploadResp.SseKmsKeyId); } CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.Equal(algorithm, completeResp.SseAlgorithm); if (algorithm == SseAlgorithm.AwsKms) { Assert.NotNull(completeResp.SseKmsKeyId); } }
public async Task MultipartTooSmall(S3Provider provider, string bucket, ISimpleClient client) { string objectKey = nameof(MultipartTooSmall); CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false); Assert.Equal(200, initResp.StatusCode); Assert.Equal(bucket, initResp.BucketName); Assert.Equal(objectKey, initResp.ObjectKey); Assert.NotNull(initResp.UploadId); //4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html //Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts. byte[] file = new byte[1024 * 1024 * 4]; byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray(); await using MemoryStream ms1 = new MemoryStream(parts[0]); UploadPartResponse uploadResp1 = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms1).ConfigureAwait(false); Assert.Equal(provider == S3Provider.BackBlazeB2 ? 400 : 200, uploadResp1.StatusCode); await using MemoryStream ms2 = new MemoryStream(parts[0]); UploadPartResponse uploadResp2 = await client.UploadPartAsync(bucket, objectKey, 2, initResp.UploadId, ms2).ConfigureAwait(false); Assert.Equal(provider == S3Provider.BackBlazeB2 ? 400 : 200, uploadResp2.StatusCode); CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false); Assert.Equal(provider == S3Provider.BackBlazeB2 ? 500 : 400, completeResp.StatusCode); }
public async Task MultipartCustomerEncryption() { string objectKey = nameof(MultipartCustomerEncryption); byte[] key = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }; byte[] keyMd5 = CryptoHelper.Md5Hash(key); CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => { req.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256; req.SseCustomerKey = key; req.SseCustomerKeyMd5 = keyMd5; }).ConfigureAwait(false); Assert.True(initResp.IsSuccess); byte[] file = new byte[1024 * 1024 * 5]; UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => { req.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256; req.SseCustomerKey = key; req.SseCustomerKeyMd5 = keyMd5; }).ConfigureAwait(false); Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm); Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1 }).ConfigureAwait(false); Assert.True(completeResp.IsSuccess); Assert.Equal(SseCustomerAlgorithm.Aes256, completeResp.SseCustomerAlgorithm); }
static void CompleteMultipartUpload() { try { List <PartETag> partEtags = new List <PartETag>(); PartETag partEtag1 = new PartETag(); partEtag1.PartNumber = 1; partEtag1.ETag = etag; partEtags.Add(partEtag1); CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest() { BucketName = bucketName, ObjectKey = objectName, UploadId = uploadId, PartETags = partEtags }; CompleteMultipartUploadResponse response = client.CompleteMultipartUpload(request); Console.WriteLine("Complete multipart upload response: {0}", response.StatusCode); } catch (ObsException ex) { Console.WriteLine("Exception errorcode: {0}, when complete multipart upload.", ex.ErrorCode); Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage); } }
AWSArchiveResult CompleteMPU(string uploadID, AmazonGlacier client, List <string> partChecksumList, FileInfo fio) { try { long fileLength = fio.Length; fileLength = new FileInfo(archiveToUpload).Length; FileStream inputFile = File.Open(archiveToUpload, FileMode.Open, FileAccess.Read); byte[] treeHash = Form1.ComputeSHA256TreeHash(inputFile); String localChecksum = BitConverter.ToString(treeHash).Replace("-", "").ToLower(); CompleteMultipartUploadRequest completeMPUrequest = new CompleteMultipartUploadRequest() { UploadId = uploadID, ArchiveSize = fileLength.ToString(), Checksum = localChecksum, VaultName = vaultName }; CompleteMultipartUploadResponse completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest); AWSArchiveResult ar = new AWSArchiveResult(); ar.ArchiveID = completeMPUresponse.CompleteMultipartUploadResult.ArchiveId; ar.Checksum = localChecksum; return(ar); } catch (Exception e) { Form1.log.Error(e.ToString()); return(new AWSArchiveResult()); } }
public async Task MultipartTooSmall() { string objectKey = nameof(MultipartTooSmall); CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false); Assert.Equal(BucketName, initResp.Bucket); Assert.Equal(objectKey, initResp.ObjectKey); Assert.NotNull(initResp.UploadId); //4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html //Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts. byte[] file = new byte[1024 * 1024 * 4]; byte[][] chunks = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray(); UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(chunks[0])).ConfigureAwait(false); Assert.True(uploadResp1.IsSuccess); Assert.NotNull(uploadResp1.ETag); UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(chunks[1])).ConfigureAwait(false); Assert.True(uploadResp2.IsSuccess); Assert.NotNull(uploadResp2.ETag); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false); Assert.False(completeResp.IsSuccess); Assert.Equal(400, completeResp.StatusCode); }
public async Task MultipartSinglePart() { string objectKey = nameof(MultipartSinglePart); CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false); Assert.True(createResp.IsSuccess); Assert.Equal(BucketName, createResp.Bucket); Assert.Equal(objectKey, createResp.ObjectKey); Assert.NotNull(createResp.UploadId); //Test lifecycle expiration Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn.Value.UtcDateTime.Date); Assert.Equal("ExpireAll", createResp.AbortRuleId); byte[] file = new byte[1024 * 1024 * 5]; file[0] = (byte)'a'; UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, createResp.UploadId, new MemoryStream(file)).ConfigureAwait(false); Assert.True(uploadResp.IsSuccess); Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.True(completeResp.IsSuccess); Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag); Assert.NotNull(completeResp.VersionId); //Test lifecycle expiration Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn.Value.UtcDateTime.Date); Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId); }
public void TestOrdinary() { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.ContentType = "text/plain"; InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = this.bucketName, Key = "test", ObjectMetadata = objectMetadata }; string uploadId = this.client.InitiateMultipartUpload(initRequest).UploadId; List <PartETag> partETags = new List <PartETag>(); for (int i = 0; i < 1; ++i) { string eTag = this.client.UploadPart(new UploadPartRequest() { BucketName = this.bucketName, Key = "test", UploadId = uploadId, PartNumber = i + 1, PartSize = 1, InputStream = new MemoryStream(Encoding.Default.GetBytes(i.ToString())) }).ETag; partETags.Add(new PartETag() { PartNumber = i + 1, ETag = eTag }); } objectMetadata = new ObjectMetadata(); Dictionary <string, string> userMetadata = new Dictionary <string, string>(); userMetadata["metakey"] = "metaValue"; objectMetadata.UserMetadata = userMetadata; objectMetadata.ContentType = "text/json"; CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest() { BucketName = this.bucketName, Key = "test", UploadId = uploadId, PartETags = partETags, ObjectMetadata = objectMetadata }; CompleteMultipartUploadResponse response = this.client.CompleteMultipartUpload(request); Assert.AreEqual(response.BucketName, this.bucketName); Assert.AreEqual(response.Key, "test"); Assert.IsNotNull(response.ETag); Assert.IsNotNull(response.Location); ObjectMetadata metadata = this.client.GetObjectMetadata(bucketName, "test"); Assert.AreEqual(metadata.ContentType, "text/plain"); string resultUserMeta = metadata.UserMetadata["metakey"]; Assert.AreEqual(resultUserMeta, "metaValue"); }
private static void UnmarshallResult(XmlUnmarshallerContext context, CompleteMultipartUploadResponse response) { int currentDepth = context.get_CurrentDepth(); int num = currentDepth + 1; if (context.get_IsStartOfDocument()) { num += 2; } while (context.Read()) { if (context.get_IsStartElement() || context.get_IsAttribute()) { if (context.TestExpression("Location", num)) { response.Location = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Bucket", num)) { response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Key", num)) { response.Key = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("ETag", num)) { response.ETag = StringUnmarshaller.GetInstance().Unmarshall(context); } } else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth) { return; } } IWebResponseData responseData = context.get_ResponseData(); if (responseData.IsHeaderPresent("x-amz-expiration")) { response.Expiration = new Expiration(responseData.GetHeaderValue("x-amz-expiration")); } if (responseData.IsHeaderPresent("x-amz-server-side-encryption")) { response.ServerSideEncryptionMethod = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption")); } if (responseData.IsHeaderPresent("x-amz-version-id")) { response.VersionId = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-version-id")); } if (responseData.IsHeaderPresent("x-amz-server-side-encryption-aws-kms-key-id")) { response.ServerSideEncryptionKeyManagementServiceKeyId = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption-aws-kms-key-id")); } if (responseData.IsHeaderPresent(S3Constants.AmzHeaderRequestCharged)) { response.RequestCharged = RequestCharged.FindValue(responseData.GetHeaderValue(S3Constants.AmzHeaderRequestCharged)); } }
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { CompleteMultipartUploadResponse response = new CompleteMultipartUploadResponse(); context.Read(); UnmarshallResult(context, response); return(response); }
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { CompleteMultipartUploadResponse response = new CompleteMultipartUploadResponse(); context.Read(); response.CompleteMultipartUploadResult = CompleteMultipartUploadResultUnmarshaller.GetInstance().Unmarshall(context); return(response); }
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context) { CompleteMultipartUploadResponse completeMultipartUploadResponse = new CompleteMultipartUploadResponse(); while (context.Read()) { if (context.get_IsStartElement()) { UnmarshallResult(context, completeMultipartUploadResponse); } } return(completeMultipartUploadResponse); }
public string EndMultiPartUpload(long archiveSize, string checksum, string uploadId) { CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest() { ArchiveSize = archiveSize.ToString(), Checksum = checksum, UploadId = uploadId, VaultName = _vault }; CompleteMultipartUploadResponse response = _amazonGlacierClient.CompleteMultipartUpload(request); return(response.CompleteMultipartUploadResult.ArchiveId); }
// After each file is uploaded it will return an ArchiveID static string CompleteMPU(string uploadID, AmazonGlacierClient client, List <string> partChecksumList, string archiveToUpload) { long fileLength = new FileInfo(archiveToUpload).Length; CompleteMultipartUploadRequest completeMPUrequest = new CompleteMultipartUploadRequest() { UploadId = uploadID, ArchiveSize = fileLength.ToString(), Checksum = TreeHashGenerator.CalculateTreeHash(partChecksumList), VaultName = vaultName }; CompleteMultipartUploadResponse completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest); return(completeMPUresponse.ArchiveId); }
public string CompletePartToCloud() { // Step 3: complete. CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(); compRequest.BucketName = this.DestinationBucket; compRequest.Key = this.DestinationFilename; compRequest.UploadId = initResponse.UploadId; compRequest.PartETags = uploadPartETags; string r = "Something went badly wrong"; using (CompleteMultipartUploadResponse completeUploadResponse = client.CompleteMultipartUpload(compRequest)) r = completeUploadResponse.ResponseXml; return(r); }
public async Task MultipartWithEncryption(S3Provider provider, string bucket, ISimpleClient client, SseAlgorithm algorithm) { string objectKey = nameof(MultipartWithEncryption); CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false); Assert.Equal(200, createResp.StatusCode); if (provider == S3Provider.AmazonS3) { Assert.Equal(algorithm, createResp.SseAlgorithm); } if (algorithm == SseAlgorithm.AwsKms) { Assert.NotNull(createResp.SseKmsKeyId); } await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]); UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false); Assert.Equal(200, uploadResp.StatusCode); if (provider == S3Provider.AmazonS3) { Assert.Equal(algorithm, uploadResp.SseAlgorithm); } if (algorithm == SseAlgorithm.AwsKms) { Assert.NotNull(uploadResp.SseKmsKeyId); } CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.Equal(200, completeResp.StatusCode); if (provider == S3Provider.AmazonS3) { Assert.Equal(algorithm, completeResp.SseAlgorithm); } if (algorithm == SseAlgorithm.AwsKms) { Assert.NotNull(completeResp.SseKmsKeyId); } }
private static void UnmarshallResult(JsonUnmarshallerContext context, CompleteMultipartUploadResponse response) { if (context.ResponseData.GetHeaderValue("Location") != null) { response.Location = context.ResponseData.GetHeaderValue("Location"); } if (context.ResponseData.GetHeaderValue("x-amz-sha256-tree-hash") != null) { response.Checksum = context.ResponseData.GetHeaderValue("x-amz-sha256-tree-hash"); } if (context.ResponseData.GetHeaderValue("x-amz-archive-id") != null) { response.ArchiveId = context.ResponseData.GetHeaderValue("x-amz-archive-id"); } return; }
public async Task MultipartUpload() { string objectKey = nameof(MultipartUpload); CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false); Assert.True(initResp.IsSuccess); Assert.Equal(BucketName, initResp.Bucket); Assert.Equal(objectKey, initResp.ObjectKey); Assert.NotNull(initResp.UploadId); byte[] file = new byte[1024 * 1024 * 10]; file[0] = (byte)'a'; file[file.Length - 1] = (byte)'b'; byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray(); UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(parts[0])).ConfigureAwait(false); Assert.True(uploadResp1.IsSuccess); Assert.NotNull(uploadResp1.ETag); UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(parts[1])).ConfigureAwait(false); Assert.True(uploadResp2.IsSuccess); Assert.NotNull(uploadResp2.ETag); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false); Assert.True(completeResp.IsSuccess); Assert.NotNull(uploadResp2.ETag); GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, objectKey).ConfigureAwait(false); //Provoke an 'InvalidArgument' error GetObjectResponse gResp1 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 0).ConfigureAwait(false); Assert.False(gResp1.IsSuccess); Assert.IsType <InvalidArgumentError>(gResp1.Error); GetObjectResponse gResp2 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 1).ConfigureAwait(false); Assert.True(gResp2.IsSuccess); Assert.Equal(file.Length / 2, (await gResp2.Content.AsDataAsync().ConfigureAwait(false)).Length); Assert.Equal(file, await getResp.Content.AsDataAsync().ConfigureAwait(false)); }
/// <summary> /// Send a CompleteMultipartUploadRequest request and return the response. /// </summary> /// <param name="existingBucketName"></param> /// <param name="keyName"></param> /// <param name="uploadID"></param> /// <returns>Task<CompleteMultipartUploadRequest></returns> public async Task <CompleteMultipartUploadRequest> CompleteMultipartUploadAsync(string existingBucketName, string keyName, string uploadId, CancellationToken token) { token.ThrowIfCancellationRequested(); _log.Debug("Called CompleteMultipartUploadAsync with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\"."); try { CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = existingBucketName, Key = keyName, UploadId = uploadId, }; // in any case request list parts to send etags for each part. var uploadedParts = await this.ListPartsAsync(existingBucketName, keyName, uploadId, token).ConfigureAwait(false); List <PartETag> eTags = new List <PartETag>(); foreach (var part in uploadedParts) { eTags.Add(new PartETag(part.PartNumber, part.ETag)); } completeRequest.AddPartETags(eTags); token.ThrowIfCancellationRequested(); CompleteMultipartUploadResponse completeResponse = await s3Client.CompleteMultipartUploadAsync(completeRequest, token).ConfigureAwait(false); return(completeRequest); } catch (Exception e) { if (!(e is TaskCanceledException || e is OperationCanceledException)) { string messagePart = " with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\""; this.LogAmazonException(messagePart, e); } throw; } }
/// <summary> /// Unmarshaller the response from the service to the response class. /// </summary> /// <param name="context"></param> /// <returns></returns> public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { CompleteMultipartUploadResponse response = new CompleteMultipartUploadResponse(); if (context.ResponseData.IsHeaderPresent("x-amz-archive-id")) { response.ArchiveId = context.ResponseData.GetHeaderValue("x-amz-archive-id"); } if (context.ResponseData.IsHeaderPresent("x-amz-sha256-tree-hash")) { response.Checksum = context.ResponseData.GetHeaderValue("x-amz-sha256-tree-hash"); } if (context.ResponseData.IsHeaderPresent("Location")) { response.Location = context.ResponseData.GetHeaderValue("Location"); } return(response); }
public string CompletePartialAndSaveResource(ReferenceRepository referenceRepository, ResourceModel.User owner, string uploadIdentifier, String originalName, String md5OfResource, List <string> etags, long totalSize) { IAmazonS3 s3Client = new AmazonS3Client(); ResourceModel.DigitalResource resource = null; ResourceType imageRT = referenceRepository.AllResourceTypes().Where(x => x.Type == "Image").FirstOrDefault(); ResourceType otherRT = referenceRepository.AllResourceTypes().Where(x => x.Type == "Other").FirstOrDefault(); List <PartETag> partTags = new List <PartETag>(); int partNumber = 1; foreach (string tag in etags) { partTags.Add(new PartETag { ETag = tag, PartNumber = partNumber }); partNumber++; } // complete the upload. CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = "piccoli", Key = md5OfResource, UploadId = uploadIdentifier, PartETags = partTags }; // Complete the upload. CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest); //create the resource object resource = CreateNewResourceObject(owner, totalSize, originalName, imageRT, otherRT, md5OfResource); SaveResource(resource);//update the database return(completeUploadResponse.ETag); }
public async Task MultipartSinglePart(S3Provider provider, string bucket, ISimpleClient client) { string objectKey = nameof(MultipartSinglePart); CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false); Assert.Equal(200, createResp.StatusCode); Assert.Equal(bucket, createResp.BucketName); Assert.Equal(objectKey, createResp.ObjectKey); Assert.NotNull(createResp.UploadId); if (provider == S3Provider.AmazonS3) { //Test lifecycle expiration Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn !.Value.UtcDateTime.Date); Assert.Equal("ExpireAll", createResp.AbortRuleId); } byte[] file = new byte[1024 * 1024 * 5]; file[0] = (byte)'a'; await using MemoryStream ms = new MemoryStream(file); UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false); Assert.Equal(200, uploadResp.StatusCode); Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag); CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.Equal(200, completeResp.StatusCode); Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag); if (provider == S3Provider.AmazonS3) { //Test lifecycle expiration Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn !.Value.UtcDateTime.Date); Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId); } }
public async Task MultipartLockMode() { string objectKey = nameof(MultipartLockMode); CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => { req.LockMode = LockMode.Governance; req.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5); }).ConfigureAwait(false); Assert.True(initResp.IsSuccess); byte[] file = new byte[1024 * 1024 * 5]; UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false); Assert.True(uploadResp.IsSuccess); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.True(completeResp.IsSuccess); }
public async Task MultipartRequestPayer() { string objectKey = nameof(MultipartRequestPayer); CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(initResp.RequestCharged); byte[] file = new byte[1024 * 1024 * 5]; UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(uploadResp.RequestCharged); ListPartsResponse listResp = await MultipartClient.ListPartsAsync(BucketName, objectKey, initResp.UploadId, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(listResp.RequestCharged); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(completeResp.RequestCharged); }
public async Task MultipartLockMode(S3Provider _, string bucket, ISimpleClient client) { string objectKey = nameof(MultipartLockMode); CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r => { r.LockMode = LockMode.Governance; r.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5); }).ConfigureAwait(false); Assert.Equal(200, initResp.StatusCode); byte[] file = new byte[1024 * 1024 * 5]; await using MemoryStream ms = new MemoryStream(file); UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r => r.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false); Assert.Equal(200, uploadResp.StatusCode); CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); Assert.Equal(200, completeResp.StatusCode); }