static void ListParts() { try { ListPartsRequest request = new ListPartsRequest() { BucketName = bucketName, ObjectKey = objectName, UploadId = uploadId, MaxParts = 10, PartNumberMarker = 1, }; ListPartsResponse response = client.ListParts(request); Console.WriteLine("List parts response: {0}", response.StatusCode); Console.WriteLine("Lis parts count: " + response.Parts.Count); foreach (PartETag part in response.Parts) { Console.WriteLine("PartNumber: {0}", part.PartNumber); Console.WriteLine("ETag: {0}", part.ETag); } } catch (ObsException ex) { Console.WriteLine("Exception errorcode: {0}, when list parts.", ex.ErrorCode); Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage); } }
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonGlacierConfig config = new AmazonGlacierConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonGlacierClient client = new AmazonGlacierClient(creds, config); ListPartsResponse resp = new ListPartsResponse(); do { ListPartsRequest req = new ListPartsRequest { Marker = resp.Marker , Limit = maxItems }; resp = client.ListParts(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.Parts) { AddObject(obj); } }while (!string.IsNullOrEmpty(resp.Marker)); }
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { ListPartsResponse response = new ListPartsResponse(); context.Read(); int targetDepth = context.CurrentDepth; while (context.ReadAtDepth(targetDepth)) { if (context.TestExpression("MultipartUploadId", targetDepth)) { response.MultipartUploadId = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("VaultARN", targetDepth)) { response.VaultARN = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("ArchiveDescription", targetDepth)) { response.ArchiveDescription = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("PartSizeInBytes", targetDepth)) { response.PartSizeInBytes = LongUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("CreationDate", targetDepth)) { response.CreationDate = DateTimeUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("Parts", targetDepth)) { var unmarshaller = new ListUnmarshaller <PartListElement, PartListElementUnmarshaller>( PartListElementUnmarshaller.GetInstance()); response.Parts = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("Marker", targetDepth)) { response.Marker = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } } return(response); }
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { ListPartsResponse response = new ListPartsResponse(); context.Read(); UnmarshallResult(context, response); return(response); }
public async Task ListParts() { await CreateTempBucketAsync(async bucket => { //We add the special characters at the end to test EncodingType support. string objName = nameof(ListParts) + "%"; CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(bucket, objName).ConfigureAwait(false); ListPartsResponse listResp1 = await MultipartClient.ListPartsAsync(bucket, objName, createResp.UploadId).ConfigureAwait(false); Assert.Equal(bucket, listResp1.BucketName); Assert.Equal(objName, listResp1.ObjectKey); Assert.Equal(createResp.UploadId, listResp1.UploadId); Assert.Equal(StorageClass.Standard, listResp1.StorageClass); Assert.Equal(0, listResp1.PartNumberMarker); Assert.Equal(0, listResp1.NextPartNumberMarker); Assert.Equal(1000, listResp1.MaxParts); Assert.False(listResp1.IsTruncated); Assert.Equal(TestConstants.TestUsername, listResp1.Owner.Name); Assert.Empty(listResp1.Parts); UploadPartResponse uploadResp; byte[] file = new byte[5 * 1024]; using (MemoryStream ms = new MemoryStream(file)) uploadResp = await MultipartClient.UploadPartAsync(bucket, objName, 1, createResp.UploadId, ms).ConfigureAwait(false); ListPartsResponse listResp2 = await MultipartClient.ListPartsAsync(bucket, objName, createResp.UploadId, req => req.EncodingType = EncodingType.Url).ConfigureAwait(false); Assert.Equal(bucket, listResp2.BucketName); Assert.Equal(WebUtility.UrlEncode(objName), listResp2.ObjectKey); //It should be encoded at this point Assert.Equal(createResp.UploadId, listResp2.UploadId); Assert.Equal(StorageClass.Standard, listResp2.StorageClass); Assert.Equal(0, listResp2.PartNumberMarker); Assert.Equal(1, listResp2.NextPartNumberMarker); Assert.Equal(1000, listResp2.MaxParts); Assert.False(listResp2.IsTruncated); Assert.Equal(TestConstants.TestUsername, listResp2.Owner.Name); S3Part part = Assert.Single(listResp2.Parts); Assert.Equal(1, part.PartNumber); Assert.Equal(DateTime.UtcNow, part.LastModified.DateTime, TimeSpan.FromSeconds(5)); Assert.Equal("\"32ca18808933aa12e979375d07048a11\"", part.ETag); Assert.Equal(file.Length, part.Size); await MultipartClient.CompleteMultipartUploadAsync(bucket, objName, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false); ListPartsResponse listResp3 = await MultipartClient.ListPartsAsync(bucket, objName, createResp.UploadId).ConfigureAwait(false); Assert.False(listResp3.IsSuccess); Assert.Equal(404, listResp3.StatusCode); }).ConfigureAwait(false); }
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { ListPartsResponse response = new ListPartsResponse(); context.Read(); response.ListPartsResult = ListPartsResultUnmarshaller.GetInstance().Unmarshall(context); return(response); }
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context) { ListPartsResponse listPartsResponse = new ListPartsResponse(); while (context.Read()) { if (context.get_IsStartElement()) { UnmarshallResult(context, listPartsResponse); } } return(listPartsResponse); }
/// <summary> /// Send a ListPartsRequest request and return a list of all uploaded parts. /// </summary> /// <param name="existingBucketName"></param> /// <param name="keyName"></param> /// <param name="uploadID"></param> /// <param name="token"></param> /// <returns>Task<List<PartDetail>></returns> public async Task <List <PartDetail> > ListPartsAsync(string existingBucketName, string keyName, string uploadId, CancellationToken token) { token.ThrowIfCancellationRequested(); _log.Debug("Called ListPartsAsync with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\"."); List <PartDetail> parts = new List <PartDetail>(); try { ListPartsRequest request = new ListPartsRequest(); request.BucketName = existingBucketName; request.Key = keyName; request.UploadId = uploadId; ListPartsResponse response = await this.s3Client.ListPartsAsync(request, token).ConfigureAwait(false); parts.AddRange(response.Parts); while (response.IsTruncated) { token.ThrowIfCancellationRequested(); request.PartNumberMarker = response.NextPartNumberMarker.ToString(); response = await this.s3Client.ListPartsAsync(request, token).ConfigureAwait(false); parts.AddRange(response.Parts); } return(parts); } catch (Exception e) { if (!(e is TaskCanceledException || e is OperationCanceledException)) { string messagePart = " with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\""; this.LogAmazonException(messagePart, e); } throw; } }
public void TestListPartsOk() { string uploadId = this.client.InitiateMultipartUpload(this.bucketName, "test").UploadId; List <string> eTags = new List <string>(); for (int i = 0; i < 10; ++i) { eTags.Add(this.client.UploadPart(new UploadPartRequest() { BucketName = this.bucketName, Key = "test", UploadId = uploadId, PartNumber = i + 1, PartSize = 1, InputStream = new MemoryStream(Encoding.Default.GetBytes(i.ToString())) }).ETag); } ListPartsResponse response = this.client.ListParts(this.bucketName, "test", uploadId); Assert.AreEqual(response.BucketName, this.bucketName); Assert.AreEqual(response.IsTruncated, false); Assert.AreEqual(response.Key, "test"); Assert.AreEqual(response.MaxParts, 1000); Assert.AreEqual(response.NextPartNumberMarker, 10); Assert.AreEqual(response.Owner.Id, this.owner.Id); Assert.AreEqual(response.PartNumberMarker, 0); Assert.AreEqual(response.UploadId, uploadId); Assert.AreEqual(BosConstants.StorageClass.Standard, response.StorageClass); List <PartSummary> parts = response.Parts; Assert.AreEqual(parts.Count, 10); for (int i = 0; i < 10; ++i) { PartSummary part = parts[i]; Assert.AreEqual(part.ETag, eTags[i]); Assert.AreEqual(part.PartNumber, i + 1); Assert.AreEqual(part.Size, 1); Assert.IsTrue(Math.Abs(part.LastModified.Subtract(DateTime.UtcNow).TotalSeconds) < 60); } }
public async Task MultipartRequestPayer() { string objectKey = nameof(MultipartRequestPayer); CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(initResp.RequestCharged); byte[] file = new byte[1024 * 1024 * 5]; UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(uploadResp.RequestCharged); ListPartsResponse listResp = await MultipartClient.ListPartsAsync(BucketName, objectKey, initResp.UploadId, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(listResp.RequestCharged); CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false); Assert.True(completeResp.RequestCharged); }
public void ObjectSamples() { { #region ListObjects Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // List all objects ListObjectsRequest listRequest = new ListObjectsRequest { BucketName = "SampleBucket", }; ListObjectsResponse listResponse; do { // Get a list of objects listResponse = client.ListObjects(listRequest); foreach (S3Object obj in listResponse.S3Objects) { Console.WriteLine("Object - " + obj.Key); Console.WriteLine(" Size - " + obj.Size); Console.WriteLine(" LastModified - " + obj.LastModified); Console.WriteLine(" Storage class - " + obj.StorageClass); } // Set the marker property listRequest.Marker = listResponse.NextMarker; } while (listResponse.IsTruncated); #endregion } { #region GetObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a GetObject request GetObjectRequest request = new GetObjectRequest { BucketName = "SampleBucket", Key = "Item1" }; // Issue request and remember to dispose of the response using (GetObjectResponse response = client.GetObject(request)) { using (StreamReader reader = new StreamReader(response.ResponseStream)) { string contents = reader.ReadToEnd(); Console.WriteLine("Object - " + response.Key); Console.WriteLine(" Version Id - " + response.VersionId); Console.WriteLine(" Contents - " + contents); } } #endregion } { #region GetObjectMetadata Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a GetObjectMetadata request GetObjectMetadataRequest request = new GetObjectMetadataRequest { BucketName = "SampleBucket", Key = "Item1" }; // Issue request and view the response GetObjectMetadataResponse response = client.GetObjectMetadata(request); Console.WriteLine("Content Length - " + response.ContentLength); Console.WriteLine("Content Type - " + response.Headers.ContentType); if (response.Expiration != null) { Console.WriteLine("Expiration Date - " + response.Expiration.ExpiryDate); Console.WriteLine("Expiration Rule Id - " + response.Expiration.RuleId); } #endregion } { #region PutObject Sample 1 // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a PutObject request PutObjectRequest request = new PutObjectRequest { BucketName = "SampleBucket", Key = "Item1", ContentBody = "This is sample content..." }; // Put object PutObjectResponse response = client.PutObject(request); #endregion } { #region PutObject Sample 2 // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a PutObject request PutObjectRequest request = new PutObjectRequest { BucketName = "SampleBucket", Key = "Item1", FilePath = "contents.txt" }; // Put object PutObjectResponse response = client.PutObject(request); #endregion } { #region PutObject Sample 3 // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a PutObject request PutObjectRequest request = new PutObjectRequest { BucketName = "SampleBucket", Key = "Item1", }; using (FileStream stream = new FileStream("contents.txt", FileMode.Open)) { request.InputStream = stream; // Put object PutObjectResponse response = client.PutObject(request); } #endregion } { #region DeleteObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a DeleteObject request DeleteObjectRequest request = new DeleteObjectRequest { BucketName = "SampleBucket", Key = "Item1" }; // Issue request client.DeleteObject(request); #endregion } { #region DeleteObjects Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a DeleteObject request DeleteObjectsRequest request = new DeleteObjectsRequest { BucketName = "SampleBucket", Objects = new List <KeyVersion> { new KeyVersion() { Key = "Item1" }, // Versioned item new KeyVersion() { Key = "Item2", VersionId = "Rej8CiBxcZKVK81cLr39j27Y5FVXghDK", }, // Item in subdirectory new KeyVersion() { Key = "Logs/error.txt" } } }; try { // Issue request DeleteObjectsResponse response = client.DeleteObjects(request); } catch (DeleteObjectsException doe) { // Catch error and list error details DeleteObjectsResponse errorResponse = doe.Response; foreach (DeletedObject deletedObject in errorResponse.DeletedObjects) { Console.WriteLine("Deleted item " + deletedObject.Key); } foreach (DeleteError deleteError in errorResponse.DeleteErrors) { Console.WriteLine("Error deleting item " + deleteError.Key); Console.WriteLine(" Code - " + deleteError.Code); Console.WriteLine(" Message - " + deleteError.Message); } } #endregion } { #region CopyObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a CopyObject request CopyObjectRequest request = new CopyObjectRequest { SourceBucket = "SampleBucket", SourceKey = "Item1", DestinationBucket = "AnotherBucket", DestinationKey = "Copy1", CannedACL = S3CannedACL.PublicRead }; // Issue request client.CopyObject(request); #endregion } { #region CopyObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a CopyObject request CopyObjectRequest request = new CopyObjectRequest { SourceBucket = "SampleBucket", SourceKey = "Item1", DestinationBucket = "AnotherBucket", DestinationKey = "Copy1", CannedACL = S3CannedACL.PublicRead }; // Issue request client.CopyObject(request); #endregion } { #region ListVersions Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Turn versioning on for a bucket client.PutBucketVersioning(new PutBucketVersioningRequest { BucketName = "SampleBucket", VersioningConfig = new S3BucketVersioningConfig { Status = "Enable" } }); // Populate bucket with multiple items, each with multiple versions PopulateBucket(client, "SampleBucket"); // Get versions ListVersionsRequest request = new ListVersionsRequest { BucketName = "SampleBucket" }; // Make paged ListVersions calls ListVersionsResponse response; do { response = client.ListVersions(request); // View information about versions foreach (var version in response.Versions) { Console.WriteLine("Key = {0}, Version = {1}, IsLatest = {2}, LastModified = {3}, Size = {4}", version.Key, version.VersionId, version.IsLatest, version.LastModified, version.Size); } request.KeyMarker = response.NextKeyMarker; request.VersionIdMarker = response.NextVersionIdMarker; } while (response.IsTruncated); #endregion } { #region Multipart Upload Sample int MB = (int)Math.Pow(2, 20); // Create a client AmazonS3Client client = new AmazonS3Client(); // Define input stream Stream inputStream = Create13MBDataStream(); // Initiate multipart upload InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest { BucketName = "SampleBucket", Key = "Item1" }; InitiateMultipartUploadResponse initResponse = client.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MB, InputStream = inputStream }; UploadPartResponse up1Response = client.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MB, InputStream = inputStream }; UploadPartResponse up2Response = client.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream }; UploadPartResponse up3Response = client.UploadPart(uploadRequest); // List parts for current upload ListPartsRequest listPartRequest = new ListPartsRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = client.ListParts(listPartRequest); Debug.Assert(listPartResponse.Parts.Count == 3); // Complete the multipart upload CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartETags = new List <PartETag> { new PartETag { ETag = up1Response.ETag, PartNumber = 1 }, new PartETag { ETag = up2Response.ETag, PartNumber = 2 }, new PartETag { ETag = up3Response.ETag, PartNumber = 3 } } }; CompleteMultipartUploadResponse compResponse = client.CompleteMultipartUpload(compRequest); #endregion } }
public void MultipartEncryptionTestInstructionFile() { string filePath = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt"; string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt"; int MEG_SIZE = (int)Math.Pow(2, 20) + 4001; long totalSize = (long)(15 * MEG_SIZE); UtilityMethods.GenerateFile(filePath, totalSize); string key = "MultipartEncryptionTestInstrcutionFile" + random.Next(); s3EncryptionClientFileMode.PutBucket(new PutBucketRequest() { BucketName = bucketName }); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.ReducedRedundancy, ContentType = "text/html", CannedACL = S3CannedACL.PublicRead }; InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MEG_SIZE, InputStream = inputStream }; UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MEG_SIZE + 4001, InputStream = inputStream }; UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; //uploadRequest.setLastPart(); UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest); Assert.AreEqual(3, listPartResponse.Parts.Count); Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest); Assert.AreEqual(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest); Assert.AreEqual(bucketName, compResponse.BucketName); Assert.IsNotNull(compResponse.ETag); Assert.AreEqual(key, compResponse.Key); Assert.IsNotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest); getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest); Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
/// <summary> /// Test helper to test a multipart upload without using the Transfer Utility /// </summary> /// <param name="algorithm">checksum algorithm</param> /// <param name="bucketName">bucket to upload the object to</param> /// <param name="disablePayloadSigning">whether the request payload should be signed</param> private void MultipartTestHelper(CoreChecksumAlgorithm algorithm, string bucketName, bool disablePayloadSigning) { var random = new Random(); var nextRandom = random.Next(); var filePath = Path.Combine(Path.GetTempPath(), "multi-" + nextRandom + ".txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + nextRandom + ".txt"); var totalSize = MegSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); string key = "key-" + random.Next(); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()) }; InitiateMultipartUploadResponse initResponse = Client.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegSize, InputStream = inputStream, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), DisablePayloadSigning = disablePayloadSigning }; UploadPartResponse up1Response = Client.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegSize, InputStream = inputStream, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), DisablePayloadSigning = disablePayloadSigning }; UploadPartResponse up2Response = Client.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), DisablePayloadSigning = disablePayloadSigning, IsLastPart = true }; UploadPartResponse up3Response = Client.UploadPart(uploadRequest); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = Client.ListParts(listPartRequest); Assert.AreEqual(3, listPartResponse.Parts.Count); AssertPartsAreEqual(up1Response, listPartResponse.Parts[0]); AssertPartsAreEqual(up2Response, listPartResponse.Parts[1]); AssertPartsAreEqual(up3Response, listPartResponse.Parts[2]); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = Client.CompleteMultipartUpload(compRequest); Assert.IsNotNull(compResponse.ETag); Assert.AreEqual(key, compResponse.Key); Assert.IsNotNull(compResponse.Location); // Get the file back from S3 and assert it is still the same. var getRequest = new GetObjectRequest { BucketName = bucketName, Key = key, ChecksumMode = ChecksumMode.ENABLED }; var getResponse = Client.GetObject(getRequest); getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); // We don't expect the checksum to be validated on getting an entire multipart object, // because it's actually the checksum-of-checksums Assert.AreEqual(CoreChecksumAlgorithm.NONE, getResponse.ResponseMetadata.ChecksumAlgorithm); Assert.AreEqual(ChecksumValidationStatus.NOT_VALIDATED, getResponse.ResponseMetadata.ChecksumValidationStatus); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
public static void MultipartEncryptionTestAPM(IAmazonS3 s3EncryptionClient, IAmazonS3 s3DecryptionClient, string bucketName) { var guid = Guid.NewGuid(); var filePath = Path.Combine(Path.GetTempPath(), $"multi-{guid}.txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), $"retrieved-{guid}.txt"); var totalSize = MegaByteSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); string key = $"key-{guid}"; Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.OneZoneInfrequentAccess, ContentType = "text/html" }; InitiateMultipartUploadResponse initResponse = null; if (IsKMSEncryptionClient(s3EncryptionClient)) { initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest); } else { initResponse = s3EncryptionClient.EndInitiateMultipartUpload( s3EncryptionClient.BeginInitiateMultipartUpload(initRequest, null, null)); } // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegaByteSize, InputStream = inputStream, }; UploadPartResponse up1Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegaByteSize, InputStream = inputStream, }; UploadPartResponse up2Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; UploadPartResponse up3Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = s3EncryptionClient.EndListParts( s3EncryptionClient.BeginListParts(listPartRequest, null, null)); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClient.EndListParts( s3EncryptionClient.BeginListParts(listPartRequest, null, null)); Assert.Equal(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = s3EncryptionClient.EndCompleteMultipartUpload( s3EncryptionClient.BeginCompleteMultipartUpload(compRequest, null, null)); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = null; if (IsKMSEncryptionClient(s3EncryptionClient)) { getResponse = s3DecryptionClient.GetObject(getRequest); } else { getResponse = s3DecryptionClient.EndGetObject( s3DecryptionClient.BeginGetObject(getRequest, null, null)); } getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = s3DecryptionClient.EndGetObjectMetadata( s3DecryptionClient.BeginGetObjectMetadata(metaDataRequest, null, null)); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
public async Task MultipartEncryptionTestInstructionFile() { string filePath = Path.Combine(Path.GetTempPath(), "MulitpartEncryptionTestInstructionFile_upload.txt"); string retrievedFilepath = Path.Combine(Path.GetTempPath(), "MulitpartEncryptionTestInstructionFile_download.txt"); int MEG_SIZE = (int)Math.Pow(2, 20); long totalSize = (long)(15 * MEG_SIZE) + 4001; UtilityMethods.GenerateFile(filePath, totalSize); _filesToDelete.Add(filePath); string key = "MultipartEncryptionTestInstrcutionFile" + random.Next(); using (Stream inputStream = File.OpenRead(filePath)) { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.ReducedRedundancy, ContentType = "text/html", CannedACL = S3CannedACL.PublicRead }; InitiateMultipartUploadResponse initResponse = await s3EncryptionClientFileMode.InitiateMultipartUploadAsync(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MEG_SIZE, InputStream = inputStream }; UploadPartResponse up1Response = await s3EncryptionClientFileMode.UploadPartAsync(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MEG_SIZE + 4001, InputStream = inputStream }; UploadPartResponse up2Response = await s3EncryptionClientFileMode.UploadPartAsync(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; //uploadRequest.setLastPart(); UploadPartResponse up3Response = await s3EncryptionClientFileMode.UploadPartAsync(uploadRequest); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = await s3EncryptionClientFileMode.ListPartsAsync(listPartRequest); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = await s3EncryptionClientFileMode.ListPartsAsync(listPartRequest); Assert.Equal(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = await s3EncryptionClientFileMode.CompleteMultipartUploadAsync(compRequest); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = await s3EncryptionClientFileMode.GetObjectAsync(getRequest); await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, System.Threading.CancellationToken.None); _filesToDelete.Add(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = await s3EncryptionClientFileMode.GetObjectMetadataAsync(metaDataRequest); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } }
public void MultipartEncryptionTestAPM(AmazonS3EncryptionClient s3EncryptionClient) { var nextRandom = random.Next(); var filePath = @"C:\temp\multi-" + nextRandom + ".txt"; var retrievedFilepath = @"C:\temp\retreived-" + nextRandom + ".txt"; var totalSize = MegSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); string key = "key-" + random.Next(); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.ReducedRedundancy, ContentType = "text/html", CannedACL = S3CannedACL.PublicRead }; InitiateMultipartUploadResponse initResponse = null; if (IsKMSEncryptionClient(s3EncryptionClient)) { initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest); } else { initResponse = s3EncryptionClient.EndInitiateMultipartUpload( s3EncryptionClient.BeginInitiateMultipartUpload(initRequest, null, null)); } // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegSize, InputStream = inputStream, }; UploadPartResponse up1Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegSize, InputStream = inputStream, }; UploadPartResponse up2Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; UploadPartResponse up3Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = s3EncryptionClient.EndListParts( s3EncryptionClient.BeginListParts(listPartRequest, null, null)); Assert.AreEqual(3, listPartResponse.Parts.Count); Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClient.EndListParts( s3EncryptionClient.BeginListParts(listPartRequest, null, null)); Assert.AreEqual(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = s3EncryptionClient.EndCompleteMultipartUpload( s3EncryptionClient.BeginCompleteMultipartUpload(compRequest, null, null)); Assert.AreEqual(bucketName, compResponse.BucketName); Assert.IsNotNull(compResponse.ETag); Assert.AreEqual(key, compResponse.Key); Assert.IsNotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = null; if (IsKMSEncryptionClient(s3EncryptionClient)) { getResponse = s3EncryptionClient.GetObject(getRequest); } else { getResponse = s3EncryptionClient.EndGetObject( s3EncryptionClient.BeginGetObject(getRequest, null, null)); } getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = s3EncryptionClient.EndGetObjectMetadata( s3EncryptionClient.BeginGetObjectMetadata(metaDataRequest, null, null)); Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
private static void UnmarshallResult(XmlUnmarshallerContext context, ListPartsResponse response) { int originalDepth = context.CurrentDepth; int targetDepth = originalDepth + 1; if (context.IsStartOfDocument) { targetDepth += 2; } while (context.Read()) { if (context.IsStartElement || context.IsAttribute) { if (context.TestExpression("Bucket", targetDepth)) { response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("Key", targetDepth)) { response.Key = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("UploadId", targetDepth)) { response.UploadId = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("PartNumberMarker", targetDepth)) { response.PartNumberMarker = IntUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("NextPartNumberMarker", targetDepth)) { response.NextPartNumberMarker = IntUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("MaxParts", targetDepth)) { response.MaxParts = IntUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("IsTruncated", targetDepth)) { response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("Part", targetDepth)) { response.Parts.Add(PartDetailUnmarshaller.Instance.Unmarshall(context)); continue; } if (context.TestExpression("Initiator", targetDepth)) { response.Initiator = InitiatorUnmarshaller.Instance.Unmarshall(context); continue; } if (context.TestExpression("Owner", targetDepth)) { response.Owner = OwnerUnmarshaller.Instance.Unmarshall(context); continue; } if (context.TestExpression("StorageClass", targetDepth)) { response.StorageClass = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } } else if (context.IsEndElement && context.CurrentDepth < originalDepth) { return; } } IWebResponseData responseData = context.ResponseData; if (responseData.IsHeaderPresent(HeaderKeys.XAmzAbortDateHeader)) { response.AbortDate = S3Transforms.ToDateTime(responseData.GetHeaderValue(HeaderKeys.XAmzAbortDateHeader)); } if (responseData.IsHeaderPresent(HeaderKeys.XAmzAbortRuleIdHeader)) { response.AbortRuleId = S3Transforms.ToString(responseData.GetHeaderValue(HeaderKeys.XAmzAbortRuleIdHeader)); } if (responseData.IsHeaderPresent(S3Constants.AmzHeaderRequestCharged)) { response.RequestCharged = RequestCharged.FindValue(responseData.GetHeaderValue(S3Constants.AmzHeaderRequestCharged)); } return; }
public static async Task MultipartEncryptionTestAsync(AmazonS3Client s3EncryptionClient, AmazonS3Client s3DecryptionClient, string bucketName) { var filePath = Path.GetTempFileName(); var retrievedFilepath = Path.GetTempFileName(); var totalSize = MegaBytesSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); var key = Guid.NewGuid().ToString(); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.OneZoneInfrequentAccess, ContentType = "text/html", }; InitiateMultipartUploadResponse initResponse = await s3EncryptionClient.InitiateMultipartUploadAsync(initRequest).ConfigureAwait(false); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegaBytesSize, InputStream = inputStream, }; UploadPartResponse up1Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegaBytesSize, InputStream = inputStream }; UploadPartResponse up2Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; UploadPartResponse up3Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false); Assert.Single(listPartResponse.Parts); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = await s3EncryptionClient.CompleteMultipartUploadAsync(compRequest).ConfigureAwait(false); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = await s3DecryptionClient.GetObjectAsync(getRequest).ConfigureAwait(false); await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, CancellationToken.None); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = await s3DecryptionClient.GetObjectMetadataAsync(metaDataRequest).ConfigureAwait(false); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Dispose(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
private static void UnmarshallResult(XmlUnmarshallerContext context, ListPartsResponse response) { int currentDepth = context.get_CurrentDepth(); int num = currentDepth + 1; if (context.get_IsStartOfDocument()) { num += 2; } while (context.Read()) { if (context.get_IsStartElement() || context.get_IsAttribute()) { if (context.TestExpression("Bucket", num)) { response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Key", num)) { response.Key = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("UploadId", num)) { response.UploadId = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("PartNumberMarker", num)) { response.PartNumberMarker = IntUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("NextPartNumberMarker", num)) { response.NextPartNumberMarker = IntUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("MaxParts", num)) { response.MaxParts = IntUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("IsTruncated", num)) { response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Part", num)) { response.Parts.Add(PartDetailUnmarshaller.Instance.Unmarshall(context)); } else if (context.TestExpression("Initiator", num)) { response.Initiator = InitiatorUnmarshaller.Instance.Unmarshall(context); } else if (context.TestExpression("Owner", num)) { response.Owner = OwnerUnmarshaller.Instance.Unmarshall(context); } else if (context.TestExpression("StorageClass", num)) { response.StorageClass = StringUnmarshaller.GetInstance().Unmarshall(context); } } else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth) { return; } } IWebResponseData responseData = context.get_ResponseData(); if (responseData.IsHeaderPresent("x-amz-abort-date")) { response.AbortDate = S3Transforms.ToDateTime(responseData.GetHeaderValue("x-amz-abort-date")); } if (responseData.IsHeaderPresent("x-amz-abort-rule-id")) { response.AbortRuleId = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-abort-rule-id")); } if (responseData.IsHeaderPresent(S3Constants.AmzHeaderRequestCharged)) { response.RequestCharged = RequestCharged.FindValue(responseData.GetHeaderValue(S3Constants.AmzHeaderRequestCharged)); } }
private static void UnmarshallResult(XmlUnmarshallerContext context, ListPartsResponse response) { int originalDepth = context.CurrentDepth; int targetDepth = originalDepth + 1; if (context.IsStartOfDocument) { targetDepth += 2; } while (context.Read()) { if (context.IsStartElement || context.IsAttribute) { if (context.TestExpression("Bucket", targetDepth)) { response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("Key", targetDepth)) { response.Key = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("UploadId", targetDepth)) { response.UploadId = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("PartNumberMarker", targetDepth)) { response.PartNumberMarker = IntUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("NextPartNumberMarker", targetDepth)) { response.NextPartNumberMarker = IntUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("MaxParts", targetDepth)) { response.MaxParts = IntUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("IsTruncated", targetDepth)) { response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("Part", targetDepth)) { response.Parts.Add(PartDetailUnmarshaller.Instance.Unmarshall(context)); continue; } if (context.TestExpression("Initiator", targetDepth)) { response.Initiator = InitiatorUnmarshaller.Instance.Unmarshall(context); continue; } if (context.TestExpression("Owner", targetDepth)) { response.Owner = OwnerUnmarshaller.Instance.Unmarshall(context); continue; } if (context.TestExpression("StorageClass", targetDepth)) { response.StorageClass = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } } else if (context.IsEndElement && context.CurrentDepth < originalDepth) { return; } } return; }
private static void UnmarshallResult(JsonUnmarshallerContext context, ListPartsResponse response) { int originalDepth = context.CurrentDepth; int targetDepth = originalDepth + 1; while (context.Read()) { if (context.TestExpression("MultipartUploadId", targetDepth)) { context.Read(); response.MultipartUploadId = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("VaultARN", targetDepth)) { context.Read(); response.VaultARN = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("ArchiveDescription", targetDepth)) { context.Read(); response.ArchiveDescription = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("PartSizeInBytes", targetDepth)) { context.Read(); response.PartSizeInBytes = LongUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("CreationDate", targetDepth)) { context.Read(); response.CreationDate = DateTimeUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.TestExpression("Parts", targetDepth)) { context.Read(); if (context.CurrentTokenType == JsonToken.Null) { response.Parts = null; continue; } response.Parts = new List <PartListElement>(); PartListElementUnmarshaller unmarshaller = PartListElementUnmarshaller.GetInstance(); while (context.Read()) { JsonToken token = context.CurrentTokenType; if (token == JsonToken.ArrayStart) { continue; } if (token == JsonToken.ArrayEnd) { break; } response.Parts.Add(unmarshaller.Unmarshall(context)); } continue; } if (context.TestExpression("Marker", targetDepth)) { context.Read(); response.Marker = StringUnmarshaller.GetInstance().Unmarshall(context); continue; } if (context.CurrentDepth <= originalDepth) { return; } } return; }
private static void Main(string[] args) { BosClient client = SampleUtil.GetSampleBosClient(); const string bucketName = "sample-bucket-multi-upload1"; //示例Bucket名称 // 初始化:创建示例Bucket client.CreateBucket(bucketName); //指定Bucket名称 string objectKey = "sample"; // 1.开始Multipart Upload InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = objectKey }; InitiateMultipartUploadResponse initiateMultipartUploadResponse = client.InitiateMultipartUpload(initiateMultipartUploadRequest); // 2.获取Bucket内的Multipart Upload ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest() { BucketName = bucketName }; ListMultipartUploadsResponse listMultipartUploadsResponse = client.ListMultipartUploads(listMultipartUploadsRequest); foreach (MultipartUploadSummary multipartUpload in listMultipartUploadsResponse.Uploads) { Console.WriteLine("Key: " + multipartUpload.Key + " UploadId: " + multipartUpload.UploadId); } // 3.分块上传,首先设置每块为 5Mb long partSize = 1024 * 1024 * 5L; FileInfo partFile = new FileInfo("d:\\lzb\\sample"); // 计算分块数目 int partCount = (int)(partFile.Length / partSize); if (partFile.Length % partSize != 0) { partCount++; } // 新建一个List保存每个分块上传后的ETag和PartNumber List <PartETag> partETags = new List <PartETag>(); for (int i = 0; i < partCount; i++) { // 获取文件流 Stream stream = partFile.OpenRead(); // 跳到每个分块的开头 long skipBytes = partSize * i; stream.Seek(skipBytes, SeekOrigin.Begin); // 计算每个分块的大小 long size = Math.Min(partSize, partFile.Length - skipBytes); // 创建UploadPartRequest,上传分块 UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.BucketName = bucketName; uploadPartRequest.Key = objectKey; uploadPartRequest.UploadId = initiateMultipartUploadResponse.UploadId; uploadPartRequest.InputStream = stream; uploadPartRequest.PartSize = size; uploadPartRequest.PartNumber = i + 1; UploadPartResponse uploadPartResponse = client.UploadPart(uploadPartRequest); // 将返回的PartETag保存到List中。 partETags.Add(new PartETag() { ETag = uploadPartResponse.ETag, PartNumber = uploadPartResponse.PartNumber }); // 关闭文件 stream.Close(); } // 4. 获取UploadId的所有Upload Part ListPartsRequest listPartsRequest = new ListPartsRequest() { BucketName = bucketName, Key = objectKey, UploadId = initiateMultipartUploadResponse.UploadId, }; // 获取上传的所有Part信息 ListPartsResponse listPartsResponse = client.ListParts(listPartsRequest); // 遍历所有Part foreach (PartSummary part in listPartsResponse.Parts) { Console.WriteLine("PartNumber: " + part.PartNumber + " ETag: " + part.ETag); } // 5. 完成分块上传 CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = objectKey, UploadId = initiateMultipartUploadResponse.UploadId, PartETags = partETags }; CompleteMultipartUploadResponse completeMultipartUploadResponse = client.CompleteMultipartUpload(completeMultipartUploadRequest); Console.WriteLine(completeMultipartUploadResponse.ETag); Console.ReadKey(); }
private async Task MultipartEncryptionTestAsync(AmazonS3EncryptionClient s3EncryptionClient) { var nextRandom = random.Next(); var filePath = @"C:\temp\multi-" + nextRandom + ".txt"; var retrievedFilepath = @"C:\temp\retreived-" + nextRandom + ".txt"; var totalSize = MegSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); string key = "key-" + random.Next(); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.ReducedRedundancy, ContentType = "text/html", CannedACL = S3CannedACL.PublicRead }; InitiateMultipartUploadResponse initResponse = await s3EncryptionClient.InitiateMultipartUploadAsync(initRequest).ConfigureAwait(false); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegSize, InputStream = inputStream, }; UploadPartResponse up1Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegSize, InputStream = inputStream, }; UploadPartResponse up2Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; UploadPartResponse up3Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false); Assert.Equal(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = await s3EncryptionClient.CompleteMultipartUploadAsync(compRequest).ConfigureAwait(false); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = await s3EncryptionClient.GetObjectAsync(getRequest).ConfigureAwait(false); await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, System.Threading.CancellationToken.None); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = await s3EncryptionClient.GetObjectMetadataAsync(metaDataRequest).ConfigureAwait(false); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Dispose(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }