public static void ValidateFileContents(IAmazonS3 s3client, string bucketName, string key, string path, string contentType) { var downloadPath = path + ".chk"; var request = new GetObjectRequest { BucketName = bucketName, Key = key, }; UtilityMethods.WaitUntil(() => { using (var response = s3client.GetObject(request)) { if (!string.IsNullOrEmpty(contentType)) { Assert.Equal(contentType, response.Headers.ContentType); } response.WriteResponseStreamToFile(downloadPath); } return(true); }, sleepSeconds: 2, maxWaitSeconds: 10); UtilityMethods.CompareFiles(path, downloadPath); }
public static void MultipartEncryptionTestAPM(IAmazonS3 s3EncryptionClient, IAmazonS3 s3DecryptionClient, string bucketName) { var guid = Guid.NewGuid(); var filePath = Path.Combine(Path.GetTempPath(), $"multi-{guid}.txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), $"retrieved-{guid}.txt"); var totalSize = MegaByteSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); string key = $"key-{guid}"; Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.OneZoneInfrequentAccess, ContentType = "text/html" }; InitiateMultipartUploadResponse initResponse = null; if (IsKMSEncryptionClient(s3EncryptionClient)) { initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest); } else { initResponse = s3EncryptionClient.EndInitiateMultipartUpload( s3EncryptionClient.BeginInitiateMultipartUpload(initRequest, null, null)); } // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegaByteSize, InputStream = inputStream, }; UploadPartResponse up1Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegaByteSize, InputStream = inputStream, }; UploadPartResponse up2Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; UploadPartResponse up3Response = s3EncryptionClient.EndUploadPart( s3EncryptionClient.BeginUploadPart(uploadRequest, null, null)); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = s3EncryptionClient.EndListParts( s3EncryptionClient.BeginListParts(listPartRequest, null, null)); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClient.EndListParts( s3EncryptionClient.BeginListParts(listPartRequest, null, null)); Assert.Equal(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = s3EncryptionClient.EndCompleteMultipartUpload( s3EncryptionClient.BeginCompleteMultipartUpload(compRequest, null, null)); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = null; if (IsKMSEncryptionClient(s3EncryptionClient)) { getResponse = s3DecryptionClient.GetObject(getRequest); } else { getResponse = s3DecryptionClient.EndGetObject( s3DecryptionClient.BeginGetObject(getRequest, null, null)); } getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = s3DecryptionClient.EndGetObjectMetadata( s3DecryptionClient.BeginGetObjectMetadata(metaDataRequest, null, null)); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
public static void MultipartEncryptionTest(AmazonS3Client s3EncryptionClient, IAmazonS3 s3DecryptionClient, string bucketName) { var guid = Guid.NewGuid(); var filePath = Path.Combine(Path.GetTempPath(), $"multi-{guid}.txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), $"retrieved-{guid}.txt"); var totalSize = MegaByteSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); var key = $"key-{guid}"; Stream inputStream = File.OpenRead(filePath); try { var initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.OneZoneInfrequentAccess, ContentType = "text/html", }; var initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest); // Upload part 1 var uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegaByteSize, InputStream = inputStream }; var up1Response = s3EncryptionClient.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegaByteSize, InputStream = inputStream }; var up2Response = s3EncryptionClient.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; var up3Response = s3EncryptionClient.UploadPart(uploadRequest); var listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; var listPartResponse = s3EncryptionClient.ListParts(listPartRequest); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClient.ListParts(listPartRequest); Assert.Equal(1, listPartResponse.Parts.Count); // Complete the response var compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); var compResponse = s3EncryptionClient.CompleteMultipartUpload(compRequest); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. var getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; var getResponse = s3DecryptionClient.GetObject(getRequest); getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); var metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; var metaDataResponse = s3DecryptionClient.GetObjectMetadata(metaDataRequest); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } #if ASYNC_AWAIT // run the async version of the same test WaitForAsyncTask(MultipartEncryptionTestAsync(s3EncryptionClient, s3DecryptionClient, bucketName)); #elif AWS_APM_API // run the APM version of the same test MultipartEncryptionTestAPM(s3EncryptionClient, s3DecryptionClient, bucketName); #endif }
public static async Task MultipartEncryptionTestAsync(AmazonS3Client s3EncryptionClient, AmazonS3Client s3DecryptionClient, string bucketName) { var filePath = Path.GetTempFileName(); var retrievedFilepath = Path.GetTempFileName(); var totalSize = MegaBytesSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); var key = Guid.NewGuid().ToString(); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.OneZoneInfrequentAccess, ContentType = "text/html", }; InitiateMultipartUploadResponse initResponse = await s3EncryptionClient.InitiateMultipartUploadAsync(initRequest).ConfigureAwait(false); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegaBytesSize, InputStream = inputStream, }; UploadPartResponse up1Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegaBytesSize, InputStream = inputStream }; UploadPartResponse up2Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; UploadPartResponse up3Response = await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false); Assert.Equal(3, listPartResponse.Parts.Count); Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false); Assert.Single(listPartResponse.Parts); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = await s3EncryptionClient.CompleteMultipartUploadAsync(compRequest).ConfigureAwait(false); Assert.Equal(bucketName, compResponse.BucketName); Assert.NotNull(compResponse.ETag); Assert.Equal(key, compResponse.Key); Assert.NotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = await s3DecryptionClient.GetObjectAsync(getRequest).ConfigureAwait(false); await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, CancellationToken.None); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = await s3DecryptionClient.GetObjectMetadataAsync(metaDataRequest).ConfigureAwait(false); Assert.Equal("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Dispose(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }