public static void ValidateFileContents(IAmazonS3 s3client, string bucketName, string key, string path, string contentType)
{
var downloadPath = path + ".chk";
var request = new GetObjectRequest
{
BucketName = bucketName,
Key = key,
};
UtilityMethods.WaitUntil(() =>
{
using (var response = s3client.GetObject(request))
{
if (!string.IsNullOrEmpty(contentType))
{
Assert.Equal(contentType, response.Headers.ContentType);
}
response.WriteResponseStreamToFile(downloadPath);
}
return(true);
}, sleepSeconds: 2, maxWaitSeconds: 10);
UtilityMethods.CompareFiles(path, downloadPath);
}
public static void MultipartEncryptionTestAPM(IAmazonS3 s3EncryptionClient, IAmazonS3 s3DecryptionClient, string bucketName)
{
var guid = Guid.NewGuid();
var filePath = Path.Combine(Path.GetTempPath(), $"multi-{guid}.txt");
var retrievedFilepath = Path.Combine(Path.GetTempPath(), $"retrieved-{guid}.txt");
var totalSize = MegaByteSize * 15;
UtilityMethods.GenerateFile(filePath, totalSize);
string key = $"key-{guid}";
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.OneZoneInfrequentAccess,
ContentType = "text/html"
};
InitiateMultipartUploadResponse initResponse = null;
if (IsKMSEncryptionClient(s3EncryptionClient))
{
initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest);
}
else
{
initResponse = s3EncryptionClient.EndInitiateMultipartUpload(
s3EncryptionClient.BeginInitiateMultipartUpload(initRequest, null, null));
}
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MegaByteSize,
InputStream = inputStream,
};
UploadPartResponse up1Response = s3EncryptionClient.EndUploadPart(
s3EncryptionClient.BeginUploadPart(uploadRequest, null, null));
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MegaByteSize,
InputStream = inputStream,
};
UploadPartResponse up2Response = s3EncryptionClient.EndUploadPart(
s3EncryptionClient.BeginUploadPart(uploadRequest, null, null));
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
UploadPartResponse up3Response = s3EncryptionClient.EndUploadPart(
s3EncryptionClient.BeginUploadPart(uploadRequest, null, null));
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = s3EncryptionClient.EndListParts(
s3EncryptionClient.BeginListParts(listPartRequest, null, null));
Assert.Equal(3, listPartResponse.Parts.Count);
Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = s3EncryptionClient.EndListParts(
s3EncryptionClient.BeginListParts(listPartRequest, null, null));
Assert.Equal(1, listPartResponse.Parts.Count);
// Complete the response
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse = s3EncryptionClient.EndCompleteMultipartUpload(
s3EncryptionClient.BeginCompleteMultipartUpload(compRequest, null, null));
Assert.Equal(bucketName, compResponse.BucketName);
Assert.NotNull(compResponse.ETag);
Assert.Equal(key, compResponse.Key);
Assert.NotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
GetObjectRequest getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectResponse getResponse = null;
if (IsKMSEncryptionClient(s3EncryptionClient))
{
getResponse = s3DecryptionClient.GetObject(getRequest);
}
else
{
getResponse = s3DecryptionClient.EndGetObject(
s3DecryptionClient.BeginGetObject(getRequest, null, null));
}
getResponse.WriteResponseStreamToFile(retrievedFilepath);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectMetadataResponse metaDataResponse = s3DecryptionClient.EndGetObjectMetadata(
s3DecryptionClient.BeginGetObjectMetadata(metaDataRequest, null, null));
Assert.Equal("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Close();
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}
public static void MultipartEncryptionTest(AmazonS3Client s3EncryptionClient, IAmazonS3 s3DecryptionClient, string bucketName)
{
var guid = Guid.NewGuid();
var filePath = Path.Combine(Path.GetTempPath(), $"multi-{guid}.txt");
var retrievedFilepath = Path.Combine(Path.GetTempPath(), $"retrieved-{guid}.txt");
var totalSize = MegaByteSize * 15;
UtilityMethods.GenerateFile(filePath, totalSize);
var key = $"key-{guid}";
Stream inputStream = File.OpenRead(filePath);
try
{
var initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.OneZoneInfrequentAccess,
ContentType = "text/html",
};
var initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest);
// Upload part 1
var uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MegaByteSize,
InputStream = inputStream
};
var up1Response = s3EncryptionClient.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MegaByteSize,
InputStream = inputStream
};
var up2Response = s3EncryptionClient.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
var up3Response = s3EncryptionClient.UploadPart(uploadRequest);
var listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
var listPartResponse = s3EncryptionClient.ListParts(listPartRequest);
Assert.Equal(3, listPartResponse.Parts.Count);
Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = s3EncryptionClient.ListParts(listPartRequest);
Assert.Equal(1, listPartResponse.Parts.Count);
// Complete the response
var compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
var compResponse = s3EncryptionClient.CompleteMultipartUpload(compRequest);
Assert.Equal(bucketName, compResponse.BucketName);
Assert.NotNull(compResponse.ETag);
Assert.Equal(key, compResponse.Key);
Assert.NotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
var getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
var getResponse = s3DecryptionClient.GetObject(getRequest);
getResponse.WriteResponseStreamToFile(retrievedFilepath);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
var metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
var metaDataResponse = s3DecryptionClient.GetObjectMetadata(metaDataRequest);
Assert.Equal("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Close();
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
#if ASYNC_AWAIT
// run the async version of the same test
WaitForAsyncTask(MultipartEncryptionTestAsync(s3EncryptionClient, s3DecryptionClient, bucketName));
#elif AWS_APM_API
// run the APM version of the same test
MultipartEncryptionTestAPM(s3EncryptionClient, s3DecryptionClient, bucketName);
#endif
}
public static async Task MultipartEncryptionTestAsync(AmazonS3Client s3EncryptionClient,
AmazonS3Client s3DecryptionClient, string bucketName)
{
var filePath = Path.GetTempFileName();
var retrievedFilepath = Path.GetTempFileName();
var totalSize = MegaBytesSize * 15;
UtilityMethods.GenerateFile(filePath, totalSize);
var key = Guid.NewGuid().ToString();
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.OneZoneInfrequentAccess,
ContentType = "text/html",
};
InitiateMultipartUploadResponse initResponse =
await s3EncryptionClient.InitiateMultipartUploadAsync(initRequest).ConfigureAwait(false);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MegaBytesSize,
InputStream = inputStream,
};
UploadPartResponse up1Response =
await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MegaBytesSize,
InputStream = inputStream
};
UploadPartResponse up2Response =
await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
UploadPartResponse up3Response =
await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false);
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse =
await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false);
Assert.Equal(3, listPartResponse.Parts.Count);
Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false);
Assert.Single(listPartResponse.Parts);
// Complete the response
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse =
await s3EncryptionClient.CompleteMultipartUploadAsync(compRequest).ConfigureAwait(false);
Assert.Equal(bucketName, compResponse.BucketName);
Assert.NotNull(compResponse.ETag);
Assert.Equal(key, compResponse.Key);
Assert.NotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
GetObjectRequest getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectResponse getResponse =
await s3DecryptionClient.GetObjectAsync(getRequest).ConfigureAwait(false);
await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, CancellationToken.None);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectMetadataResponse metaDataResponse =
await s3DecryptionClient.GetObjectMetadataAsync(metaDataRequest).ConfigureAwait(false);
Assert.Equal("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Dispose();
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}