public void TestV4SignedHeadersPut(CoreChecksumAlgorithm algorithm)
{
var putRequest = new PutObjectRequest()
{
BucketName = _bucketName,
Key = $"sigv4-headers-{algorithm}",
ContentBody = _testContent,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()),
UseChunkEncoding = false
};
PutAndGetChecksumTestHelper(algorithm, putRequest);
}
public void TestV4aSignedTrailersPut(CoreChecksumAlgorithm algorithm)
{
var putRequest = new PutObjectRequest()
{
BucketName = _mrapArn,
Key = $"sigv4a-trailers-{algorithm}",
ContentBody = _testContent,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()),
UseChunkEncoding = true
};
PutAndGetChecksumTestHelper(algorithm, putRequest);
}
public void TestV4UnsignedTrailersPut(CoreChecksumAlgorithm algorithm)
{
var putRequest = new PutObjectRequest()
{
BucketName = _bucketName,
Key = $"sigv4-unsignedcontent-trailers-{algorithm}",
ContentBody = _testContent,
DisablePayloadSigning = true,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()),
UseChunkEncoding = true
};
PutAndGetChecksumTestHelper(algorithm, putRequest);
}
public void TestSingleUploadViaTransferUtility(CoreChecksumAlgorithm algorithm)
{
var transferConfig = new TransferUtilityConfig {
MinSizeBeforePartUpload = 6000000
};
var transfer = new TransferUtility(Client, transferConfig);
var content = new string('a', 5000000);
var key = UtilityMethods.GenerateName(nameof(ChecksumTests));
var filePath = Path.Combine(Path.GetTempPath(), key + ".txt");
var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + key + ".txt");
try
{
// Create the file
using (StreamWriter writer = File.CreateText(filePath))
{
writer.Write(content);
}
var uploadRequest = new TransferUtilityUploadRequest
{
BucketName = _bucketName,
Key = key,
FilePath = filePath,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString())
};
transfer.Upload(uploadRequest);
// Get the file back from S3 and assert it is still the same.
var getRequest = new GetObjectRequest
{
BucketName = _bucketName,
Key = uploadRequest.Key,
ChecksumMode = ChecksumMode.ENABLED
};
var getResponse = Client.GetObject(getRequest);
var getBody = new StreamReader(getResponse.ResponseStream).ReadToEnd();
Assert.AreEqual(content, getBody);
Assert.AreEqual(algorithm.ToString(), getResponse.ResponseMetadata.ChecksumAlgorithm.ToString(), true);
Assert.AreEqual(ChecksumValidationStatus.PENDING_RESPONSE_READ, getResponse.ResponseMetadata.ChecksumValidationStatus);
// This should validate the checksum, so "assert" that no exceptions are thrown,
// though it doesn't expose the response metadata like above
transfer.Download(new TransferUtilityDownloadRequest
{
BucketName = _bucketName,
Key = uploadRequest.Key,
FilePath = retrievedFilepath,
ChecksumMode = ChecksumMode.ENABLED
});
}
finally
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}
public void TestMultipartUploadViaTransferUtility(CoreChecksumAlgorithm algorithm)
{
var transferConfig = new TransferUtilityConfig {
MinSizeBeforePartUpload = 6000000
};
var transfer = new TransferUtility(Client, transferConfig);
var content = new string('a', 7000000);
var key = UtilityMethods.GenerateName(nameof(ChecksumTests));
var filePath = Path.Combine(Path.GetTempPath(), key + ".txt");
var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + key + ".txt");
try
{
// Create the file
using (StreamWriter writer = File.CreateText(filePath))
{
writer.Write(content);
}
var uploadRequest = new TransferUtilityUploadRequest
{
BucketName = _bucketName,
Key = key,
FilePath = filePath,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString())
};
transfer.Upload(uploadRequest);
// Get the file back from S3 and assert it is still the same.
GetObjectRequest getRequest = new GetObjectRequest
{
BucketName = _bucketName,
Key = uploadRequest.Key,
ChecksumMode = ChecksumMode.ENABLED
};
var getResponse = Client.GetObject(getRequest);
var getBody = new StreamReader(getResponse.ResponseStream).ReadToEnd();
Assert.AreEqual(content, getBody);
// We don't expect the checksum to be validated on getting an entire multipart object,
// because it's actually the checksum-of-checksums
Assert.AreEqual(CoreChecksumAlgorithm.NONE, getResponse.ResponseMetadata.ChecksumAlgorithm);
Assert.AreEqual(ChecksumValidationStatus.NOT_VALIDATED, getResponse.ResponseMetadata.ChecksumValidationStatus);
// Similarily we don't expect this to validate either,
// though it doesn't expose the reponse metadata
transfer.Download(new TransferUtilityDownloadRequest
{
BucketName = _bucketName,
Key = uploadRequest.Key,
FilePath = retrievedFilepath,
ChecksumMode = ChecksumMode.ENABLED
});
}
finally
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}
/// <summary>
/// Test helper to test a multipart upload without using the Transfer Utility
/// </summary>
/// <param name="algorithm">checksum algorithm</param>
/// <param name="bucketName">bucket to upload the object to</param>
/// <param name="disablePayloadSigning">whether the request payload should be signed</param>
private void MultipartTestHelper(CoreChecksumAlgorithm algorithm, string bucketName, bool disablePayloadSigning)
{
var random = new Random();
var nextRandom = random.Next();
var filePath = Path.Combine(Path.GetTempPath(), "multi-" + nextRandom + ".txt");
var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + nextRandom + ".txt");
var totalSize = MegSize * 15;
UtilityMethods.GenerateFile(filePath, totalSize);
string key = "key-" + random.Next();
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString())
};
InitiateMultipartUploadResponse initResponse = Client.InitiateMultipartUpload(initRequest);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MegSize,
InputStream = inputStream,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()),
DisablePayloadSigning = disablePayloadSigning
};
UploadPartResponse up1Response = Client.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MegSize,
InputStream = inputStream,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()),
DisablePayloadSigning = disablePayloadSigning
};
UploadPartResponse up2Response = Client.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()),
DisablePayloadSigning = disablePayloadSigning,
IsLastPart = true
};
UploadPartResponse up3Response = Client.UploadPart(uploadRequest);
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = Client.ListParts(listPartRequest);
Assert.AreEqual(3, listPartResponse.Parts.Count);
AssertPartsAreEqual(up1Response, listPartResponse.Parts[0]);
AssertPartsAreEqual(up2Response, listPartResponse.Parts[1]);
AssertPartsAreEqual(up3Response, listPartResponse.Parts[2]);
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse = Client.CompleteMultipartUpload(compRequest);
Assert.IsNotNull(compResponse.ETag);
Assert.AreEqual(key, compResponse.Key);
Assert.IsNotNull(compResponse.Location);
// Get the file back from S3 and assert it is still the same.
var getRequest = new GetObjectRequest
{
BucketName = bucketName,
Key = key,
ChecksumMode = ChecksumMode.ENABLED
};
var getResponse = Client.GetObject(getRequest);
getResponse.WriteResponseStreamToFile(retrievedFilepath);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
// We don't expect the checksum to be validated on getting an entire multipart object,
// because it's actually the checksum-of-checksums
Assert.AreEqual(CoreChecksumAlgorithm.NONE, getResponse.ResponseMetadata.ChecksumAlgorithm);
Assert.AreEqual(ChecksumValidationStatus.NOT_VALIDATED, getResponse.ResponseMetadata.ChecksumValidationStatus);
}
finally
{
inputStream.Close();
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}
