public void CalculateChecksumTest(CoreChecksumAlgorithm algorithm, string content, string expectedBase64Checksum) { var contentBytes = Encoding.Default.GetBytes(content); var computedChecksum = Convert.ToBase64String(CryptoUtilFactory.GetChecksumInstance(algorithm).ComputeHash(contentBytes)); Assert.AreEqual(expectedBase64Checksum, computedChecksum); }
[DataRow(true, CoreChecksumAlgorithm.SHA256, "x-amz-checksum-crc32", "x-amz-checksum-crc32")] // Checksum specified but another was set, still calculate it anyway. public void TestRequestChecksumSelection(bool fallbackToMD5, CoreChecksumAlgorithm checksumAlgorithm, string originalHeaderKey, string expectedHeaderKey) { var mock = new Mock <IRequest>(); var headers = new Dictionary <string, string>(); var request = mock.Object; mock.SetupGet(x => x.Headers).Returns(headers); mock.SetupGet(x => x.Content).Returns(Encoding.ASCII.GetBytes("foo")); if (!string.IsNullOrEmpty(originalHeaderKey)) { headers.Add(originalHeaderKey, "foo"); } ChecksumUtils.SetRequestChecksum(request, checksumAlgorithm.ToString(), fallbackToMD5); if (!string.IsNullOrEmpty(expectedHeaderKey)) { Assert.IsTrue(request.Headers.ContainsKey(expectedHeaderKey)); Assert.IsFalse(string.IsNullOrEmpty(request.Headers[expectedHeaderKey])); } else { Assert.AreEqual(0, request.Headers.Count); } }
/// <summary> /// Puts and gets an object using a flexible checksum /// </summary> /// <param name="algorithm">Checksum algorithm to use</param> /// <param name="putRequest">PutObject request</param> private void PutAndGetChecksumTestHelper(CoreChecksumAlgorithm algorithm, PutObjectRequest putRequest) { Client.PutObject(putRequest); var getObjectAttributesRequest = new GetObjectAttributesRequest() { BucketName = putRequest.BucketName, Key = putRequest.Key, ObjectAttributes = new List <ObjectAttributes> { ObjectAttributes.Checksum } }; var getObjectAttributesResponse = Client.GetObjectAttributes(getObjectAttributesRequest); Assert.IsNotNull(getObjectAttributesResponse); var getRequest = new GetObjectRequest { BucketName = putRequest.BucketName, Key = putRequest.Key, ChecksumMode = ChecksumMode.ENABLED }; var response = Client.GetObject(getRequest); Assert.AreEqual(algorithm, response.ResponseMetadata.ChecksumAlgorithm); Assert.AreEqual(ChecksumValidationStatus.PENDING_RESPONSE_READ, response.ResponseMetadata.ChecksumValidationStatus); // Ensures checksum was calculated, an exception will have been thrown if it didn't match new StreamReader(response.ResponseStream).ReadToEnd(); response.ResponseStream.Dispose(); }
/// <summary> /// Initiates a stream wrapper to append trailing headers to an unsigned payload, /// with a trailing checksum /// </summary> /// <param name="baseStream">Stream to wrap</param> /// <param name="trailingHeaders">Header keys and values to append after the stream's conent</param> /// <param name="checksumAlgorithm">Algorithm to use to calculate the stream's checksum</param> public TrailingHeadersWrapperStream( Stream baseStream, IDictionary <string, string> trailingHeaders, CoreChecksumAlgorithm checksumAlgorithm) : this(baseStream, trailingHeaders) { if (checksumAlgorithm != CoreChecksumAlgorithm.NONE) { _checksumAlgorithm = checksumAlgorithm; _hashAlgorithm = CryptoUtilFactory.GetChecksumInstance(checksumAlgorithm); } }
public void TestV4SignedHeadersPut(CoreChecksumAlgorithm algorithm) { var putRequest = new PutObjectRequest() { BucketName = _bucketName, Key = $"sigv4-headers-{algorithm}", ContentBody = _testContent, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), UseChunkEncoding = false }; PutAndGetChecksumTestHelper(algorithm, putRequest); }
public void TestV4aSignedTrailersPut(CoreChecksumAlgorithm algorithm) { var putRequest = new PutObjectRequest() { BucketName = _mrapArn, Key = $"sigv4a-trailers-{algorithm}", ContentBody = _testContent, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), UseChunkEncoding = true }; PutAndGetChecksumTestHelper(algorithm, putRequest); }
public void TestTrailingChecksum(string checksumKey, CoreChecksumAlgorithm algorithm, string expectedContent) { var contentStream = new MemoryStream(Encoding.Default.GetBytes("Hello world")); var trailingHeaders = new Dictionary <string, string> { { checksumKey, "" } // checksum will be calculated as the stream is read then replaced at the end }; var wrappedStream = new TrailingHeadersWrapperStream(contentStream, trailingHeaders, algorithm); var actualContent = new StreamReader(wrappedStream).ReadToEnd(); Assert.AreEqual(expectedContent.Length, wrappedStream.Length); Assert.AreEqual(expectedContent, actualContent); }
public void TestV4UnsignedTrailersPut(CoreChecksumAlgorithm algorithm) { var putRequest = new PutObjectRequest() { BucketName = _bucketName, Key = $"sigv4-unsignedcontent-trailers-{algorithm}", ContentBody = _testContent, DisablePayloadSigning = true, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), UseChunkEncoding = true }; PutAndGetChecksumTestHelper(algorithm, putRequest); }
public void TestTrailingChecksum(string checksumKey, CoreChecksumAlgorithm algorithm, string expectedContent) { var contentStream = new MemoryStream(Encoding.Default.GetBytes("Hello world")); var trailingHeaders = new Dictionary <string, string> { { checksumKey, "" } // checksum will be calculated as the stream is read then replaced at the end }; var headerSigningResult = new AWS4SigningResult(_accessKey, _fixedSigningTimestamp, "", "", _signingKey, _headerSignature); var wrappedStream = new ChunkedUploadWrapperStream(contentStream, 8192, headerSigningResult, algorithm, trailingHeaders); var actualContent = new StreamReader(wrappedStream).ReadToEnd(); Assert.AreEqual(expectedContent.Length, wrappedStream.Length); Assert.AreEqual(expectedContent, actualContent); }
/// <summary> /// Initializes a chunked upload stream with one or more trailing headers, /// which may include a trailing checksum header /// </summary> /// <param name="stream">Stream to wrap</param> /// <param name="wrappedStreamBufferSize">Size of buffer used for reading from stream</param> /// <param name="headerSigningResult">SigV4 or SigV4a signing result for the request's headers</param> /// <param name="trailingChecksum">Algorithm to use to calculate the stream's checksum</param> /// <param name="trailingHeaders">Trailing headers to append after the wrapped stream</param> public ChunkedUploadWrapperStream(Stream stream, int wrappedStreamBufferSize, AWSSigningResultBase headerSigningResult, CoreChecksumAlgorithm trailingChecksum, IDictionary <string, string> trailingHeaders) : this(stream, wrappedStreamBufferSize, headerSigningResult) { if (trailingChecksum != CoreChecksumAlgorithm.NONE) { _trailingChecksum = trailingChecksum; _hashAlgorithm = CryptoUtilFactory.GetChecksumInstance(trailingChecksum); } _trailingHeadersConsumed = false; _trailingHeaders = trailingHeaders; }
/// <summary> /// Returns the length of the base64 encoded checksum of the specifed hashing algorithm /// </summary> /// <param name="algorithm">Hashing algorithm </param> /// <returns>Length of the base64 encoded checksum</returns> public static int GetChecksumBase64Length(CoreChecksumAlgorithm algorithm) { switch (algorithm) { case CoreChecksumAlgorithm.SHA1: return(SHA1_BASE64_LENGTH); case CoreChecksumAlgorithm.SHA256: return(SHA56_BASE64_LENGTH); case CoreChecksumAlgorithm.CRC32: case CoreChecksumAlgorithm.CRC32C: return(CRC32_BASE64_LENGTH); default: throw new AmazonClientException($"Unable to determine the base64-encoded length of {algorithm}"); } }
/// <summary> /// Returns a new instance of the specified hashing algorithm /// </summary> /// <param name="algorithm">Hashing algorithm to instantiate</param> /// <returns>New instance of the given algorithm</returns> public static HashAlgorithm GetChecksumInstance(CoreChecksumAlgorithm algorithm) { switch (algorithm) { case CoreChecksumAlgorithm.SHA1: return(new SHA1Managed()); case CoreChecksumAlgorithm.SHA256: return(CryptoUtil.CreateSHA256Instance()); case CoreChecksumAlgorithm.CRC32: return(new CrtCrc32()); case CoreChecksumAlgorithm.CRC32C: return(new CrtCrc32c()); default: throw new AmazonClientException($"Unable to instantiate checksum algorithm {algorithm}"); } }
/// <summary> /// Calculates the length in bytes of a TrailingChecksumWrapperStream initialized /// with the given trailing headers and optional checksum /// </summary> /// <param name="trailingHeaders">Dictionary of trailing headers</param> /// <param name="checksumAlgorithm">Trailing checksum</param> /// <param name="baseStreamLength">Length of the base stream in bytes</param> /// <returns>Length of a TrailingChecksumWrapperStream with given parameters, in bytes</returns> public static long CalculateLength(IDictionary <string, string> trailingHeaders, CoreChecksumAlgorithm checksumAlgorithm, long baseStreamLength) { var prefixLength = baseStreamLength.ToString("X", CultureInfo.InvariantCulture).Length; var trailingHeaderLength = 0; if (trailingHeaders != null) { foreach (var key in trailingHeaders.Keys) { if (checksumAlgorithm != CoreChecksumAlgorithm.NONE && ChecksumUtils.GetChecksumHeaderKey(checksumAlgorithm) == key) { trailingHeaderLength += key.Length + CryptoUtilFactory.GetChecksumBase64Length(checksumAlgorithm) + HEADER_ROW_PADDING_LENGTH; } else { trailingHeaderLength += key.Length + trailingHeaders[key].Length + HEADER_ROW_PADDING_LENGTH; } } } return(prefixLength + NEWLINE_LENGTH + baseStreamLength + NEWLINE_LENGTH + EMPTY_CHUNK_LENGTH + trailingHeaderLength + NEWLINE_LENGTH); }
public void TestV4aUnsignedMultipartUpload(CoreChecksumAlgorithm algorithm) { MultipartTestHelper(algorithm, _mrapArn, true); }
public void TestV4UnsignedMultipartUpload(CoreChecksumAlgorithm algorithm) { MultipartTestHelper(algorithm, _bucketName, true); }
/// <summary> /// Test helper to test a multipart upload without using the Transfer Utility /// </summary> /// <param name="algorithm">checksum algorithm</param> /// <param name="bucketName">bucket to upload the object to</param> /// <param name="disablePayloadSigning">whether the request payload should be signed</param> private void MultipartTestHelper(CoreChecksumAlgorithm algorithm, string bucketName, bool disablePayloadSigning) { var random = new Random(); var nextRandom = random.Next(); var filePath = Path.Combine(Path.GetTempPath(), "multi-" + nextRandom + ".txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + nextRandom + ".txt"); var totalSize = MegSize * 15; UtilityMethods.GenerateFile(filePath, totalSize); string key = "key-" + random.Next(); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()) }; InitiateMultipartUploadResponse initResponse = Client.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MegSize, InputStream = inputStream, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), DisablePayloadSigning = disablePayloadSigning }; UploadPartResponse up1Response = Client.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MegSize, InputStream = inputStream, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), DisablePayloadSigning = disablePayloadSigning }; UploadPartResponse up2Response = Client.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()), DisablePayloadSigning = disablePayloadSigning, IsLastPart = true }; UploadPartResponse up3Response = Client.UploadPart(uploadRequest); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = Client.ListParts(listPartRequest); Assert.AreEqual(3, listPartResponse.Parts.Count); AssertPartsAreEqual(up1Response, listPartResponse.Parts[0]); AssertPartsAreEqual(up2Response, listPartResponse.Parts[1]); AssertPartsAreEqual(up3Response, listPartResponse.Parts[2]); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = Client.CompleteMultipartUpload(compRequest); Assert.IsNotNull(compResponse.ETag); Assert.AreEqual(key, compResponse.Key); Assert.IsNotNull(compResponse.Location); // Get the file back from S3 and assert it is still the same. var getRequest = new GetObjectRequest { BucketName = bucketName, Key = key, ChecksumMode = ChecksumMode.ENABLED }; var getResponse = Client.GetObject(getRequest); getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); // We don't expect the checksum to be validated on getting an entire multipart object, // because it's actually the checksum-of-checksums Assert.AreEqual(CoreChecksumAlgorithm.NONE, getResponse.ResponseMetadata.ChecksumAlgorithm); Assert.AreEqual(ChecksumValidationStatus.NOT_VALIDATED, getResponse.ResponseMetadata.ChecksumValidationStatus); } finally { inputStream.Close(); if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
public void TestGetObjectResponseInvalidChecksum_ThrowsException(string header, string checksumValue, CoreChecksumAlgorithm expectedAlgorithm) { Exception exception = null; try { TestGetObjectResponseValidChecksum(header, checksumValue, expectedAlgorithm); } catch (Exception e) { exception = e; } Assert.IsNotNull(exception); Assert.IsInstanceOfType(exception, typeof(AmazonClientException)); Assert.AreEqual(exception.Message, "Expected hash not equal to calculated hash"); }
public void TestGetObjectResponseValidChecksum(string header, string checksumValue, CoreChecksumAlgorithm expectedAlgorithm) { Tester.Reset(); var context = CreateTestContext(); var request = new GetObjectRequest { BucketName = "foo", Key = "bar", ChecksumMode = ChecksumMode.ENABLED }; ((RequestContext)context.RequestContext).OriginalRequest = request; ((RequestContext)context.RequestContext).Request = new GetObjectRequestMarshaller().Marshall(request); ((RequestContext)context.RequestContext).Unmarshaller = GetObjectResponseUnmarshaller.Instance; var expectedResponseBody = "Hello world"; var response = MockWebResponse.Create(HttpStatusCode.OK, new Dictionary <string, string>(), expectedResponseBody); response.Headers.Add("Content-Length", "11"); response.Headers.Add(header, checksumValue); context.ResponseContext.HttpResponse = new HttpWebRequestResponseData(response); RuntimePipeline.InvokeSync(context); Assert.AreEqual(1, Tester.CallCount); Assert.IsInstanceOfType(context.ResponseContext.Response, typeof(GetObjectResponse)); var getObjectResponse = context.ResponseContext.Response as GetObjectResponse; Assert.AreEqual(expectedAlgorithm, getObjectResponse.ResponseMetadata.ChecksumAlgorithm); Assert.AreEqual(ChecksumValidationStatus.PENDING_RESPONSE_READ, getObjectResponse.ResponseMetadata.ChecksumValidationStatus); // Read the stream to the end to finish checksum calcuation and validation // This implicitly asserts that the checksum is valid because an exception would be thrown otherwise var responseBody = new StreamReader(getObjectResponse.ResponseStream).ReadToEnd(); Assert.AreEqual(expectedResponseBody, responseBody); }
public void TestMultipartUploadViaTransferUtility(CoreChecksumAlgorithm algorithm) { var transferConfig = new TransferUtilityConfig { MinSizeBeforePartUpload = 6000000 }; var transfer = new TransferUtility(Client, transferConfig); var content = new string('a', 7000000); var key = UtilityMethods.GenerateName(nameof(ChecksumTests)); var filePath = Path.Combine(Path.GetTempPath(), key + ".txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + key + ".txt"); try { // Create the file using (StreamWriter writer = File.CreateText(filePath)) { writer.Write(content); } var uploadRequest = new TransferUtilityUploadRequest { BucketName = _bucketName, Key = key, FilePath = filePath, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()) }; transfer.Upload(uploadRequest); // Get the file back from S3 and assert it is still the same. GetObjectRequest getRequest = new GetObjectRequest { BucketName = _bucketName, Key = uploadRequest.Key, ChecksumMode = ChecksumMode.ENABLED }; var getResponse = Client.GetObject(getRequest); var getBody = new StreamReader(getResponse.ResponseStream).ReadToEnd(); Assert.AreEqual(content, getBody); // We don't expect the checksum to be validated on getting an entire multipart object, // because it's actually the checksum-of-checksums Assert.AreEqual(CoreChecksumAlgorithm.NONE, getResponse.ResponseMetadata.ChecksumAlgorithm); Assert.AreEqual(ChecksumValidationStatus.NOT_VALIDATED, getResponse.ResponseMetadata.ChecksumValidationStatus); // Similarily we don't expect this to validate either, // though it doesn't expose the reponse metadata transfer.Download(new TransferUtilityDownloadRequest { BucketName = _bucketName, Key = uploadRequest.Key, FilePath = retrievedFilepath, ChecksumMode = ChecksumMode.ENABLED }); } finally { if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
public void TestSingleUploadViaTransferUtility(CoreChecksumAlgorithm algorithm) { var transferConfig = new TransferUtilityConfig { MinSizeBeforePartUpload = 6000000 }; var transfer = new TransferUtility(Client, transferConfig); var content = new string('a', 5000000); var key = UtilityMethods.GenerateName(nameof(ChecksumTests)); var filePath = Path.Combine(Path.GetTempPath(), key + ".txt"); var retrievedFilepath = Path.Combine(Path.GetTempPath(), "retreived-" + key + ".txt"); try { // Create the file using (StreamWriter writer = File.CreateText(filePath)) { writer.Write(content); } var uploadRequest = new TransferUtilityUploadRequest { BucketName = _bucketName, Key = key, FilePath = filePath, ChecksumAlgorithm = ChecksumAlgorithm.FindValue(algorithm.ToString()) }; transfer.Upload(uploadRequest); // Get the file back from S3 and assert it is still the same. var getRequest = new GetObjectRequest { BucketName = _bucketName, Key = uploadRequest.Key, ChecksumMode = ChecksumMode.ENABLED }; var getResponse = Client.GetObject(getRequest); var getBody = new StreamReader(getResponse.ResponseStream).ReadToEnd(); Assert.AreEqual(content, getBody); Assert.AreEqual(algorithm.ToString(), getResponse.ResponseMetadata.ChecksumAlgorithm.ToString(), true); Assert.AreEqual(ChecksumValidationStatus.PENDING_RESPONSE_READ, getResponse.ResponseMetadata.ChecksumValidationStatus); // This should validate the checksum, so "assert" that no exceptions are thrown, // though it doesn't expose the response metadata like above transfer.Download(new TransferUtilityDownloadRequest { BucketName = _bucketName, Key = uploadRequest.Key, FilePath = retrievedFilepath, ChecksumMode = ChecksumMode.ENABLED }); } finally { if (File.Exists(filePath)) { File.Delete(filePath); } if (File.Exists(retrievedFilepath)) { File.Delete(retrievedFilepath); } } }
/// <summary> /// Generates the name of the header key to use for a given checksum algorithm /// </summary> /// <param name="checksumAlgorithm">Checksum algorithm</param> /// <returns>Name of the HTTP header key for the given algorithm</returns> internal static string GetChecksumHeaderKey(CoreChecksumAlgorithm checksumAlgorithm) { return($"x-amz-checksum-{checksumAlgorithm.ToString().ToLower()}"); }
/// <summary> /// Computes the total size of the data payload, including the chunk metadata /// and optional trailing headers. Called externally so as to be able to set /// the correct Content-Length header value. /// </summary> /// <param name="originalLength">Length of the wrapped stream</param> /// <param name="signatureLength">Length of the signature for each chunk, in bytes</param> /// <param name="trailingHeaders">Optional trailing headers</param> /// <param name="trailingChecksum">Optional checksum algorithm for a trailing header</param> /// <returns>Total size of the wrapped payload, in bytes</returns> public static long ComputeChunkedContentLength(long originalLength, int signatureLength, IDictionary <string, string> trailingHeaders, CoreChecksumAlgorithm trailingChecksum) { if (originalLength < 0) { throw new ArgumentOutOfRangeException("originalLength", "Expected 0 or greater value for originalLength."); } int trailingHeaderLength = 0; long chunkedContentLength; // Calculate the size of the chunked content, before trailing headers/checksum if (originalLength == 0) { chunkedContentLength = CalculateChunkHeaderLength(0, signatureLength); } else { var maxSizeChunks = originalLength / DefaultChunkSize; var remainingBytes = originalLength % DefaultChunkSize; chunkedContentLength = maxSizeChunks * CalculateChunkHeaderLength(DefaultChunkSize, signatureLength) + (remainingBytes > 0 ? CalculateChunkHeaderLength(remainingBytes, signatureLength) : 0) + CalculateChunkHeaderLength(0, signatureLength); } if (trailingHeaders?.Count > 0) { foreach (var key in trailingHeaders.Keys) { // If the trailing checksum key is already in dictionary, use the // expected length since the checksum value may not be set yet. if (trailingChecksum != CoreChecksumAlgorithm.NONE && ChecksumUtils.GetChecksumHeaderKey(trailingChecksum) == key) { trailingHeaderLength += key.Length + CryptoUtilFactory.GetChecksumBase64Length(trailingChecksum) + HEADER_ROW_PADDING_LENGTH; } else { trailingHeaderLength += key.Length + trailingHeaders[key].Length + HEADER_ROW_PADDING_LENGTH; } } trailingHeaderLength += TRAILING_HEADER_SIGNATURE_KEY.Length + signatureLength + HEADER_ROW_PADDING_LENGTH; } return(chunkedContentLength + trailingHeaderLength); }