public void ChecksumAndOtherTrailingHeaders()
{
var contentStream = new MemoryStream(Encoding.Default.GetBytes("Hello world"));
var trailingHeaders = new Dictionary <string, string>
{
{ "header-a", "value-a" },
{ "header-b", "value-b" },
{ "x-amz-checksum-sha256", "" } // checksum will be calculated as the stream is read then replaced at the end
};
var headerSigningResult = new AWS4SigningResult(_accessKey, _fixedSigningTimestamp, "", "", _signingKey, _headerSignature);
var wrappedStream = new ChunkedUploadWrapperStream(contentStream, 8192, headerSigningResult, CoreChecksumAlgorithm.SHA256, trailingHeaders);
var actualContent = new StreamReader(wrappedStream).ReadToEnd();
var expectedContent =
"B;chunk-signature=6a4d50a3307c001ad83900a73442136a0a0f203520fd8c0e966f655cc830bbe8\r\n" +
"Hello world\r\n" +
"0;chunk-signature=9384094dc67fd7c29a4c7e0aa3866233b3774e41d1470b8f51a96becbd91f60c\r\n" +
"header-a:value-a\r\n" +
"header-b:value-b\r\n" +
"x-amz-checksum-sha256:ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=\r\n" +
"x-amz-trailer-signature:0e357651fe17d33ea0d4173b2fd745d596354a66347ee8b5e4de44036c62becb\r\n" +
"\r\n";
Assert.AreEqual(expectedContent.Length, wrappedStream.Length);
Assert.AreEqual(expectedContent, actualContent);
}
public void NonChecksumTrailingHeaders()
{
var contentStream = new MemoryStream(Encoding.Default.GetBytes("Hello world"));
var trailingHeaders = new Dictionary <string, string>
{
{ "header-a", "value-a" },
{ "header-b", "value-b" }
};
var headerSigningResult = new AWS4SigningResult(_accessKey, _fixedSigningTimestamp, "", "", _signingKey, _headerSignature);
var wrappedStream = new ChunkedUploadWrapperStream(contentStream, 8192, headerSigningResult, CoreChecksumAlgorithm.NONE, trailingHeaders);
var actualContent = new StreamReader(wrappedStream).ReadToEnd();
var expectedContent =
"B;chunk-signature=6a4d50a3307c001ad83900a73442136a0a0f203520fd8c0e966f655cc830bbe8\r\n" +
"Hello world\r\n" +
"0;chunk-signature=9384094dc67fd7c29a4c7e0aa3866233b3774e41d1470b8f51a96becbd91f60c\r\n" +
"header-a:value-a\r\n" +
"header-b:value-b\r\n" +
"x-amz-trailer-signature:ff2899b910cb941057f921d11738b5d875f2e117ea9504cfea4f9c7a788a0852\r\n" +
"\r\n";
Assert.AreEqual(expectedContent.Length, wrappedStream.Length);
Assert.AreEqual(expectedContent, actualContent);
}
public void LargeData()
{
var data = new string('a', 100000);
var contentStream = new MemoryStream(Encoding.Default.GetBytes(data));
var trailingHeaders = new Dictionary <string, string>
{
{ "x-amz-checksum-sha256", "" } // checksum will be calculated as the stream is read then replaced at the end
};
var headerSigningResult = new AWS4SigningResult(_accessKey, _fixedSigningTimestamp, "", "", _signingKey, _headerSignature);
var wrappedStream = new ChunkedUploadWrapperStream(contentStream, 8192, headerSigningResult, CoreChecksumAlgorithm.SHA256, trailingHeaders);
var actualContent = new StreamReader(wrappedStream).ReadToEnd();
var expectedContent =
"14000;chunk-signature=dd6818ebe851d9f6006431fac25c71960881bf8d86501344f19a43c1a2c2a9a7\r\n" +
$"{new string('a', 81920)}\r\n" +
"46A0;chunk-signature=88f059571f2bc312c8fd7ebd2082f587561ca83a6d5bb863e6e3d4ef78819256\r\n" +
$"{new string('a', 18080)}\r\n" +
"0;chunk-signature=07857eed4f1a53ae7d042e0c410053d610ed947827ca1c5572ed401666732805\r\n" +
"x-amz-checksum-sha256:bRzyLXzAmwhd/CXuGh864CZYBMYHvCB0rSU7zIL9ge4=\r\n" +
"x-amz-trailer-signature:cd8a0ba8bc4abaaa5f6e9921c30ca6e018110bcbaa3ea876e7b777b924bb4009\r\n" +
"\r\n";
Assert.AreEqual(expectedContent.Length, wrappedStream.Length);
Assert.AreEqual(expectedContent, actualContent);
}
internal string GetExpectedCanonicalRequestForChunkedTrailersSigningTest()
{
var trailingHeaders = new Dictionary <string, string>
{
{ "x-amz-foo", "bar" }
};
return(string.Join('\n',
"PUT",
"/examplebucket/chunkObject.txt",
"",
"content-encoding:aws-chunked",
"content-length:" + ChunkedUploadWrapperStream.ComputeChunkedContentLength(Chunk1Size + Chunk2Size,
ChunkedUploadWrapperStream.V4A_SIGNATURE_LENGTH,
trailingHeaders,
CoreChecksumAlgorithm.NONE).ToString(),
"host:s3.amazonaws.com",
"x-amz-content-sha256:STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER",
"x-amz-date:20150830T123600Z",
"x-amz-decoded-content-length:" + (Chunk1Size + Chunk2Size).ToString(),
"x-amz-region-set:us-east-1",
"x-amz-storage-class:REDUCED_REDUNDANCY",
"x-amz-trailer:x-amz-foo",
"",
"content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set;x-amz-storage-class;x-amz-trailer",
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"));
}
private string BuildV4aChunkedStringToSignHelper(AWS4aSigningResult headerResult, string previousSignature, int chunkSize)
{
return(ChunkedUploadWrapperStream.BuildChunkedStringToSign(
"AWS4-ECDSA-P256-SHA256-PAYLOAD",
headerResult.ISO8601DateTime,
headerResult.Scope,
previousSignature.TrimEnd('*'),
chunkSize,
CreateChunkStream(chunkSize).ToArray()));
}
public static string SetRequestBodyHash(IRequest request)
{
string value = null;
bool flag = request.Headers.TryGetValue("X-Amz-Content-SHA256", out value);
if (flag && !request.UseChunkEncoding)
{
return(value);
}
if (request.UseChunkEncoding)
{
value = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
if (request.Headers.ContainsKey("Content-Length"))
{
request.Headers["X-Amz-Decoded-Content-Length"] = request.Headers["Content-Length"];
long originalLength = long.Parse(request.Headers["Content-Length"], CultureInfo.InvariantCulture);
request.Headers["Content-Length"] = ChunkedUploadWrapperStream.ComputeChunkedContentLength(originalLength).ToString(CultureInfo.InvariantCulture);
}
if (request.Headers.ContainsKey("Content-Encoding"))
{
string text = request.Headers["Content-Encoding"];
if (!text.Contains("aws-chunked"))
{
request.Headers["Content-Encoding"] = text + ", " + "aws-chunked";
}
}
else
{
request.Headers["Content-Encoding"] = "aws-chunked";
}
}
else if (request.ContentStream != null)
{
value = request.ComputeContentStreamHash();
}
else
{
byte[] requestPayloadBytes = GetRequestPayloadBytes(request);
value = AWSSDKUtils.ToHex(CryptoUtilFactory.CryptoInstance.ComputeSHA256Hash(requestPayloadBytes), lowercase: true);
}
if (value != null)
{
if (flag)
{
request.Headers["X-Amz-Content-SHA256"] = value;
}
else
{
request.Headers.Add("X-Amz-Content-SHA256", value);
}
}
return(value);
}
/// <summary>
/// If the caller has already set the x-amz-content-sha256 header with a pre-computed
/// content hash, or it is present as ContentStreamHash on the request instance, return
/// the value to be used in request canonicalization.
/// If not set as a header or in the request, attempt to compute a hash based on
/// inspection of the style of the request content.
/// </summary>
/// <param name="request"></param>
/// <returns>
/// The computed hash, whether already set in headers or computed here. Null
/// if we were not able to compute a hash.
/// </returns>
public static string SetRequestBodyHash(IRequest request)
{
string computedContentHash = null;
if (request.Headers.TryGetValue(XAmzContentSha256, out computedContentHash))
{
return(computedContentHash);
}
if (request.UseChunkEncoding)
{
computedContentHash = StreamingBodySha256;
if (request.Headers.ContainsKey("Content-Length"))
{
// substitute the originally declared content length with the true size of
// the data we'll upload, which is inflated with chunk metadata
request.Headers[XAmzDecodedContentLength] = request.Headers["Content-Length"];
var originalContentLength = long.Parse(request.Headers["Content-Length"], CultureInfo.InvariantCulture);
request.Headers["Content-Length"]
= ChunkedUploadWrapperStream.ComputeChunkedContentLength(originalContentLength).ToString(CultureInfo.InvariantCulture);
}
request.Headers["Content-Encoding"] = "aws-chunked";
}
else
{
if (request.ContentStream != null)
{
computedContentHash = request.ComputeContentStreamHash();
}
else
{
byte[] payloadHashBytes;
if (request.Content != null)
{
payloadHashBytes = CryptoUtilFactory.CryptoInstance.ComputeSHA256Hash(request.Content);
}
else
{
var payload = request.UseQueryString ? "" : GetRequestPayload(request);
payloadHashBytes = CryptoUtilFactory.CryptoInstance.ComputeSHA256Hash(Encoding.UTF8.GetBytes(payload));
}
computedContentHash = AWSSDKUtils.ToHex(payloadHashBytes, true);
}
}
if (computedContentHash != null)
{
request.Headers.Add(XAmzContentSha256, computedContentHash);
}
return(computedContentHash);
}
public void TestTrailingChecksum(string checksumKey, CoreChecksumAlgorithm algorithm, string expectedContent)
{
var contentStream = new MemoryStream(Encoding.Default.GetBytes("Hello world"));
var trailingHeaders = new Dictionary <string, string>
{
{ checksumKey, "" } // checksum will be calculated as the stream is read then replaced at the end
};
var headerSigningResult = new AWS4SigningResult(_accessKey, _fixedSigningTimestamp, "", "", _signingKey, _headerSignature);
var wrappedStream = new ChunkedUploadWrapperStream(contentStream, 8192, headerSigningResult, algorithm, trailingHeaders);
var actualContent = new StreamReader(wrappedStream).ReadToEnd();
Assert.AreEqual(expectedContent.Length, wrappedStream.Length);
Assert.AreEqual(expectedContent, actualContent);
}
internal string GetExpectedCanonicalRequestForChunkedSigningTest()
{
return(String.Join('\n',
"PUT",
"/examplebucket/chunkObject.txt",
"",
"content-encoding:aws-chunked",
"content-length:" + ChunkedUploadWrapperStream.ComputeChunkedContentLength(Chunk1Size + Chunk2Size, ChunkedUploadWrapperStream.V4A_SIGNATURE_LENGTH).ToString(),
"host:s3.amazonaws.com",
"x-amz-content-sha256:STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD",
"x-amz-date:20150830T123600Z",
"x-amz-decoded-content-length:" + (Chunk1Size + Chunk2Size).ToString(),
"x-amz-region-set:us-east-1",
"x-amz-storage-class:REDUCED_REDUNDANCY",
"",
"content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set;x-amz-storage-class",
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"));
}
public void SmallBuffer()
{
var contentStream = new MemoryStream(Encoding.Default.GetBytes("Hello world"));
var trailingHeaders = new Dictionary <string, string>
{
{ "x-amz-checksum-sha256", "" } // checksum will be calculated as the stream is read then replaced at the end
};
var headerSigningResult = new AWS4SigningResult(_accessKey, _fixedSigningTimestamp, "", "", _signingKey, _headerSignature);
var wrappedStream = new ChunkedUploadWrapperStream(contentStream, 1, headerSigningResult, CoreChecksumAlgorithm.SHA256, trailingHeaders);
var bytesRead = -1;
var bufferSize = 1;
var buffer = new byte[bufferSize];
var outputAsString = "";
while (bytesRead != 0)
{
bytesRead = wrappedStream.Read(buffer, 0, bufferSize);
outputAsString += Encoding.Default.GetString(buffer, 0, bytesRead);
}
Assert.AreEqual(expectedSHA256Content.Length, wrappedStream.Length);
Assert.AreEqual(expectedSHA256Content, outputAsString);
}
void SetContent(HttpRequestMessage requestMessage, IRequestData state)
{
var request = state.Request;
if (requestMessage.Method == HttpMethod.Get ||
requestMessage.Method == HttpMethod.Delete ||
requestMessage.Method == HttpMethod.Head)
{
return;
}
byte[] requestData;
if (request.UseQueryString)
{
requestMessage.Content = new ByteArrayContent(new Byte[0]);
}
else if (request.ContentStream != null)
{
var eventStream = new EventStream(request.ContentStream, true);
var tracker = new StreamReadTracker(this, request.OriginalRequest.StreamUploadProgressCallback,
request.ContentStream.Length, this.Config.ProgressUpdateInterval);
eventStream.OnRead += tracker.ReadProgress;
Stream finalStream;
StreamContent content;
if (state.Request.UseChunkEncoding && state.Request.AWS4SignerResult != null)
{
finalStream = new ChunkedUploadWrapperStream(eventStream, Config.BufferSize, state.Request.AWS4SignerResult);
}
else
{
finalStream = eventStream;
}
content = new StreamContent(finalStream, this.Config.BufferSize);
requestMessage.Content = content;
requestMessage.Content.Headers.ContentLength = finalStream.Length;
}
else if ((requestData = GetRequestData(request)) != null)
{
state.Metrics.AddProperty(Metric.RequestSize, requestData.Length);
requestMessage.Content = new ByteArrayContent(requestData);
}
else
{
requestMessage.Content = new ByteArrayContent(new Byte[0]);
}
requestMessage.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(request.Headers["Content-Type"]);
if (requestMessage.Content != null)
{
if (request.Headers.ContainsKey("Content-Range"))
{
requestMessage.Content.Headers.Add("Content-Range", request.Headers["Content-Range"]);
}
if (request.Headers.ContainsKey("Content-MD5"))
{
requestMessage.Content.Headers.TryAddWithoutValidation("Content-MD5", request.Headers["Content-MD5"]);
}
if (request.Headers.ContainsKey("Content-Encoding"))
{
requestMessage.Content.Headers.TryAddWithoutValidation("Content-Encoding", request.Headers["Content-Encoding"]);
}
if (request.Headers.ContainsKey("Content-Disposition"))
{
requestMessage.Content.Headers.TryAddWithoutValidation("Content-Disposition", request.Headers["Content-Disposition"]);
}
DateTime expires;
if (request.Headers.ContainsKey("Expires") &&
DateTime.TryParse(request.Headers["Expires"], CultureInfo.InvariantCulture, DateTimeStyles.None, out expires))
{
requestMessage.Content.Headers.Expires = expires;
}
}
}