private async Task <string> CompleteMultiUpload(
string url, HttpClient client, long archiveSize,
string payloadTreeHash)
{
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Post, url)
{
Headers =
{
{ "x-amz-glacier-version", "2012-06-01" },
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-archive-size", archiveSize.ToString() },
{ "x-amz-content-sha256", payloadHash },
{ "x-amz-sha256-tree-hash", payloadTreeHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
return(ReadArchiveId(response));
}
public async Task DeleteArchive(string archiveId)
{
var url = $"{GetUrl()}/archives/{archiveId}";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var content = new HttpRequestMessage(HttpMethods.Delete, url)
{
Headers =
{
{ "x-amz-glacier-version", "2012-06-01" },
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(content.Headers);
var client = GetClient();
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.SendAsync(content, CancellationToken);
if (response.IsSuccessStatusCode)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
private async Task <string> GetUploadId(string url,
string archiveDescription, long lengthPerPartPowerOf2)
{
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Post, url)
{
Headers =
{
{ "x-amz-glacier-version", "2012-06-01" },
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-archive-description", archiveDescription },
{ "x-amz-part-size", lengthPerPartPowerOf2.ToString() },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient();
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
return(response.Headers
.GetValues("x-amz-multipart-upload-id")
.First());
}
private async Task AbortMultiUpload(HttpClient client, string url)
{
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Delete, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode)
{
return;
}
// The specified multipart upload does not exist.
// The upload ID might be invalid,
// or the multipart upload might have been aborted or completed.
if (response.StatusCode == HttpStatusCode.NotFound)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
private void CompleteMultiUpload(string url, HttpClient client,
List <Tuple <int, string> > partNumbersWithEtag)
{
var now = SystemTime.UtcNow;
var doc = CreateCompleteMultiUploadDocument(partNumbersWithEtag);
var xmlString = doc.OuterXml;
var requestMessage = new HttpRequestMessage(HttpMethods.Post, url)
{
Content = new StringContent(xmlString, Encoding.UTF8, "text/plain")
};
UpdateHeaders(requestMessage.Headers, now, stream: null, RavenAwsHelper.CalculatePayloadHashFromString(xmlString));
var headers = ConvertToHeaders(requestMessage.Headers);
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = client.SendAsync(requestMessage, CancellationToken).Result;
if (response.IsSuccessStatusCode)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
protected virtual void UpdateHeaders(HttpHeaders headers, DateTime now, Stream stream, string payloadHash = null)
{
headers.Add("x-amz-date", RavenAwsHelper.ConvertToString(now));
headers.Add("x-amz-content-sha256", payloadHash ?? RavenAwsHelper.CalculatePayloadHash(stream));
if (_sessionToken != null)
{
headers.Add("x-amz-security-token", _sessionToken);
}
}
public async Task DeleteMultipleObjects(List <string> objects)
{
var url = $"{GetUrl()}/?delete";
var now = SystemTime.UtcNow;
var xml = new XElement("Delete");
foreach (var objectPath in objects)
{
var obj = new XElement("Object");
var key = new XElement("Key", objectPath);
obj.Add(key);
xml.Add(obj);
}
var xmlString = xml.ToString();
var md5Hash = CalculateMD5Hash(xmlString);
var requestMessage = new HttpRequestMessage(HttpMethods.Post, url)
{
Content = new StringContent(xmlString, Encoding.UTF8, "text/plain")
{
Headers =
{
{ "Content-Length", xmlString.Length.ToString(CultureInfo.InvariantCulture) },
{ "Content-MD5", md5Hash }
}
}
};
UpdateHeaders(requestMessage.Headers, now, null, RavenAwsHelper.CalculatePayloadHashFromString(xmlString));
var headers = ConvertToHeaders(requestMessage.Headers);
headers.Add("Content-MD5", md5Hash);
var client = GetClient(TimeSpan.FromHours(1));
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.SendAsync(requestMessage);
if (response.IsSuccessStatusCode)
{
return;
}
if (response.StatusCode == HttpStatusCode.NotFound)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
private async Task <string> MultiPartUpload(string archiveDescription, Stream stream)
{
var streamLength = stream.Length;
if (streamLength > MultiPartUploadLimitInBytes)
{
throw new InvalidOperationException(@"Can't upload more than 40TB to Amazon Glacier, " +
$"current upload size: {new Size(streamLength).HumaneSize}");
}
UploadProgress?.SetTotal(streamLength);
UploadProgress?.ChangeType(UploadType.Chunked);
// using a chunked upload we can upload up to 10,000 chunks, 4GB max each
// we limit every chunk to a minimum of 128MB
// constraints: the part size must be a megabyte(1024KB)
// multiplied by a power of 2—for example,
// 1048576(1MB), 2097152(2MB), 4194304(4MB), 8388608(8 MB), and so on.
// the minimum allowable part size is 1MB and the maximum is 4GB(4096 MB).
var maxLengthPerPart = Math.Max(MinOnePartUploadSizeLimitInBytes, stream.Length / 10000);
const long maxPartLength = 4L * 1024 * 1024 * 1024; // 4GB
var lengthPerPartPowerOf2 = Math.Min(GetNextPowerOf2(maxLengthPerPart), maxPartLength);
var baseUrl = $"{GetUrl()}/multipart-uploads";
var uploadId = await GetUploadId(baseUrl, archiveDescription, lengthPerPartPowerOf2);
var client = GetClient(TimeSpan.FromDays(7));
var uploadUrl = $"{baseUrl}/{uploadId}";
var fullStreamPayloadTreeHash = RavenAwsHelper.CalculatePayloadTreeHash(stream);
try
{
while (stream.Position < streamLength)
{
var length = Math.Min(lengthPerPartPowerOf2, streamLength - stream.Position);
await UploadPart(stream, client, uploadUrl, length, retryCount : 0);
}
return(await CompleteMultiUpload(uploadUrl, client, streamLength, fullStreamPayloadTreeHash));
}
catch (Exception)
{
await AbortMultiUpload(uploadUrl, client);
throw;
}
finally
{
UploadProgress?.ChangeState(UploadState.Done);
}
}
private async Task <string> GetBucketLocation(bool returnWhenNotFound = false)
{
using (UseRegionInvariantRequest())
{
var url = $"{GetUrl()}?location";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Get, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient();
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.StatusCode == HttpStatusCode.NotFound)
{
if (returnWhenNotFound)
{
return(null);
}
throw new BucketNotFoundException($"Bucket name '{_bucketName}' doesn't exist!");
}
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
using (var stream = await response.Content.ReadAsStreamAsync())
using (var reader = new StreamReader(stream))
{
var xElement = XElement.Load(reader);
var value = xElement.Value;
// when the bucket's region is US East (N. Virginia - us-east-1),
// Amazon S3 returns an empty string for the bucket's region
return(value == string.Empty ? DefaultRegion : value);
}
}
}
public async Task PutObject(string key, Stream stream, Dictionary <string, string> metadata)
{
await TestConnection();
if (stream.Length > MaxUploadPutObjectSizeInBytes)
{
// for objects over 256MB
await MultiPartUpload(key, stream, metadata);
return;
}
var url = $"{GetUrl()}/{key}";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream);
Progress?.UploadProgress.SetTotal(stream.Length);
// stream is disposed by the HttpClient
var content = new ProgressableStreamContent(stream, Progress)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
foreach (var metadataKey in metadata.Keys)
{
content.Headers.Add("x-amz-meta-" + metadataKey.ToLower(), metadata[metadataKey]);
}
var headers = ConvertToHeaders(content.Headers);
var client = GetClient(TimeSpan.FromHours(24));
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.PutAsync(url, content, CancellationToken);
Progress?.UploadProgress.ChangeState(UploadState.Done);
if (response.IsSuccessStatusCode)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
public async Task <string> UploadArchive(Stream stream, string archiveDescription)
{
await TestConnection();
if (stream.Length > MaxUploadArchiveSizeInBytes)
{
// for objects over 256MB
return(await MultiPartUpload(archiveDescription, stream));
}
var url = $"{GetUrl()}/archives";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream);
var payloadTreeHash = RavenAwsHelper.CalculatePayloadTreeHash(stream);
UploadProgress?.SetTotal(stream.Length);
// stream is disposed by the HttpClient
var content = new ProgressableStreamContent(stream, UploadProgress)
{
Headers =
{
{ "x-amz-glacier-version", "2012-06-01" },
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash },
{ "x-amz-sha256-tree-hash", payloadTreeHash },
{ "x-amz-archive-description", archiveDescription },
{ "Content-Length", stream.Length.ToString(CultureInfo.InvariantCulture) }
}
};
var headers = ConvertToHeaders(content.Headers);
var client = GetClient(TimeSpan.FromHours(24));
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.PostAsync(url, content, CancellationToken);
UploadProgress?.ChangeState(UploadState.Done);
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
return(ReadArchiveId(response));
}
public async Task PutBucket(string awsRegion = null)
{
// we set the bucket region in the request message
using (UseRegionInvariantRequest())
{
var doc = CreatePutBucketXmlDocument(awsRegion ?? AwsRegion);
var xmlString = doc.OuterXml;
var url = GetUrl();
var now = SystemTime.UtcNow;
var hasLocationConstraint = AwsRegion != DefaultRegion;
var payloadHash = hasLocationConstraint ?
RavenAwsHelper.CalculatePayloadHashFromString(xmlString) :
RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Put, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
},
Content = hasLocationConstraint == false ?
null : new StringContent(xmlString, Encoding.UTF8, "text/plain")
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient(TimeSpan.FromHours(1));
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode)
{
return;
}
if (response.StatusCode == HttpStatusCode.NotFound)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
}
public async Task DeleteBucket()
{
var region = await GetBucketLocation(returnWhenNotFound : true);
if (region == null)
{
return;
}
using (UseSpecificRegion(region))
{
var url = GetUrl();
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Delete, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient();
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode)
{
return;
}
if (response.StatusCode == HttpStatusCode.NotFound)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
}
public AuthenticationHeaderValue CalculateAuthorizationHeaderValue(HttpMethod httpMethod, string url, DateTime date, IDictionary <string, string> httpHeaders)
{
var canonicalRequestHash = CalculateCanonicalRequestHash(httpMethod, url, httpHeaders, out string signedHeaders);
var signingKey = CalculateSigningKey(date, ServiceName);
using (var hash = new HMACSHA256(signingKey))
{
var regionForRequest = IsRegionInvariantRequest ? DefaultRegion : AwsRegion;
var scope = $"{date:yyyyMMdd}/{regionForRequest}/{ServiceName}/aws4_request";
var stringToHash = $"AWS4-HMAC-SHA256\n{RavenAwsHelper.ConvertToString(date)}\n{scope}\n{canonicalRequestHash}";
var hashedString = hash.ComputeHash(Encoding.UTF8.GetBytes(stringToHash));
var signature = RavenAwsHelper.ConvertToHex(hashedString);
var credentials = $"{_awsAccessKey}/{date:yyyyMMdd}/{regionForRequest}/{ServiceName}/aws4_request";
return(new AuthenticationHeaderValue("AWS4-HMAC-SHA256", $"Credential={credentials},SignedHeaders={signedHeaders},Signature={signature}"));
}
}
private async Task <string> GetBucketPermission()
{
var url = $"{GetUrl()}?acl";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Get, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient();
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.StatusCode == HttpStatusCode.NotFound)
{
throw new BucketNotFoundException($"Bucket name '{_bucketName}' doesn't exist!");
}
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
using (var stream = await response.Content.ReadAsStreamAsync())
using (var reader = new StreamReader(stream))
{
var xDocument = XDocument.Load(reader);
var permissionElement = xDocument
.Descendants()
.First(x => x.Name.LocalName == "Permission");
return(permissionElement.Value);
}
}
private async Task <string> GetUploadId(string baseUrl, Dictionary <string, string> metadata)
{
var url = $"{baseUrl}?uploads";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Post, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
foreach (var metadataKey in metadata.Keys)
{
requestMessage.Headers.Add("x-amz-meta-" + metadataKey.ToLower(), metadata[metadataKey]);
}
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient();
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
using (var stream = await response.Content.ReadAsStreamAsync())
using (var reader = new StreamReader(stream))
{
var xDocument = XDocument.Load(reader);
var permissionElement = xDocument
.Descendants()
.First(x => x.Name.LocalName == "UploadId");
return(permissionElement.Value);
}
}
private static string CalculateCanonicalRequestHash(HttpMethod httpMethod, string url, IDictionary <string, string> httpHeaders, out string signedHeaders)
{
var isGet = httpMethod == HttpMethods.Get;
var uri = new Uri(url);
var canonicalUri = uri.AbsolutePath;
var query = QueryHelpers.ParseQuery(uri.Query);
var canonicalQueryString = query
.OrderBy(parameter => parameter.Key)
.Select(parameter => parameter.Value.Aggregate(string.Empty, (current, value) =>
current + $"{parameter.Key}={value.Trim()}&"))
.Aggregate(string.Empty, (current, parameter) => current + parameter);
if (canonicalQueryString.EndsWith("&"))
{
canonicalQueryString = canonicalQueryString.Substring(0, canonicalQueryString.Length - 1);
}
var headers = httpHeaders
.Where(x => isGet == false || x.Key.StartsWith("Date", StringComparison.OrdinalIgnoreCase) == false)
.OrderBy(x => x.Key);
var canonicalHeaders = headers
.Aggregate(string.Empty, (current, parameter) => current + $"{parameter.Key.ToLower()}:{parameter.Value.Trim()}\n");
signedHeaders = headers
.Aggregate(string.Empty, (current, parameter) => current + parameter.Key.ToLower() + ";");
if (signedHeaders.EndsWith(";"))
{
signedHeaders = signedHeaders.Substring(0, signedHeaders.Length - 1);
}
using (var hash = SHA256.Create())
{
var hashedPayload = httpHeaders["x-amz-content-sha256"];
var canonicalRequest = $"{httpMethod}\n{canonicalUri}\n{canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\n{hashedPayload}";
return(RavenAwsHelper.ConvertToHex(hash.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest))));
}
}
public async Task <Blob> GetObject(string key)
{
var url = $"{GetUrl()}/{key}";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Get, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient();
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers);
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
var data = await response.Content.ReadAsStreamAsync();
var metadataHeaders = response.Headers.ToDictionary(x => x.Key, x => x.Value.FirstOrDefault());
return(new Blob(data, metadataHeaders));
}
private async Task <bool> VaultExists()
{
var url = GetUrl();
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var content = new HttpRequestMessage(HttpMethods.Get, url)
{
Headers =
{
{ "x-amz-glacier-version", "2012-06-01" },
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(content.Headers);
var client = GetClient();
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.SendAsync(content, CancellationToken);
if (response.IsSuccessStatusCode)
{
return(true);
}
if (response.StatusCode == HttpStatusCode.NotFound)
{
return(false);
}
await response.Content.ReadAsStringAsync();
throw StorageException.FromResponseMessage(response);
}
public async Task DeleteObject(string key)
{
var url = $"{GetUrl()}/{key}";
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(null);
var requestMessage = new HttpRequestMessage(HttpMethods.Delete, url)
{
Headers =
{
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash }
}
};
var headers = ConvertToHeaders(requestMessage.Headers);
var client = GetClient(TimeSpan.FromHours(1));
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = await client.SendAsync(requestMessage, CancellationToken);
if (response.IsSuccessStatusCode)
{
return;
}
if (response.StatusCode == HttpStatusCode.NotFound)
{
return;
}
throw StorageException.FromResponseMessage(response);
}
private void UploadPart(Stream baseStream, HttpClient client, string url, long length, int retryCount)
{
// saving the position if we need to retry
var position = baseStream.Position;
using (var subStream = new SubStream(baseStream, offset: 0, length: length))
{
var now = SystemTime.UtcNow;
// stream is disposed by the HttpClient
var content = new ProgressableStreamContent(subStream, Progress)
{
Headers =
{
{ "x-amz-sha256-tree-hash", RavenAwsHelper.CalculatePayloadTreeHash(subStream) },
{ "Content-Range", $"bytes {position}-{position+length-1}/*" },
{ "Content-Length", subStream.Length.ToString(CultureInfo.InvariantCulture) }
}
};
UpdateHeaders(content.Headers, now, subStream);
var headers = ConvertToHeaders(content.Headers);
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers);
try
{
var response = client.PutAsync(url, content, CancellationToken).Result;
if (response.IsSuccessStatusCode)
{
return;
}
if (retryCount == MaxRetriesForMultiPartUpload)
{
throw StorageException.FromResponseMessage(response);
}
}
catch (Exception)
{
if (retryCount == MaxRetriesForMultiPartUpload)
{
throw;
}
}
// revert the uploaded count before retry
Progress?.UploadProgress.UpdateUploaded(-content.Uploaded);
}
// wait for one second before trying again to send the request
// maybe there was a network issue?
CancellationToken.WaitHandle.WaitOne(1000);
CancellationToken.ThrowIfCancellationRequested();
retryCount++;
if (_logger?.IsInfoEnabled == true)
{
_logger.Info($"Trying to send the request again. Retries count: '{retryCount}', VaultName: '{_vaultName}'.");
}
// restore the stream position before retrying
baseStream.Position = position;
UploadPart(baseStream, client, url, length, retryCount);
}
public void DeleteMultipleObjects(List <string> objects)
{
if (objects.Count == 0)
{
return;
}
var url = $"{GetUrl()}/?delete";
var now = SystemTime.UtcNow;
var xml = new XElement("Delete");
foreach (var objectPath in objects)
{
var obj = new XElement("Object");
var key = new XElement("Key", objectPath);
obj.Add(key);
xml.Add(obj);
}
var xmlString = xml.ToString();
var md5Hash = CalculateMD5Hash(xmlString);
var requestMessage = new HttpRequestMessage(HttpMethods.Post, url)
{
Content = new StringContent(xmlString, Encoding.UTF8, "text/plain")
{
Headers =
{
{ "Content-Length", xmlString.Length.ToString(CultureInfo.InvariantCulture) },
{ "Content-MD5", md5Hash }
}
}
};
UpdateHeaders(requestMessage.Headers, now, null, RavenAwsHelper.CalculatePayloadHashFromString(xmlString));
var headers = ConvertToHeaders(requestMessage.Headers);
headers.Add("Content-MD5", md5Hash);
var client = GetClient(TimeSpan.FromHours(1));
var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers);
client.DefaultRequestHeaders.Authorization = authorizationHeaderValue;
var response = client.SendAsync(requestMessage).Result;
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
using (var stream = response.Content.ReadAsStreamAsync().Result)
using (var reader = new StreamReader(stream))
{
var xElement = XElement.Load(reader);
var ns = xElement.Name.Namespace;
var errors = xElement.Elements(ns + "Error").Select(x =>
new
{
Key = x.Element(ns + "Key").Value,
Code = x.Element(ns + "Code").Value,
Message = x.Element(ns + "Message").Value
}
)
.GroupBy(x => x.Code)
.Select(x => new
{
Message = x.First().Message,
Keys = x.Select(e => e.Key).ToList(),
Count = x.Count()
})
.ToList();
if (errors.Count == 0)
{
return;
}
var totalFailedToDelete = errors.Sum(x => x.Keys.Count);
var totalDeleted = xElement.Elements(ns + "Deleted").Count();
var message = $"Failed to delete {totalFailedToDelete} file{Pluralize(totalFailedToDelete)}";
if (_logger != null && _logger.IsInfoEnabled)
{
var failedToDeleteReasons = errors.Aggregate(string.Empty, (current, error) =>
current + $"Reason: {error.Message}, Files ({error.Count}): {string.Join(", ", error.Keys)}. ");
_logger.Info($"{message}. Successfully deleted {totalDeleted} file{Pluralize(totalDeleted)}. {failedToDeleteReasons}");
}
string Pluralize(int num)
{
return(num == 0 || num > 1 ? "s" : string.Empty);
}
var reasons = errors.Select(x => x.Message);
throw new InvalidOperationException(message + $" because of: {string.Join(", ", reasons)}");
}
}
private async Task UploadPart(Stream baseStream, HttpClient client, string url, long length, int retryCount)
{
// saving the position if we need to retry
var position = baseStream.Position;
using (var subStream = new SubStream(baseStream, offset: 0, length: length))
{
var now = SystemTime.UtcNow;
var payloadHash = RavenAwsHelper.CalculatePayloadHash(subStream);
var payloadTreeHash = RavenAwsHelper.CalculatePayloadTreeHash(subStream);
// stream is disposed by the HttpClient
var content = new ProgressableStreamContent(subStream, UploadProgress)
{
Headers =
{
{ "x-amz-glacier-version", "2012-06-01" },
{ "x-amz-date", RavenAwsHelper.ConvertToString(now) },
{ "x-amz-content-sha256", payloadHash },
{ "x-amz-sha256-tree-hash", payloadTreeHash },
{ "Content-Range", $"bytes {position}-{position+length-1}/*" },
{ "Content-Length", subStream.Length.ToString(CultureInfo.InvariantCulture) }
}
};
var headers = ConvertToHeaders(content.Headers);
client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers);
try
{
var response = await client.PutAsync(url, content, CancellationToken);
if (response.IsSuccessStatusCode)
{
return;
}
if (retryCount == MaxRetriesForMultiPartUpload)
{
throw StorageException.FromResponseMessage(response);
}
}
catch (Exception)
{
if (retryCount == MaxRetriesForMultiPartUpload)
{
throw;
}
}
// revert the uploaded count before retry
UploadProgress?.UpdateUploaded(-content.Uploaded);
}
// wait for one second before trying again to send the request
// maybe there was a network issue?
await Task.Delay(1000);
CancellationToken.ThrowIfCancellationRequested();
// restore the stream position before retrying
baseStream.Position = position;
await UploadPart(baseStream, client, url, length, ++retryCount);
}
