private async Task <string> CompleteMultiUpload( string url, HttpClient client, long archiveSize, string payloadTreeHash) { var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Post, url) { Headers = { { "x-amz-glacier-version", "2012-06-01" }, { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-archive-size", archiveSize.ToString() }, { "x-amz-content-sha256", payloadHash }, { "x-amz-sha256-tree-hash", payloadTreeHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } return(ReadArchiveId(response)); }
public async Task DeleteArchive(string archiveId) { var url = $"{GetUrl()}/archives/{archiveId}"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var content = new HttpRequestMessage(HttpMethods.Delete, url) { Headers = { { "x-amz-glacier-version", "2012-06-01" }, { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(content.Headers); var client = GetClient(); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.SendAsync(content, CancellationToken); if (response.IsSuccessStatusCode) { return; } throw StorageException.FromResponseMessage(response); }
private async Task <string> GetUploadId(string url, string archiveDescription, long lengthPerPartPowerOf2) { var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Post, url) { Headers = { { "x-amz-glacier-version", "2012-06-01" }, { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-archive-description", archiveDescription }, { "x-amz-part-size", lengthPerPartPowerOf2.ToString() }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } return(response.Headers .GetValues("x-amz-multipart-upload-id") .First()); }
private async Task AbortMultiUpload(HttpClient client, string url) { var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Delete, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode) { return; } // The specified multipart upload does not exist. // The upload ID might be invalid, // or the multipart upload might have been aborted or completed. if (response.StatusCode == HttpStatusCode.NotFound) { return; } throw StorageException.FromResponseMessage(response); }
private void CompleteMultiUpload(string url, HttpClient client, List <Tuple <int, string> > partNumbersWithEtag) { var now = SystemTime.UtcNow; var doc = CreateCompleteMultiUploadDocument(partNumbersWithEtag); var xmlString = doc.OuterXml; var requestMessage = new HttpRequestMessage(HttpMethods.Post, url) { Content = new StringContent(xmlString, Encoding.UTF8, "text/plain") }; UpdateHeaders(requestMessage.Headers, now, stream: null, RavenAwsHelper.CalculatePayloadHashFromString(xmlString)); var headers = ConvertToHeaders(requestMessage.Headers); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = client.SendAsync(requestMessage, CancellationToken).Result; if (response.IsSuccessStatusCode) { return; } throw StorageException.FromResponseMessage(response); }
protected virtual void UpdateHeaders(HttpHeaders headers, DateTime now, Stream stream, string payloadHash = null) { headers.Add("x-amz-date", RavenAwsHelper.ConvertToString(now)); headers.Add("x-amz-content-sha256", payloadHash ?? RavenAwsHelper.CalculatePayloadHash(stream)); if (_sessionToken != null) { headers.Add("x-amz-security-token", _sessionToken); } }
public async Task DeleteMultipleObjects(List <string> objects) { var url = $"{GetUrl()}/?delete"; var now = SystemTime.UtcNow; var xml = new XElement("Delete"); foreach (var objectPath in objects) { var obj = new XElement("Object"); var key = new XElement("Key", objectPath); obj.Add(key); xml.Add(obj); } var xmlString = xml.ToString(); var md5Hash = CalculateMD5Hash(xmlString); var requestMessage = new HttpRequestMessage(HttpMethods.Post, url) { Content = new StringContent(xmlString, Encoding.UTF8, "text/plain") { Headers = { { "Content-Length", xmlString.Length.ToString(CultureInfo.InvariantCulture) }, { "Content-MD5", md5Hash } } } }; UpdateHeaders(requestMessage.Headers, now, null, RavenAwsHelper.CalculatePayloadHashFromString(xmlString)); var headers = ConvertToHeaders(requestMessage.Headers); headers.Add("Content-MD5", md5Hash); var client = GetClient(TimeSpan.FromHours(1)); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.SendAsync(requestMessage); if (response.IsSuccessStatusCode) { return; } if (response.StatusCode == HttpStatusCode.NotFound) { return; } throw StorageException.FromResponseMessage(response); }
private async Task <string> MultiPartUpload(string archiveDescription, Stream stream) { var streamLength = stream.Length; if (streamLength > MultiPartUploadLimitInBytes) { throw new InvalidOperationException(@"Can't upload more than 40TB to Amazon Glacier, " + $"current upload size: {new Size(streamLength).HumaneSize}"); } UploadProgress?.SetTotal(streamLength); UploadProgress?.ChangeType(UploadType.Chunked); // using a chunked upload we can upload up to 10,000 chunks, 4GB max each // we limit every chunk to a minimum of 128MB // constraints: the part size must be a megabyte(1024KB) // multiplied by a power of 2—for example, // 1048576(1MB), 2097152(2MB), 4194304(4MB), 8388608(8 MB), and so on. // the minimum allowable part size is 1MB and the maximum is 4GB(4096 MB). var maxLengthPerPart = Math.Max(MinOnePartUploadSizeLimitInBytes, stream.Length / 10000); const long maxPartLength = 4L * 1024 * 1024 * 1024; // 4GB var lengthPerPartPowerOf2 = Math.Min(GetNextPowerOf2(maxLengthPerPart), maxPartLength); var baseUrl = $"{GetUrl()}/multipart-uploads"; var uploadId = await GetUploadId(baseUrl, archiveDescription, lengthPerPartPowerOf2); var client = GetClient(TimeSpan.FromDays(7)); var uploadUrl = $"{baseUrl}/{uploadId}"; var fullStreamPayloadTreeHash = RavenAwsHelper.CalculatePayloadTreeHash(stream); try { while (stream.Position < streamLength) { var length = Math.Min(lengthPerPartPowerOf2, streamLength - stream.Position); await UploadPart(stream, client, uploadUrl, length, retryCount : 0); } return(await CompleteMultiUpload(uploadUrl, client, streamLength, fullStreamPayloadTreeHash)); } catch (Exception) { await AbortMultiUpload(uploadUrl, client); throw; } finally { UploadProgress?.ChangeState(UploadState.Done); } }
private async Task <string> GetBucketLocation(bool returnWhenNotFound = false) { using (UseRegionInvariantRequest()) { var url = $"{GetUrl()}?location"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Get, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.StatusCode == HttpStatusCode.NotFound) { if (returnWhenNotFound) { return(null); } throw new BucketNotFoundException($"Bucket name '{_bucketName}' doesn't exist!"); } if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } using (var stream = await response.Content.ReadAsStreamAsync()) using (var reader = new StreamReader(stream)) { var xElement = XElement.Load(reader); var value = xElement.Value; // when the bucket's region is US East (N. Virginia - us-east-1), // Amazon S3 returns an empty string for the bucket's region return(value == string.Empty ? DefaultRegion : value); } } }
public async Task PutObject(string key, Stream stream, Dictionary <string, string> metadata) { await TestConnection(); if (stream.Length > MaxUploadPutObjectSizeInBytes) { // for objects over 256MB await MultiPartUpload(key, stream, metadata); return; } var url = $"{GetUrl()}/{key}"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream); Progress?.UploadProgress.SetTotal(stream.Length); // stream is disposed by the HttpClient var content = new ProgressableStreamContent(stream, Progress) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; foreach (var metadataKey in metadata.Keys) { content.Headers.Add("x-amz-meta-" + metadataKey.ToLower(), metadata[metadataKey]); } var headers = ConvertToHeaders(content.Headers); var client = GetClient(TimeSpan.FromHours(24)); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.PutAsync(url, content, CancellationToken); Progress?.UploadProgress.ChangeState(UploadState.Done); if (response.IsSuccessStatusCode) { return; } throw StorageException.FromResponseMessage(response); }
public async Task <string> UploadArchive(Stream stream, string archiveDescription) { await TestConnection(); if (stream.Length > MaxUploadArchiveSizeInBytes) { // for objects over 256MB return(await MultiPartUpload(archiveDescription, stream)); } var url = $"{GetUrl()}/archives"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(stream); var payloadTreeHash = RavenAwsHelper.CalculatePayloadTreeHash(stream); UploadProgress?.SetTotal(stream.Length); // stream is disposed by the HttpClient var content = new ProgressableStreamContent(stream, UploadProgress) { Headers = { { "x-amz-glacier-version", "2012-06-01" }, { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash }, { "x-amz-sha256-tree-hash", payloadTreeHash }, { "x-amz-archive-description", archiveDescription }, { "Content-Length", stream.Length.ToString(CultureInfo.InvariantCulture) } } }; var headers = ConvertToHeaders(content.Headers); var client = GetClient(TimeSpan.FromHours(24)); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.PostAsync(url, content, CancellationToken); UploadProgress?.ChangeState(UploadState.Done); if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } return(ReadArchiveId(response)); }
public async Task PutBucket(string awsRegion = null) { // we set the bucket region in the request message using (UseRegionInvariantRequest()) { var doc = CreatePutBucketXmlDocument(awsRegion ?? AwsRegion); var xmlString = doc.OuterXml; var url = GetUrl(); var now = SystemTime.UtcNow; var hasLocationConstraint = AwsRegion != DefaultRegion; var payloadHash = hasLocationConstraint ? RavenAwsHelper.CalculatePayloadHashFromString(xmlString) : RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Put, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } }, Content = hasLocationConstraint == false ? null : new StringContent(xmlString, Encoding.UTF8, "text/plain") }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(TimeSpan.FromHours(1)); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode) { return; } if (response.StatusCode == HttpStatusCode.NotFound) { return; } throw StorageException.FromResponseMessage(response); } }
public async Task DeleteBucket() { var region = await GetBucketLocation(returnWhenNotFound : true); if (region == null) { return; } using (UseSpecificRegion(region)) { var url = GetUrl(); var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Delete, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode) { return; } if (response.StatusCode == HttpStatusCode.NotFound) { return; } throw StorageException.FromResponseMessage(response); } }
public AuthenticationHeaderValue CalculateAuthorizationHeaderValue(HttpMethod httpMethod, string url, DateTime date, IDictionary <string, string> httpHeaders) { var canonicalRequestHash = CalculateCanonicalRequestHash(httpMethod, url, httpHeaders, out string signedHeaders); var signingKey = CalculateSigningKey(date, ServiceName); using (var hash = new HMACSHA256(signingKey)) { var regionForRequest = IsRegionInvariantRequest ? DefaultRegion : AwsRegion; var scope = $"{date:yyyyMMdd}/{regionForRequest}/{ServiceName}/aws4_request"; var stringToHash = $"AWS4-HMAC-SHA256\n{RavenAwsHelper.ConvertToString(date)}\n{scope}\n{canonicalRequestHash}"; var hashedString = hash.ComputeHash(Encoding.UTF8.GetBytes(stringToHash)); var signature = RavenAwsHelper.ConvertToHex(hashedString); var credentials = $"{_awsAccessKey}/{date:yyyyMMdd}/{regionForRequest}/{ServiceName}/aws4_request"; return(new AuthenticationHeaderValue("AWS4-HMAC-SHA256", $"Credential={credentials},SignedHeaders={signedHeaders},Signature={signature}")); } }
private async Task <string> GetBucketPermission() { var url = $"{GetUrl()}?acl"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Get, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.StatusCode == HttpStatusCode.NotFound) { throw new BucketNotFoundException($"Bucket name '{_bucketName}' doesn't exist!"); } if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } using (var stream = await response.Content.ReadAsStreamAsync()) using (var reader = new StreamReader(stream)) { var xDocument = XDocument.Load(reader); var permissionElement = xDocument .Descendants() .First(x => x.Name.LocalName == "Permission"); return(permissionElement.Value); } }
private async Task <string> GetUploadId(string baseUrl, Dictionary <string, string> metadata) { var url = $"{baseUrl}?uploads"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Post, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; foreach (var metadataKey in metadata.Keys) { requestMessage.Headers.Add("x-amz-meta-" + metadataKey.ToLower(), metadata[metadataKey]); } var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } using (var stream = await response.Content.ReadAsStreamAsync()) using (var reader = new StreamReader(stream)) { var xDocument = XDocument.Load(reader); var permissionElement = xDocument .Descendants() .First(x => x.Name.LocalName == "UploadId"); return(permissionElement.Value); } }
private static string CalculateCanonicalRequestHash(HttpMethod httpMethod, string url, IDictionary <string, string> httpHeaders, out string signedHeaders) { var isGet = httpMethod == HttpMethods.Get; var uri = new Uri(url); var canonicalUri = uri.AbsolutePath; var query = QueryHelpers.ParseQuery(uri.Query); var canonicalQueryString = query .OrderBy(parameter => parameter.Key) .Select(parameter => parameter.Value.Aggregate(string.Empty, (current, value) => current + $"{parameter.Key}={value.Trim()}&")) .Aggregate(string.Empty, (current, parameter) => current + parameter); if (canonicalQueryString.EndsWith("&")) { canonicalQueryString = canonicalQueryString.Substring(0, canonicalQueryString.Length - 1); } var headers = httpHeaders .Where(x => isGet == false || x.Key.StartsWith("Date", StringComparison.OrdinalIgnoreCase) == false) .OrderBy(x => x.Key); var canonicalHeaders = headers .Aggregate(string.Empty, (current, parameter) => current + $"{parameter.Key.ToLower()}:{parameter.Value.Trim()}\n"); signedHeaders = headers .Aggregate(string.Empty, (current, parameter) => current + parameter.Key.ToLower() + ";"); if (signedHeaders.EndsWith(";")) { signedHeaders = signedHeaders.Substring(0, signedHeaders.Length - 1); } using (var hash = SHA256.Create()) { var hashedPayload = httpHeaders["x-amz-content-sha256"]; var canonicalRequest = $"{httpMethod}\n{canonicalUri}\n{canonicalQueryString}\n{canonicalHeaders}\n{signedHeaders}\n{hashedPayload}"; return(RavenAwsHelper.ConvertToHex(hash.ComputeHash(Encoding.UTF8.GetBytes(canonicalRequest)))); } }
public async Task <Blob> GetObject(string key) { var url = $"{GetUrl()}/{key}"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Get, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers); var response = await client.SendAsync(requestMessage, CancellationToken); if (response.StatusCode == HttpStatusCode.NotFound) { return(null); } if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } var data = await response.Content.ReadAsStreamAsync(); var metadataHeaders = response.Headers.ToDictionary(x => x.Key, x => x.Value.FirstOrDefault()); return(new Blob(data, metadataHeaders)); }
private async Task <bool> VaultExists() { var url = GetUrl(); var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var content = new HttpRequestMessage(HttpMethods.Get, url) { Headers = { { "x-amz-glacier-version", "2012-06-01" }, { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(content.Headers); var client = GetClient(); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Get, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.SendAsync(content, CancellationToken); if (response.IsSuccessStatusCode) { return(true); } if (response.StatusCode == HttpStatusCode.NotFound) { return(false); } await response.Content.ReadAsStringAsync(); throw StorageException.FromResponseMessage(response); }
public async Task DeleteObject(string key) { var url = $"{GetUrl()}/{key}"; var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(null); var requestMessage = new HttpRequestMessage(HttpMethods.Delete, url) { Headers = { { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash } } }; var headers = ConvertToHeaders(requestMessage.Headers); var client = GetClient(TimeSpan.FromHours(1)); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Delete, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = await client.SendAsync(requestMessage, CancellationToken); if (response.IsSuccessStatusCode) { return; } if (response.StatusCode == HttpStatusCode.NotFound) { return; } throw StorageException.FromResponseMessage(response); }
private void UploadPart(Stream baseStream, HttpClient client, string url, long length, int retryCount) { // saving the position if we need to retry var position = baseStream.Position; using (var subStream = new SubStream(baseStream, offset: 0, length: length)) { var now = SystemTime.UtcNow; // stream is disposed by the HttpClient var content = new ProgressableStreamContent(subStream, Progress) { Headers = { { "x-amz-sha256-tree-hash", RavenAwsHelper.CalculatePayloadTreeHash(subStream) }, { "Content-Range", $"bytes {position}-{position+length-1}/*" }, { "Content-Length", subStream.Length.ToString(CultureInfo.InvariantCulture) } } }; UpdateHeaders(content.Headers, now, subStream); var headers = ConvertToHeaders(content.Headers); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers); try { var response = client.PutAsync(url, content, CancellationToken).Result; if (response.IsSuccessStatusCode) { return; } if (retryCount == MaxRetriesForMultiPartUpload) { throw StorageException.FromResponseMessage(response); } } catch (Exception) { if (retryCount == MaxRetriesForMultiPartUpload) { throw; } } // revert the uploaded count before retry Progress?.UploadProgress.UpdateUploaded(-content.Uploaded); } // wait for one second before trying again to send the request // maybe there was a network issue? CancellationToken.WaitHandle.WaitOne(1000); CancellationToken.ThrowIfCancellationRequested(); retryCount++; if (_logger?.IsInfoEnabled == true) { _logger.Info($"Trying to send the request again. Retries count: '{retryCount}', VaultName: '{_vaultName}'."); } // restore the stream position before retrying baseStream.Position = position; UploadPart(baseStream, client, url, length, retryCount); }
public void DeleteMultipleObjects(List <string> objects) { if (objects.Count == 0) { return; } var url = $"{GetUrl()}/?delete"; var now = SystemTime.UtcNow; var xml = new XElement("Delete"); foreach (var objectPath in objects) { var obj = new XElement("Object"); var key = new XElement("Key", objectPath); obj.Add(key); xml.Add(obj); } var xmlString = xml.ToString(); var md5Hash = CalculateMD5Hash(xmlString); var requestMessage = new HttpRequestMessage(HttpMethods.Post, url) { Content = new StringContent(xmlString, Encoding.UTF8, "text/plain") { Headers = { { "Content-Length", xmlString.Length.ToString(CultureInfo.InvariantCulture) }, { "Content-MD5", md5Hash } } } }; UpdateHeaders(requestMessage.Headers, now, null, RavenAwsHelper.CalculatePayloadHashFromString(xmlString)); var headers = ConvertToHeaders(requestMessage.Headers); headers.Add("Content-MD5", md5Hash); var client = GetClient(TimeSpan.FromHours(1)); var authorizationHeaderValue = CalculateAuthorizationHeaderValue(HttpMethods.Post, url, now, headers); client.DefaultRequestHeaders.Authorization = authorizationHeaderValue; var response = client.SendAsync(requestMessage).Result; if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } using (var stream = response.Content.ReadAsStreamAsync().Result) using (var reader = new StreamReader(stream)) { var xElement = XElement.Load(reader); var ns = xElement.Name.Namespace; var errors = xElement.Elements(ns + "Error").Select(x => new { Key = x.Element(ns + "Key").Value, Code = x.Element(ns + "Code").Value, Message = x.Element(ns + "Message").Value } ) .GroupBy(x => x.Code) .Select(x => new { Message = x.First().Message, Keys = x.Select(e => e.Key).ToList(), Count = x.Count() }) .ToList(); if (errors.Count == 0) { return; } var totalFailedToDelete = errors.Sum(x => x.Keys.Count); var totalDeleted = xElement.Elements(ns + "Deleted").Count(); var message = $"Failed to delete {totalFailedToDelete} file{Pluralize(totalFailedToDelete)}"; if (_logger != null && _logger.IsInfoEnabled) { var failedToDeleteReasons = errors.Aggregate(string.Empty, (current, error) => current + $"Reason: {error.Message}, Files ({error.Count}): {string.Join(", ", error.Keys)}. "); _logger.Info($"{message}. Successfully deleted {totalDeleted} file{Pluralize(totalDeleted)}. {failedToDeleteReasons}"); } string Pluralize(int num) { return(num == 0 || num > 1 ? "s" : string.Empty); } var reasons = errors.Select(x => x.Message); throw new InvalidOperationException(message + $" because of: {string.Join(", ", reasons)}"); } }
private async Task UploadPart(Stream baseStream, HttpClient client, string url, long length, int retryCount) { // saving the position if we need to retry var position = baseStream.Position; using (var subStream = new SubStream(baseStream, offset: 0, length: length)) { var now = SystemTime.UtcNow; var payloadHash = RavenAwsHelper.CalculatePayloadHash(subStream); var payloadTreeHash = RavenAwsHelper.CalculatePayloadTreeHash(subStream); // stream is disposed by the HttpClient var content = new ProgressableStreamContent(subStream, UploadProgress) { Headers = { { "x-amz-glacier-version", "2012-06-01" }, { "x-amz-date", RavenAwsHelper.ConvertToString(now) }, { "x-amz-content-sha256", payloadHash }, { "x-amz-sha256-tree-hash", payloadTreeHash }, { "Content-Range", $"bytes {position}-{position+length-1}/*" }, { "Content-Length", subStream.Length.ToString(CultureInfo.InvariantCulture) } } }; var headers = ConvertToHeaders(content.Headers); client.DefaultRequestHeaders.Authorization = CalculateAuthorizationHeaderValue(HttpMethods.Put, url, now, headers); try { var response = await client.PutAsync(url, content, CancellationToken); if (response.IsSuccessStatusCode) { return; } if (retryCount == MaxRetriesForMultiPartUpload) { throw StorageException.FromResponseMessage(response); } } catch (Exception) { if (retryCount == MaxRetriesForMultiPartUpload) { throw; } } // revert the uploaded count before retry UploadProgress?.UpdateUploaded(-content.Uploaded); } // wait for one second before trying again to send the request // maybe there was a network issue? await Task.Delay(1000); CancellationToken.ThrowIfCancellationRequested(); // restore the stream position before retrying baseStream.Position = position; await UploadPart(baseStream, client, url, length, ++retryCount); }