protected override async Task <StorageContent> OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken)
{
// the Azure SDK will treat a starting / as an absolute URL,
// while we may be working in a subdirectory of a storage container
// trim the starting slash to treat it as a relative path
string name = GetName(resourceUri).TrimStart('/');
CloudBlockBlob blob = GetBlockBlobReference(name);
await _throttle.WaitAsync();
try
{
string content;
using (var originalStream = new MemoryStream())
{
await blob.DownloadToStreamAsync(originalStream, cancellationToken);
originalStream.Seek(0, SeekOrigin.Begin);
if (blob.Properties.ContentEncoding == "gzip")
{
using (var uncompressedStream = new GZipStream(originalStream, CompressionMode.Decompress))
{
using (var reader = new StreamReader(uncompressedStream))
{
content = await reader.ReadToEndAsync();
}
}
}
else
{
using (var reader = new StreamReader(originalStream))
{
content = await reader.ReadToEndAsync();
}
}
}
return(new StringStorageContentWithETag(content, blob.Properties.ETag));
}
catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
if (Verbose)
{
Trace.WriteLine(string.Format("Can't load '{0}'. Blob doesn't exist", resourceUri));
}
return(null);
}
finally
{
_throttle.Release();
}
}
private async Task <T> ReadAsync <T>(
HiveType hive,
string path,
string typeName,
bool allow404)
{
var blob = GetBlobReference(hive, path);
_logger.LogInformation(
"Reading {TypeName} from container {Container} at path {Path}.",
typeName,
GetContainerName(hive),
path);
await _throttle.WaitAsync();
try
{
T result;
using (var blobStream = await blob.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
{
Stream readStream;
if (blob.Properties.ContentEncoding == "gzip")
{
readStream = new GZipStream(blobStream, CompressionMode.Decompress);
}
else
{
readStream = blobStream;
}
using (readStream)
using (var streamReader = new StreamReader(readStream))
using (var jsonTextReader = new JsonTextReader(streamReader))
{
result = NuGetJsonSerialization.Serializer.Deserialize <T>(jsonTextReader);
}
}
_logger.LogInformation(
"Finished reading {TypeName} from container {Container} at path {Path} with Content-Encoding {ContentEncoding}.",
typeName,
GetContainerName(hive),
path,
blob.Properties.ContentEncoding);
return(result);
}
catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
_logger.LogInformation(
"No blob in container {Container} at path {Path} exists.",
GetContainerName(hive),
path,
blob.Properties.ContentEncoding);
if (allow404)
{
return(default(T));
}
else
{
throw;
}
}
finally
{
_throttle.Release();
}
}
/// <summary>
/// Read bytes from the request URL.
/// </summary>
/// <param name="srcOffset">The position from the beginning of the request URL's response body to start reading.</param>
/// <param name="dst">The destination buffer to write bytes to.</param>
/// <param name="dstOffset">The offset in the destination buffer.</param>
/// <param name="count">The maximum number of bytes to read.</param>
/// <returns>The number of bytes read.</returns>
public async Task <int> ReadAsync(long srcOffset, byte[] dst, int dstOffset, int count)
{
return(await RetryHelper.RetryAsync(async lastException =>
{
await _httpThrottle.WaitAsync();
try
{
using (var request = new HttpRequestMessage(HttpMethod.Get, _requestUri))
{
if (_sendXMsVersionHeader)
{
request.Headers.TryAddWithoutValidation("x-ms-version", "2013-08-15");
}
if (_etag != null)
{
if (lastException is MiniZipHttpException ex &&
ex.StatusCode == HttpStatusCode.PreconditionFailed &&
_allowETagVariants == true)
{
// Swap out the etag version (quoted vs. unquoted, related to an old Azure Blob Storage
// bug) if there is an HTTP 412 Precondition Failed. This may be caused by the wrong
// version of the etag being cached by an intermediate layer.
_etag = _etag == _etagNoQuotes ? _etagQuotes : _etagNoQuotes;
}
request.Headers.TryAddWithoutValidation("If-Match", _etag);
}
request.Headers.Range = new RangeHeaderValue(srcOffset, (srcOffset + count) - 1);
using (var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
{
if (response.StatusCode != HttpStatusCode.PartialContent)
{
throw await response.ToHttpExceptionAsync(string.Format(
Strings.NonPartialContentHttpResponse,
(int)response.StatusCode,
response.ReasonPhrase));
}
if (_requireContentRange || response.Content.Headers.ContentRange != null)
{
if (response.Content.Headers.ContentRange == null)
{
throw await response.ToHttpExceptionAsync(Strings.ContentRangeHeaderNotFound);
}
if (!response.Content.Headers.ContentRange.HasRange ||
response.Content.Headers.ContentRange.Unit != HttpConstants.BytesUnit ||
response.Content.Headers.ContentRange.From != srcOffset ||
response.Content.Headers.ContentRange.To != (srcOffset + count) - 1)
{
throw await response.ToHttpExceptionAsync(Strings.InvalidContentRangeHeader);
}
if (response.Content.Headers.ContentRange.Length != _length)
{
throw await response.ToHttpExceptionAsync(string.Format(
Strings.LengthOfHttpContentChanged,
response.Content.Headers.ContentRange.Length,
_length));
}
}
using (var stream = await response.Content.ReadAsStreamAsync())
{
return await stream.ReadToEndAsync(dst, dstOffset, count);
}
}
}
}
finally
{
_httpThrottle.Release();
}
}));
}
private async Task <Tuple <Stream, ILookup <string, string> > > GetStreamAndHeadersAsync(Uri requestUri)
{
// Determine if the exists endpoint's length and whether it supports range requests.
var info = await RetryHelper.RetryAsync(async lastException =>
{
using (var request = new HttpRequestMessage(HttpMethod.Head, requestUri))
{
if (SendXMsVersionHeader)
{
request.Headers.TryAddWithoutValidation("x-ms-version", "2013-08-15");
}
await _httpThrottle.WaitAsync();
try
{
using (var response = await _httpClient.SendAsync(request))
{
if (!response.IsSuccessStatusCode)
{
throw await response.ToHttpExceptionAsync(
string.Format(
Strings.UnsuccessfulHttpStatusCodeWhenGettingLength,
(int)response.StatusCode,
response.ReasonPhrase));
}
if (response.Content?.Headers?.ContentLength == null)
{
throw await response.ToHttpExceptionAsync(Strings.ContentLengthHeaderNotFound);
}
// If unspecified, only allow etag variants if the server appears to be Azure Blob Storage.
bool allowETagVariants;
if (!AllowETagVariants.HasValue &&
response.Headers.TryGetValues("x-ms-version", out _) &&
response.Headers.TryGetValues("x-ms-blob-type", out _))
{
allowETagVariants = true;
}
else
{
allowETagVariants = AllowETagVariants.GetValueOrDefault(false);
}
if (RequireAcceptRanges &&
(response.Headers.AcceptRanges == null ||
!response.Headers.AcceptRanges.Contains(HttpConstants.BytesUnit)))
{
throw await response.ToHttpExceptionAsync(string.Format(
Strings.AcceptRangesBytesValueNotFoundFormat,
HttpConstants.BytesUnit));
}
var length = response.Content.Headers.ContentLength.Value;
var etagBehavior = ETagBehavior;
// Even though we may be sending the x-ms-version header to Azure Blob Storage, it's
// possible a CDN or other intermediate layer has cached the response ETag header without
// quotes. This is why we don't use the parsed response.Headers.ETag value.
string etag = null;
if (response.Headers.TryGetValues("ETag", out var etags) && etags.Any())
{
etag = etags.First();
}
if (etag != null && (etag.StartsWith("W/") || etagBehavior == ETagBehavior.Ignore))
{
etag = null;
}
if (etag == null && etagBehavior == ETagBehavior.Required)
{
throw await response.ToHttpExceptionAsync(string.Format(
Strings.MissingETagHeader,
nameof(MiniZip.ETagBehavior),
nameof(ETagBehavior.Required)));
}
var headers = Enumerable.Empty <KeyValuePair <string, IEnumerable <string> > >()
.Concat(response.Headers)
.Concat(response.Content.Headers)
.SelectMany(x => x.Value.Select(y => new { x.Key, Value = y }))
.ToLookup(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase);
return(new { Length = length, ETag = etag, AllowETagVariants = allowETagVariants, Headers = headers });
}
}
finally
{
_httpThrottle.Release();
}
}
});
var httpRangeReader = new HttpRangeReader(
_httpClient,
requestUri,
info.Length,
info.ETag,
RequireContentRange,
SendXMsVersionHeader,
info.AllowETagVariants,
_httpThrottle);
var bufferSizeProvider = BufferSizeProvider ?? NullBufferSizeProvider.Instance;
var stream = new BufferedRangeStream(httpRangeReader, info.Length, bufferSizeProvider);
return(Tuple.Create <Stream, ILookup <string, string> >(stream, info.Headers));
}