/// <summary>
/// Initialize the buffered range reader stream provided request URL.
/// </summary>
/// <param name="requestUri">The request URL.</param>
/// <returns>The buffered range reader stream.</returns>
public async Task <Stream> GetStreamAsync(Uri requestUri)
{
// Determine if the exists endpoint's length and whether it supports range requests.
using (var request = new HttpRequestMessage(HttpMethod.Head, requestUri))
using (var response = await _httpClient.SendAsync(request))
{
if (!response.IsSuccessStatusCode)
{
throw new MiniZipException(string.Format(
Strings.UnsuccessfulHttpStatusCodeWhenGettingLength,
(int)response.StatusCode,
response.ReasonPhrase));
}
if (response.Content?.Headers?.ContentLength == null)
{
throw new MiniZipException(Strings.ContentLengthHeaderNotFound);
}
if (response.Headers.AcceptRanges == null ||
!response.Headers.AcceptRanges.Contains(HttpConstants.BytesUnit))
{
throw new MiniZipException(string.Format(
Strings.AcceptRangesBytesValueNotFoundFormat,
HttpConstants.BytesUnit));
}
var length = response.Content.Headers.ContentLength.Value;
var etagBehavior = ETagBehavior;
var etag = response.Headers?.ETag;
if (etag != null && (etag.IsWeak || etagBehavior == ETagBehavior.Ignore))
{
etag = null;
}
if (etag == null && etagBehavior == ETagBehavior.Required)
{
throw new MiniZipException(string.Format(
Strings.MissingETagHeader,
nameof(MiniZip.ETagBehavior),
nameof(ETagBehavior.Required)));
}
var httpRangeReader = new HttpRangeReader(_httpClient, requestUri, length, etag);
var bufferSizeProvider = new ZipBufferSizeProvider(FirstBufferSize, SecondBufferSize, BufferGrowthExponent);
var stream = new BufferedRangeStream(httpRangeReader, length, bufferSizeProvider);
return(stream);
}
}
public ReadAsync()
{
_buffer = Enumerable.Range(0, 100).Select(x => (byte)x).ToArray();
_rangeReader = new Mock <BufferRangeReader>(_buffer)
{
CallBase = true
};
_length = _buffer.Length;
_bufferSizeProvider = new Mock <IBufferSizeProvider>();
_bufferSizeProvider
.Setup(x => x.GetNextBufferSize())
.Returns(10);
_target = new BufferedRangeStream(
_rangeReader.Object,
_length,
_bufferSizeProvider.Object);
_target.Position = 80;
_outputBuffer = new byte[_length];
_extraBuffer = new byte[_length];
}
public async Task WithSelfUsingFileRangeReader(string path)
{
// Arrange
using (var memoryStream = TestUtility.BufferTestData(path))
{
var fullPath = Path.Combine(TestUtility.TestDataDirectory, path);
var length = new FileInfo(fullPath).Length;
var fileRangeReader = new FileRangeReader(fullPath);
var bufferSizeProvider = new ZipBufferSizeProvider(firstBufferSize: 1, secondBufferSize: 1, exponent: 2);
using (var bufferedRangeStream = new BufferedRangeStream(fileRangeReader, length, bufferSizeProvider))
{
// Act
var a = await TestUtility.ReadWithMiniZipAsync(memoryStream);
var b = await TestUtility.ReadWithMiniZipAsync(bufferedRangeStream);
// Assert
TestUtility.VerifyJsonEquals(a.Data, b.Data);
Assert.Equal(a.Success, b.Success);
Assert.Equal(a.Exception?.Message, b.Exception?.Message);
Assert.Equal(a.Exception?.GetType(), b.Exception?.GetType());
}
}
}
/// <summary>
/// Initialize the buffered range reader stream provided request URL.
/// </summary>
/// <param name="requestUri">The request URL.</param>
/// <returns>The buffered range reader stream.</returns>
public async Task <Stream> GetStreamAsync(Uri requestUri)
{
// Determine if the exists endpoint's length and whether it supports range requests.
var info = await RetryHelper.RetryAsync(async() =>
{
using (var request = new HttpRequestMessage(HttpMethod.Head, requestUri))
{
if (SendXMsVersionHeader)
{
request.Headers.TryAddWithoutValidation("x-ms-version", "2013-08-15");
}
using (var response = await _httpClient.SendAsync(request))
{
if (!response.IsSuccessStatusCode)
{
throw new MiniZipHttpStatusCodeException(
string.Format(
Strings.UnsuccessfulHttpStatusCodeWhenGettingLength,
(int)response.StatusCode,
response.ReasonPhrase),
response.StatusCode,
response.ReasonPhrase);
}
if (response.Content?.Headers?.ContentLength == null)
{
throw new MiniZipException(Strings.ContentLengthHeaderNotFound);
}
if (RequireAcceptRanges &&
(response.Headers.AcceptRanges == null ||
!response.Headers.AcceptRanges.Contains(HttpConstants.BytesUnit)))
{
throw new MiniZipException(string.Format(
Strings.AcceptRangesBytesValueNotFoundFormat,
HttpConstants.BytesUnit));
}
var length = response.Content.Headers.ContentLength.Value;
var etagBehavior = ETagBehavior;
var etag = response.Headers?.ETag;
if (etag != null && (etag.IsWeak || etagBehavior == ETagBehavior.Ignore))
{
etag = null;
}
if (etag == null && etagBehavior == ETagBehavior.Required)
{
throw new MiniZipException(string.Format(
Strings.MissingETagHeader,
nameof(MiniZip.ETagBehavior),
nameof(ETagBehavior.Required)));
}
return(new { Length = length, ETag = etag });
}
}
});
var httpRangeReader = new HttpRangeReader(_httpClient, requestUri, info.Length, info.ETag);
var bufferSizeProvider = BufferSizeProvider ?? NullBufferSizeProvider.Instance;
var stream = new BufferedRangeStream(httpRangeReader, info.Length, bufferSizeProvider);
return(stream);
}
private async Task <Tuple <Stream, ILookup <string, string> > > GetStreamAndHeadersAsync(Uri requestUri)
{
// Determine if the exists endpoint's length and whether it supports range requests.
var info = await RetryHelper.RetryAsync(async lastException =>
{
using (var request = new HttpRequestMessage(HttpMethod.Head, requestUri))
{
if (SendXMsVersionHeader)
{
request.Headers.TryAddWithoutValidation("x-ms-version", "2013-08-15");
}
await _httpThrottle.WaitAsync();
try
{
using (var response = await _httpClient.SendAsync(request))
{
if (!response.IsSuccessStatusCode)
{
throw await response.ToHttpExceptionAsync(
string.Format(
Strings.UnsuccessfulHttpStatusCodeWhenGettingLength,
(int)response.StatusCode,
response.ReasonPhrase));
}
if (response.Content?.Headers?.ContentLength == null)
{
throw await response.ToHttpExceptionAsync(Strings.ContentLengthHeaderNotFound);
}
// If unspecified, only allow etag variants if the server appears to be Azure Blob Storage.
bool allowETagVariants;
if (!AllowETagVariants.HasValue &&
response.Headers.TryGetValues("x-ms-version", out _) &&
response.Headers.TryGetValues("x-ms-blob-type", out _))
{
allowETagVariants = true;
}
else
{
allowETagVariants = AllowETagVariants.GetValueOrDefault(false);
}
if (RequireAcceptRanges &&
(response.Headers.AcceptRanges == null ||
!response.Headers.AcceptRanges.Contains(HttpConstants.BytesUnit)))
{
throw await response.ToHttpExceptionAsync(string.Format(
Strings.AcceptRangesBytesValueNotFoundFormat,
HttpConstants.BytesUnit));
}
var length = response.Content.Headers.ContentLength.Value;
var etagBehavior = ETagBehavior;
// Even though we may be sending the x-ms-version header to Azure Blob Storage, it's
// possible a CDN or other intermediate layer has cached the response ETag header without
// quotes. This is why we don't use the parsed response.Headers.ETag value.
string etag = null;
if (response.Headers.TryGetValues("ETag", out var etags) && etags.Any())
{
etag = etags.First();
}
if (etag != null && (etag.StartsWith("W/") || etagBehavior == ETagBehavior.Ignore))
{
etag = null;
}
if (etag == null && etagBehavior == ETagBehavior.Required)
{
throw await response.ToHttpExceptionAsync(string.Format(
Strings.MissingETagHeader,
nameof(MiniZip.ETagBehavior),
nameof(ETagBehavior.Required)));
}
var headers = Enumerable.Empty <KeyValuePair <string, IEnumerable <string> > >()
.Concat(response.Headers)
.Concat(response.Content.Headers)
.SelectMany(x => x.Value.Select(y => new { x.Key, Value = y }))
.ToLookup(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase);
return(new { Length = length, ETag = etag, AllowETagVariants = allowETagVariants, Headers = headers });
}
}
finally
{
_httpThrottle.Release();
}
}
});
var httpRangeReader = new HttpRangeReader(
_httpClient,
requestUri,
info.Length,
info.ETag,
RequireContentRange,
SendXMsVersionHeader,
info.AllowETagVariants,
_httpThrottle);
var bufferSizeProvider = BufferSizeProvider ?? NullBufferSizeProvider.Instance;
var stream = new BufferedRangeStream(httpRangeReader, info.Length, bufferSizeProvider);
return(Tuple.Create <Stream, ILookup <string, string> >(stream, info.Headers));
}