private void CheckValidParse(string input, RangeHeaderValue expectedResult)
{
RangeHeaderValue result = RangeHeaderValue.Parse(input);
Assert.Equal(expectedResult, result);
}
/// <summary>
/// Run a simple HTTP server that listens for a few test URIs.
/// The server exits when a client requests "/Quit".
/// </summary>
/// <param name="prefix">Prefix at which the server should listen</param>
private async Task RunTestServer(string prefix)
{
using (var httpListener = new HttpListener())
{
httpListener.Prefixes.Add(prefix);
httpListener.Start();
while (httpListener.IsListening)
{
var context = await httpListener.GetContextAsync().ConfigureAwait(false);
var requestUri = context.Request.Url;
var response = context.Response;
if (requestUri.AbsolutePath.EndsWith("/Quit"))
{
// Shut down the HTTP server.
response.Close();
httpListener.Stop();
continue;
}
// All downloader requests should include ?alt=media.
Assert.Equal("media", context.Request.QueryString["alt"]);
response.ContentType = "text/plain";
response.SendChunked = true; // Avoid having to set Content-Length.
Stream outStream = new MemoryStream();
if (requestUri.AbsolutePath.EndsWith("/EchoUrl"))
{
// Return the URL that we saw.
byte[] uriBytes = Encoding.UTF8.GetBytes(requestUri.AbsoluteUri);
outStream.Write(uriBytes, 0, uriBytes.Length);
}
else if (requestUri.AbsolutePath.EndsWith("/BadRequestJson"))
{
// Return 400 with a JSON-encoded error.
var apiResponse = new StandardResponse <object> {
Error = BadRequestError
};
var apiResponseText = new NewtonsoftJsonSerializer().Serialize(apiResponse);
byte[] apiResponseBytes = Encoding.UTF8.GetBytes(apiResponseText);
response.StatusCode = (int)HttpStatusCode.BadRequest;
outStream.Write(apiResponseBytes, 0, apiResponseBytes.Length);
}
else if (requestUri.AbsolutePath.EndsWith("/NotFoundPlainText"))
{
// Return 404 with a plaintext error.
response.StatusCode = (int)HttpStatusCode.NotFound;
byte[] errorBytes = Encoding.UTF8.GetBytes(NotFoundError);
outStream.Write(errorBytes, 0, errorBytes.Length);
}
else if (requestUri.AbsolutePath.EndsWith("/GzipContent"))
{
// Return gzip-compressed content.
using (var gzipStream = new GZipStream(outStream, CompressionMode.Compress, true))
{
gzipStream.Write(MediaContent, 0, MediaContent.Length);
}
response.AddHeader("Content-Encoding", "gzip");
}
else if (requestUri.AbsolutePath.EndsWith("/NoContent"))
{
// Return 206 and no content.
response.StatusCode = (int)HttpStatusCode.NoContent;
}
else
{
// Return plaintext content.
outStream.Write(MediaContent, 0, MediaContent.Length);
}
outStream.Position = 0;
// Provide rudimentary, non-robust support for Range.
string rangeHeader = context.Request.Headers["Range"];
if (rangeHeader != null && response.StatusCode == (int)HttpStatusCode.OK)
{
var range = RangeHeaderValue.Parse(rangeHeader);
var firstRange = range.Ranges.First();
long from = firstRange.From ?? 0;
long to = Math.Min(outStream.Length - 1, firstRange.To ?? long.MaxValue);
var contentRangeHeader = new ContentRangeHeaderValue(from, to, outStream.Length);
response.AddHeader("Content-Range", contentRangeHeader.ToString());
outStream.Position = from;
outStream.SetLength(to + 1);
}
await outStream.CopyToAsync(response.OutputStream).ConfigureAwait(false);
response.Close();
}
}
}
private HttpWebRequest CreateRequest(Request messageRequest)
{
var request = WebRequest.CreateHttp(messageRequest.Uri.ToUri());
request.Method = messageRequest.Method.Method;
request.Proxy = _proxy;
foreach (var messageRequestHeader in messageRequest.Headers)
{
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.ContentLength, StringComparison.OrdinalIgnoreCase))
{
request.ContentLength = int.Parse(messageRequestHeader.Value, CultureInfo.InvariantCulture);
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Host, StringComparison.OrdinalIgnoreCase))
{
request.Host = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Date, StringComparison.OrdinalIgnoreCase))
{
request.Date = DateTime.Parse(messageRequestHeader.Value, CultureInfo.InvariantCulture);
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.ContentType, StringComparison.OrdinalIgnoreCase))
{
request.ContentType = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.UserAgent, StringComparison.OrdinalIgnoreCase))
{
request.UserAgent = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Accept, StringComparison.OrdinalIgnoreCase))
{
request.Accept = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Referer, StringComparison.OrdinalIgnoreCase))
{
request.Referer = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Range, StringComparison.OrdinalIgnoreCase))
{
var value = RangeHeaderValue.Parse(messageRequestHeader.Value);
if (value.Unit != "bytes")
{
throw new InvalidOperationException("Only ranges with bytes unit supported.");
}
foreach (var rangeItem in value.Ranges)
{
if (rangeItem.From == null)
{
throw new InvalidOperationException("Only ranges with Offset supported.");
}
if (rangeItem.To == null)
{
request.AddRange(rangeItem.From.Value);
}
else
{
request.AddRange(rangeItem.From.Value, rangeItem.To.Value);
}
}
continue;
}
request.Headers.Add(messageRequestHeader.Name, messageRequestHeader.Value);
}
return(request);
}
private void CheckInvalidParse(string?input)
{
Assert.Throws <FormatException>(() => RangeHeaderValue.Parse(input));
}
/// <summary>
/// Insert a request HTTP header
/// </summary>
/// <param name="header">A <see cref="System.Net.Http.Headers.HttpRequestHeaders"/> object on wich the request header will be recorded.</param>
/// <param name="name">The header attribute name.</param>
/// <param name="value">The header attribute value.</param>
public void AddRequestHeader(HttpRequestHeaders header, string name, string value)
{
if (name.Equals("Authorization", StringComparison.OrdinalIgnoreCase))
{
header.Authorization = new AuthenticationHeaderValue(value);
}
else if (name.Equals("Cache-Control", StringComparison.OrdinalIgnoreCase))
{
header.CacheControl = CacheControlHeaderValue.Parse(value);
}
else if (name.Equals("Date", StringComparison.OrdinalIgnoreCase))
{
header.Date = DateTimeOffset.Parse(value);
}
else if (name.Equals("If-Modified-Since", StringComparison.OrdinalIgnoreCase))
{
header.IfModifiedSince = DateTimeOffset.Parse(value);
}
else if (name.Equals("If-Range", StringComparison.OrdinalIgnoreCase))
{
header.IfRange = RangeConditionHeaderValue.Parse(value);
}
else if (name.Equals("If-Unmodified-Since", StringComparison.OrdinalIgnoreCase))
{
header.IfUnmodifiedSince = DateTimeOffset.Parse(value);
}
else if (name.Equals("Max-Forwards", StringComparison.OrdinalIgnoreCase))
{
header.MaxForwards = int.Parse(value);
}
else if (name.Equals("Proxy-Authorization", StringComparison.OrdinalIgnoreCase))
{
header.ProxyAuthorization = AuthenticationHeaderValue.Parse(value);
}
else if (name.Equals("Range", StringComparison.OrdinalIgnoreCase))
{
header.Range = RangeHeaderValue.Parse(value);
}
else if (name.Equals("Referrer", StringComparison.OrdinalIgnoreCase))
{
header.Referrer = new Uri(value);
}
else
{
try{
header.Add(name, value);
}catch (ArgumentException) {
if (header.GetType().GetProperty(name.Replace("-", "")) != null)
{
header.GetType().GetProperty(name.Replace("-", "")).SetValue(header, value);
}
else
{
throw;
}
}
}
}
private HttpWebRequest CreateRequest(Request messageRequest)
{
var request = WebRequest.CreateHttp(messageRequest.Uri.ToUri());
// Don't disable the default proxy when there is no environment proxy configured
if (_environmentProxy != null)
{
request.Proxy = _environmentProxy;
}
_configureRequest(request);
request.Method = messageRequest.Method.Method;
foreach (var messageRequestHeader in messageRequest.Headers)
{
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.ContentLength, StringComparison.OrdinalIgnoreCase))
{
request.ContentLength = long.Parse(messageRequestHeader.Value, CultureInfo.InvariantCulture);
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Host, StringComparison.OrdinalIgnoreCase))
{
request.Host = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Date, StringComparison.OrdinalIgnoreCase))
{
request.Date = DateTime.Parse(messageRequestHeader.Value, CultureInfo.InvariantCulture);
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.ContentType, StringComparison.OrdinalIgnoreCase))
{
request.ContentType = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.UserAgent, StringComparison.OrdinalIgnoreCase))
{
request.UserAgent = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Accept, StringComparison.OrdinalIgnoreCase))
{
request.Accept = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Referer, StringComparison.OrdinalIgnoreCase))
{
request.Referer = messageRequestHeader.Value;
continue;
}
if (string.Equals(messageRequestHeader.Name, HttpHeader.Names.Range, StringComparison.OrdinalIgnoreCase))
{
var value = RangeHeaderValue.Parse(messageRequestHeader.Value);
if (value.Unit != "bytes")
{
throw new InvalidOperationException("Only ranges with bytes unit supported.");
}
foreach (var rangeItem in value.Ranges)
{
if (rangeItem.From == null)
{
throw new InvalidOperationException("Only ranges with Offset supported.");
}
if (rangeItem.To == null)
{
request.AddRange(rangeItem.From.Value);
}
else
{
request.AddRange(rangeItem.From.Value, rangeItem.To.Value);
}
}
continue;
}
request.Headers.Add(messageRequestHeader.Name, messageRequestHeader.Value);
}
if (request.ContentLength == -1 &&
messageRequest.Content != null &&
messageRequest.Content.TryComputeLength(out var length))
{
request.ContentLength = length;
}
if (request.ContentLength != -1)
{
// disable buffering when the content length is known
// as the content stream is re-playable and we don't want to allocate extra buffers
request.AllowWriteStreamBuffering = false;
}
return(request);
}
public async Task <StreamingResult> GetContents(FileIdentifier identifier, CancellationToken cancellationToken)
{
if (!ModelState.IsValid)
{
throw new ModelValidationException(ModelState);
}
if (identifier == null)
{
throw new ArgumentNullException(nameof(identifier));
}
if (!identifier.IsValid)
{
throw new InvalidIdentifierException();
}
var fileModel = await FileStore.GetOneAsync(identifier);
if (fileModel == null)
{
throw new ObjectDoesNotExistException();
}
long from = 0;
long to = fileModel.Length - 1;
if (Request.Headers["Range"].Any())
{
var range = RangeHeaderValue.Parse(Request.Headers["Range"].ToString()).Ranges.FirstOrDefault();
if (range.From.HasValue)
{
from = range.From.Value;
}
if (range.To.HasValue)
{
to = Math.Min(range.To.Value, to);
}
Response.StatusCode = (int)HttpStatusCode.PartialContent;
Response.Headers.Add("Content-Range", new ContentRangeHeaderValue(
from,
to,
fileModel.Length
).ToString());
}
long rangeLength = to - from + 1;
var contentDisposition = new ContentDispositionHeaderValue("attachment");
contentDisposition.SetHttpFileName(fileModel.Name);
Response.Headers[HeaderNames.ContentDisposition] = contentDisposition.ToString();
// tell the base class not to screw with our result
SuppressWrapper = true;
return(new StreamingResult(async response =>
{
Response.ContentLength = rangeLength;
Response.ContentType = fileModel.MimeType ?? "application/octet-stream";
await FileContentsService.DownloadAsync(fileModel, Response.Body, from, to, cancellationToken);
}));
}
async Task DoDownloadPackage(PackageDef package, FileStream fileStream, CancellationToken cancellationToken)
{
bool finished = false;
try
{
using (HttpClientHandler hch = new HttpClientHandler()
{
UseProxy = true, Proxy = WebRequest.GetSystemWebProxy()
})
using (HttpClient hc = new HttpClient(hch)
{
Timeout = Timeout.InfiniteTimeSpan
})
{
HttpResponseMessage response = null;
hc.DefaultRequestHeaders.Add("OpenTAP", PluginManager.GetOpenTapAssembly().SemanticVersion.ToString());
var retries = 60;
var downloadedBytes = 0;
var totalSize = -1L;
while (retries > 0)
{
if (retries < 60)
{
log.Debug($"Retrying {61 - retries}/60");
}
hc.DefaultRequestHeaders.Range = RangeHeaderValue.Parse($"bytes={downloadedBytes}-");
try
{
if (package.PackageSource is HttpRepositoryPackageDefSource httpSource && string.IsNullOrEmpty(httpSource.DirectUrl) == false)
{
log.Info($"Downloading package directly from: '{httpSource.DirectUrl}'.");
var message = new HttpRequestMessage(HttpMethod.Get, new Uri(httpSource.DirectUrl));
try
{
response = await hc.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (response.IsSuccessStatusCode == false)
{
throw new Exception($"Request to '{httpSource.DirectUrl}' failed with status code: {response.StatusCode}.");
}
}
catch (Exception e)
{
log.Warning($"Could not download package directly from: '{httpSource.DirectUrl}'. Downloading package normally.");
log.Debug(e);
response = await hc.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
}
}
else
{
var message = new HttpRequestMessage(HttpMethod.Get,
new Uri(Url + "/" + ApiVersion + "/DownloadPackage" +
$"/{Uri.EscapeDataString(package.Name)}" +
$"?version={Uri.EscapeDataString(package.Version.ToString())}" +
$"&os={Uri.EscapeDataString(package.OS)}" +
$"&architecture={Uri.EscapeDataString(package.Architecture.ToString())}"));
response = await hc.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
}
if (totalSize < 0)
{
totalSize = response.Content.Headers.ContentLength ?? 1;
}
// Download the package
using (var responseStream = await response.Content.ReadAsStreamAsync())
{
if (response.IsSuccessStatusCode == false)
{
throw new HttpRequestException($"The download request failed with {response.StatusCode}.");
}
var buffer = new byte[4096];
int read = 0;
var task = Task.Run(() =>
{
do
{
read = responseStream.Read(buffer, 0, 4096);
fileStream.Write(buffer, 0, read);
downloadedBytes += read;
} while (read > 0);
finished = true;
}, cancellationToken);
ConsoleUtils.PrintProgressTillEnd(task, "Downloading", () => fileStream.Position, () => totalSize);
}
if (finished)
{
break;
}
}
catch (Exception e)
{
response.Dispose();
retries--;
if (retries <= 0 || cancellationToken.IsCancellationRequested)
{
throw;
}
log.Debug("Failed to download package.");
log.Debug(e);
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
