private static async Task <ODataServiceDocumentResourceV2> CreateODataServiceDocumentResourceV2(
SourceRepository source,
DateTime utcNow,
ILogger log,
CancellationToken token)
{
var url = source.PackageSource.Source;
var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);
var client = httpSourceResource.HttpSource;
// Get the service document and record the URL after any redirects.
string lastRequestUri;
try
{
lastRequestUri = await client.ProcessResponseAsync(
new HttpSourceRequest(() => HttpRequestMessageFactory.Create(HttpMethod.Get, url, log)),
response =>
{
if (response.RequestMessage == null)
{
return(Task.FromResult(url));
}
return(Task.FromResult(response.RequestMessage.RequestUri.ToString()));
},
log,
token);
}
catch (Exception ex) when(!(ex is FatalProtocolException) && (!(ex is OperationCanceledException)))
{
string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, source.PackageSource.Source);
log.LogError(message + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex));
throw new FatalProtocolException(message, ex);
}
// Trim the query string or any trailing slash.
var builder = new UriBuilder(lastRequestUri)
{
Query = null
};
var baseAddress = builder.Uri.AbsoluteUri.Trim('/');
return(new ODataServiceDocumentResourceV2(baseAddress, DateTime.UtcNow));
}
internal async Task <XDocument> LoadXmlAsync(
string uri,
bool ignoreNotFounds,
ILogger log,
CancellationToken token)
{
return(await _httpSource.ProcessResponseAsync(
new HttpSourceRequest(
() =>
{
var request = HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/atom+xml"));
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
return request;
}),
async response =>
{
if (response.StatusCode == HttpStatusCode.OK)
{
var networkStream = await response.Content.ReadAsStreamAsync();
return LoadXml(networkStream);
}
else if (ignoreNotFounds && response.StatusCode == HttpStatusCode.NotFound)
{
// Treat "404 Not Found" as an empty response.
return null;
}
else if (response.StatusCode == HttpStatusCode.NoContent)
{
// Always treat "204 No Content" as exactly that.
return null;
}
else
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToFetchV2Feed,
uri,
(int)response.StatusCode,
response.ReasonPhrase));
}
},
log,
token));
}
/// <summary>
/// Caching Get request.
/// </summary>
public async Task <T> GetAsync <T>(
HttpSourceCachedRequest request,
Func <HttpSourceResult, Task <T> > processAsync,
ILogger log,
CancellationToken token)
{
var cacheResult = HttpCacheUtility.InitializeHttpCacheResult(
HttpCacheDirectory,
_sourceUri,
request.CacheKey,
request.CacheContext);
return(await ConcurrencyUtilities.ExecuteWithFileLockedAsync(
cacheResult.CacheFile,
action : async lockedToken =>
{
cacheResult.Stream = TryReadCacheFile(request.Uri, cacheResult.MaxAge, cacheResult.CacheFile);
if (cacheResult.Stream != null)
{
log.LogInformation(string.Format(CultureInfo.InvariantCulture, " " + Strings.Http_RequestLog, "CACHE", request.Uri));
// Validate the content fetched from the cache.
try
{
request.EnsureValidContents?.Invoke(cacheResult.Stream);
cacheResult.Stream.Seek(0, SeekOrigin.Begin);
var httpSourceResult = new HttpSourceResult(
HttpSourceResultStatus.OpenedFromDisk,
cacheResult.CacheFile,
cacheResult.Stream);
return await processAsync(httpSourceResult);
}
catch (Exception e)
{
cacheResult.Stream.Dispose();
cacheResult.Stream = null;
string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_InvalidCacheEntry, request.Uri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(e);
log.LogWarning(message);
}
}
Func <HttpRequestMessage> requestFactory = () =>
{
var requestMessage = HttpRequestMessageFactory.Create(HttpMethod.Get, request.Uri, log);
foreach (var acceptHeaderValue in request.AcceptHeaderValues)
{
requestMessage.Headers.Accept.Add(acceptHeaderValue);
}
return requestMessage;
};
Func <Task <ThrottledResponse> > throttledResponseFactory = () => GetThrottledResponse(
requestFactory,
request.RequestTimeout,
request.DownloadTimeout,
request.MaxTries,
log,
lockedToken);
using (var throttledResponse = await throttledResponseFactory())
{
if (request.IgnoreNotFounds && throttledResponse.Response.StatusCode == HttpStatusCode.NotFound)
{
var httpSourceResult = new HttpSourceResult(HttpSourceResultStatus.NotFound);
return await processAsync(httpSourceResult);
}
if (throttledResponse.Response.StatusCode == HttpStatusCode.NoContent)
{
// Ignore reading and caching the empty stream.
var httpSourceResult = new HttpSourceResult(HttpSourceResultStatus.NoContent);
return await processAsync(httpSourceResult);
}
throttledResponse.Response.EnsureSuccessStatusCode();
if (!request.CacheContext.DirectDownload)
{
await HttpCacheUtility.CreateCacheFileAsync(
cacheResult,
throttledResponse.Response,
request.EnsureValidContents,
lockedToken);
using (var httpSourceResult = new HttpSourceResult(
HttpSourceResultStatus.OpenedFromDisk,
cacheResult.CacheFile,
cacheResult.Stream))
{
return await processAsync(httpSourceResult);
}
}
else
{
// Note that we do not execute the content validator on the response stream when skipping
// the cache. We cannot seek on the network stream and it is not valuable to download the
// content twice just to validate the first time (considering that the second download could
// be different from the first thus rendering the first validation meaningless).
using (var stream = await throttledResponse.Response.Content.ReadAsStreamAsync())
using (var httpSourceResult = new HttpSourceResult(
HttpSourceResultStatus.OpenedFromNetwork,
cacheFileName: null,
stream: stream))
{
return await processAsync(httpSourceResult);
}
}
}
},
token : token));
}
internal async Task <XDocument> LoadXmlAsync(
string uri,
string cacheKey,
bool ignoreNotFounds,
SourceCacheContext sourceCacheContext,
ILogger log,
CancellationToken token)
{
if (cacheKey != null && sourceCacheContext != null)
{
var httpSourceCacheContext = HttpSourceCacheContext.Create(sourceCacheContext, 0);
try
{
return(await _httpSource.GetAsync(
new HttpSourceCachedRequest(
uri,
cacheKey,
httpSourceCacheContext)
{
AcceptHeaderValues =
{
new MediaTypeWithQualityHeaderValue("application/atom+xml"),
new MediaTypeWithQualityHeaderValue("application/xml")
},
EnsureValidContents = stream => HttpStreamValidation.ValidateXml(uri, stream),
MaxTries = 1,
IgnoreNotFounds = ignoreNotFounds
},
async response =>
{
if (ignoreNotFounds && response.Status == HttpSourceResultStatus.NotFound)
{
// Treat "404 Not Found" as an empty response.
return null;
}
else if (response.Status == HttpSourceResultStatus.NoContent)
{
// Always treat "204 No Content" as exactly that.
return null;
}
else
{
return await LoadXmlAsync(response.Stream, token);
}
},
log,
token));
}
catch (Exception ex)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToFetchV2FeedHttp,
uri,
ex.Message);
throw new FatalProtocolException(message, ex);
}
}
else
{
// return results without httpCache
return(await _httpSource.ProcessResponseAsync(
new HttpSourceRequest(
() =>
{
var request = HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/atom+xml"));
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
return request;
})
{
IsRetry = true
},
async response =>
{
if (response.StatusCode == HttpStatusCode.OK)
{
var networkStream = await response.Content.ReadAsStreamAsync();
return await LoadXmlAsync(networkStream, token);
}
else if (ignoreNotFounds && response.StatusCode == HttpStatusCode.NotFound)
{
// Treat "404 Not Found" as an empty response.
return null;
}
else if (response.StatusCode == HttpStatusCode.NoContent)
{
// Always treat "204 No Content" as exactly that.
return null;
}
else
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToFetchV2Feed,
uri,
(int)response.StatusCode,
response.ReasonPhrase));
}
},
sourceCacheContext,
log,
token));
}
}
public HttpSourceRequest(Uri uri, ILogger log)
: this(() => HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log))
{
}
/// <summary>
/// Caching Get request.
/// </summary>
public async Task <HttpSourceResult> GetAsync(
HttpSourceCachedRequest request,
ILogger log,
CancellationToken token)
{
var result = HttpCacheUtility.InitializeHttpCacheResult(
HttpCacheDirectory,
_baseUri,
request.CacheKey,
request.CacheContext);
return(await ConcurrencyUtilities.ExecuteWithFileLockedAsync(
result.CacheFile,
action : async lockedToken =>
{
result.Stream = TryReadCacheFile(request.Uri, result.MaxAge, result.CacheFile);
if (result.Stream != null)
{
log.LogInformation(string.Format(CultureInfo.InvariantCulture, " " + Strings.Http_RequestLog, "CACHE", request.Uri));
// Validate the content fetched from the cache.
try
{
request.EnsureValidContents?.Invoke(result.Stream);
result.Stream.Seek(0, SeekOrigin.Begin);
return new HttpSourceResult(
HttpSourceResultStatus.OpenedFromDisk,
result.CacheFile,
result.Stream);
}
catch (Exception e)
{
result.Stream.Dispose();
result.Stream = null;
string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_InvalidCacheEntry, request.Uri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(e);
log.LogWarning(message);
}
}
Func <HttpRequestMessage> requestFactory = () =>
{
var requestMessage = HttpRequestMessageFactory.Create(HttpMethod.Get, request.Uri, log);
foreach (var acceptHeaderValue in request.AcceptHeaderValues)
{
requestMessage.Headers.Accept.Add(acceptHeaderValue);
}
return requestMessage;
};
Func <Task <ThrottledResponse> > throttledResponseFactory = () => GetThrottledResponse(
requestFactory,
request.RequestTimeout,
request.DownloadTimeout,
log,
lockedToken);
using (var throttledResponse = await throttledResponseFactory())
{
if (request.IgnoreNotFounds && throttledResponse.Response.StatusCode == HttpStatusCode.NotFound)
{
return new HttpSourceResult(HttpSourceResultStatus.NotFound);
}
if (throttledResponse.Response.StatusCode == HttpStatusCode.NoContent)
{
// Ignore reading and caching the empty stream.
return new HttpSourceResult(HttpSourceResultStatus.NoContent);
}
throttledResponse.Response.EnsureSuccessStatusCode();
await HttpCacheUtility.CreateCacheFileAsync(
result,
request.Uri,
throttledResponse.Response,
request.CacheContext,
request.EnsureValidContents,
lockedToken);
return new HttpSourceResult(
HttpSourceResultStatus.OpenedFromDisk,
result.CacheFile,
result.Stream);
}
},
token : token));
}