private static async Task <ODataServiceDocumentResourceV2> CreateODataServiceDocumentResourceV2(
            SourceRepository source,
            DateTime utcNow,
            ILogger log,
            CancellationToken token)
        {
            var url = source.PackageSource.Source;
            var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);

            var client = httpSourceResource.HttpSource;

            // Get the service document and record the URL after any redirects.
            string lastRequestUri;

            try
            {
                lastRequestUri = await client.ProcessResponseAsync(
                    new HttpSourceRequest(() => HttpRequestMessageFactory.Create(HttpMethod.Get, url, log)),
                    response =>
                {
                    if (response.RequestMessage == null)
                    {
                        return(Task.FromResult(url));
                    }

                    return(Task.FromResult(response.RequestMessage.RequestUri.ToString()));
                },
                    log,
                    token);
            }
            catch (Exception ex) when(!(ex is FatalProtocolException) && (!(ex is OperationCanceledException)))
            {
                string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, source.PackageSource.Source);

                log.LogError(message + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex));

                throw new FatalProtocolException(message, ex);
            }

            // Trim the query string or any trailing slash.
            var builder = new UriBuilder(lastRequestUri)
            {
                Query = null
            };
            var baseAddress = builder.Uri.AbsoluteUri.Trim('/');

            return(new ODataServiceDocumentResourceV2(baseAddress, DateTime.UtcNow));
        }
Beispiel #2
0
 internal async Task <XDocument> LoadXmlAsync(
     string uri,
     bool ignoreNotFounds,
     ILogger log,
     CancellationToken token)
 {
     return(await _httpSource.ProcessResponseAsync(
                new HttpSourceRequest(
                    () =>
     {
         var request = HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log);
         request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/atom+xml"));
         request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
         return request;
     }),
                async response =>
     {
         if (response.StatusCode == HttpStatusCode.OK)
         {
             var networkStream = await response.Content.ReadAsStreamAsync();
             return LoadXml(networkStream);
         }
         else if (ignoreNotFounds && response.StatusCode == HttpStatusCode.NotFound)
         {
             // Treat "404 Not Found" as an empty response.
             return null;
         }
         else if (response.StatusCode == HttpStatusCode.NoContent)
         {
             // Always treat "204 No Content" as exactly that.
             return null;
         }
         else
         {
             throw new FatalProtocolException(string.Format(
                                                  CultureInfo.CurrentCulture,
                                                  Strings.Log_FailedToFetchV2Feed,
                                                  uri,
                                                  (int)response.StatusCode,
                                                  response.ReasonPhrase));
         }
     },
                log,
                token));
 }
Beispiel #3
0
        /// <summary>
        /// Caching Get request.
        /// </summary>
        public async Task <T> GetAsync <T>(
            HttpSourceCachedRequest request,
            Func <HttpSourceResult, Task <T> > processAsync,
            ILogger log,
            CancellationToken token)
        {
            var cacheResult = HttpCacheUtility.InitializeHttpCacheResult(
                HttpCacheDirectory,
                _sourceUri,
                request.CacheKey,
                request.CacheContext);

            return(await ConcurrencyUtilities.ExecuteWithFileLockedAsync(
                       cacheResult.CacheFile,
                       action : async lockedToken =>
            {
                cacheResult.Stream = TryReadCacheFile(request.Uri, cacheResult.MaxAge, cacheResult.CacheFile);

                if (cacheResult.Stream != null)
                {
                    log.LogInformation(string.Format(CultureInfo.InvariantCulture, "  " + Strings.Http_RequestLog, "CACHE", request.Uri));

                    // Validate the content fetched from the cache.
                    try
                    {
                        request.EnsureValidContents?.Invoke(cacheResult.Stream);

                        cacheResult.Stream.Seek(0, SeekOrigin.Begin);

                        var httpSourceResult = new HttpSourceResult(
                            HttpSourceResultStatus.OpenedFromDisk,
                            cacheResult.CacheFile,
                            cacheResult.Stream);

                        return await processAsync(httpSourceResult);
                    }
                    catch (Exception e)
                    {
                        cacheResult.Stream.Dispose();
                        cacheResult.Stream = null;

                        string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_InvalidCacheEntry, request.Uri)
                                         + Environment.NewLine
                                         + ExceptionUtilities.DisplayMessage(e);
                        log.LogWarning(message);
                    }
                }

                Func <HttpRequestMessage> requestFactory = () =>
                {
                    var requestMessage = HttpRequestMessageFactory.Create(HttpMethod.Get, request.Uri, log);

                    foreach (var acceptHeaderValue in request.AcceptHeaderValues)
                    {
                        requestMessage.Headers.Accept.Add(acceptHeaderValue);
                    }

                    return requestMessage;
                };

                Func <Task <ThrottledResponse> > throttledResponseFactory = () => GetThrottledResponse(
                    requestFactory,
                    request.RequestTimeout,
                    request.DownloadTimeout,
                    request.MaxTries,
                    log,
                    lockedToken);

                using (var throttledResponse = await throttledResponseFactory())
                {
                    if (request.IgnoreNotFounds && throttledResponse.Response.StatusCode == HttpStatusCode.NotFound)
                    {
                        var httpSourceResult = new HttpSourceResult(HttpSourceResultStatus.NotFound);

                        return await processAsync(httpSourceResult);
                    }

                    if (throttledResponse.Response.StatusCode == HttpStatusCode.NoContent)
                    {
                        // Ignore reading and caching the empty stream.
                        var httpSourceResult = new HttpSourceResult(HttpSourceResultStatus.NoContent);

                        return await processAsync(httpSourceResult);
                    }

                    throttledResponse.Response.EnsureSuccessStatusCode();

                    if (!request.CacheContext.DirectDownload)
                    {
                        await HttpCacheUtility.CreateCacheFileAsync(
                            cacheResult,
                            throttledResponse.Response,
                            request.EnsureValidContents,
                            lockedToken);

                        using (var httpSourceResult = new HttpSourceResult(
                                   HttpSourceResultStatus.OpenedFromDisk,
                                   cacheResult.CacheFile,
                                   cacheResult.Stream))
                        {
                            return await processAsync(httpSourceResult);
                        }
                    }
                    else
                    {
                        // Note that we do not execute the content validator on the response stream when skipping
                        // the cache. We cannot seek on the network stream and it is not valuable to download the
                        // content twice just to validate the first time (considering that the second download could
                        // be different from the first thus rendering the first validation meaningless).
                        using (var stream = await throttledResponse.Response.Content.ReadAsStreamAsync())
                            using (var httpSourceResult = new HttpSourceResult(
                                       HttpSourceResultStatus.OpenedFromNetwork,
                                       cacheFileName: null,
                                       stream: stream))
                            {
                                return await processAsync(httpSourceResult);
                            }
                    }
                }
            },
                       token : token));
        }
Beispiel #4
0
        internal async Task <XDocument> LoadXmlAsync(
            string uri,
            string cacheKey,
            bool ignoreNotFounds,
            SourceCacheContext sourceCacheContext,
            ILogger log,
            CancellationToken token)
        {
            if (cacheKey != null && sourceCacheContext != null)
            {
                var httpSourceCacheContext = HttpSourceCacheContext.Create(sourceCacheContext, 0);

                try
                {
                    return(await _httpSource.GetAsync(
                               new HttpSourceCachedRequest(
                                   uri,
                                   cacheKey,
                                   httpSourceCacheContext)
                    {
                        AcceptHeaderValues =
                        {
                            new MediaTypeWithQualityHeaderValue("application/atom+xml"),
                            new MediaTypeWithQualityHeaderValue("application/xml")
                        },
                        EnsureValidContents = stream => HttpStreamValidation.ValidateXml(uri, stream),
                        MaxTries = 1,
                        IgnoreNotFounds = ignoreNotFounds
                    },
                               async response =>
                    {
                        if (ignoreNotFounds && response.Status == HttpSourceResultStatus.NotFound)
                        {
                            // Treat "404 Not Found" as an empty response.
                            return null;
                        }
                        else if (response.Status == HttpSourceResultStatus.NoContent)
                        {
                            // Always treat "204 No Content" as exactly that.
                            return null;
                        }
                        else
                        {
                            return await LoadXmlAsync(response.Stream, token);
                        }
                    },
                               log,
                               token));
                }
                catch (Exception ex)
                {
                    var message = string.Format(
                        CultureInfo.CurrentCulture,
                        Strings.Log_FailedToFetchV2FeedHttp,
                        uri,
                        ex.Message);

                    throw new FatalProtocolException(message, ex);
                }
            }
            else
            {
                // return results without httpCache
                return(await _httpSource.ProcessResponseAsync(
                           new HttpSourceRequest(
                               () =>
                {
                    var request = HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log);
                    request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/atom+xml"));
                    request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
                    return request;
                })
                {
                    IsRetry = true
                },
                           async response =>
                {
                    if (response.StatusCode == HttpStatusCode.OK)
                    {
                        var networkStream = await response.Content.ReadAsStreamAsync();
                        return await LoadXmlAsync(networkStream, token);
                    }
                    else if (ignoreNotFounds && response.StatusCode == HttpStatusCode.NotFound)
                    {
                        // Treat "404 Not Found" as an empty response.
                        return null;
                    }
                    else if (response.StatusCode == HttpStatusCode.NoContent)
                    {
                        // Always treat "204 No Content" as exactly that.
                        return null;
                    }
                    else
                    {
                        throw new FatalProtocolException(string.Format(
                                                             CultureInfo.CurrentCulture,
                                                             Strings.Log_FailedToFetchV2Feed,
                                                             uri,
                                                             (int)response.StatusCode,
                                                             response.ReasonPhrase));
                    }
                },
                           sourceCacheContext,
                           log,
                           token));
            }
        }
Beispiel #5
0
 public HttpSourceRequest(Uri uri, ILogger log)
     : this(() => HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log))
 {
 }
Beispiel #6
0
        /// <summary>
        /// Caching Get request.
        /// </summary>
        public async Task <HttpSourceResult> GetAsync(
            HttpSourceCachedRequest request,
            ILogger log,
            CancellationToken token)
        {
            var result = HttpCacheUtility.InitializeHttpCacheResult(
                HttpCacheDirectory,
                _baseUri,
                request.CacheKey,
                request.CacheContext);

            return(await ConcurrencyUtilities.ExecuteWithFileLockedAsync(
                       result.CacheFile,
                       action : async lockedToken =>
            {
                result.Stream = TryReadCacheFile(request.Uri, result.MaxAge, result.CacheFile);

                if (result.Stream != null)
                {
                    log.LogInformation(string.Format(CultureInfo.InvariantCulture, "  " + Strings.Http_RequestLog, "CACHE", request.Uri));

                    // Validate the content fetched from the cache.
                    try
                    {
                        request.EnsureValidContents?.Invoke(result.Stream);

                        result.Stream.Seek(0, SeekOrigin.Begin);

                        return new HttpSourceResult(
                            HttpSourceResultStatus.OpenedFromDisk,
                            result.CacheFile,
                            result.Stream);
                    }
                    catch (Exception e)
                    {
                        result.Stream.Dispose();
                        result.Stream = null;

                        string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_InvalidCacheEntry, request.Uri)
                                         + Environment.NewLine
                                         + ExceptionUtilities.DisplayMessage(e);
                        log.LogWarning(message);
                    }
                }

                Func <HttpRequestMessage> requestFactory = () =>
                {
                    var requestMessage = HttpRequestMessageFactory.Create(HttpMethod.Get, request.Uri, log);

                    foreach (var acceptHeaderValue in request.AcceptHeaderValues)
                    {
                        requestMessage.Headers.Accept.Add(acceptHeaderValue);
                    }

                    return requestMessage;
                };

                Func <Task <ThrottledResponse> > throttledResponseFactory = () => GetThrottledResponse(
                    requestFactory,
                    request.RequestTimeout,
                    request.DownloadTimeout,
                    log,
                    lockedToken);

                using (var throttledResponse = await throttledResponseFactory())
                {
                    if (request.IgnoreNotFounds && throttledResponse.Response.StatusCode == HttpStatusCode.NotFound)
                    {
                        return new HttpSourceResult(HttpSourceResultStatus.NotFound);
                    }

                    if (throttledResponse.Response.StatusCode == HttpStatusCode.NoContent)
                    {
                        // Ignore reading and caching the empty stream.
                        return new HttpSourceResult(HttpSourceResultStatus.NoContent);
                    }

                    throttledResponse.Response.EnsureSuccessStatusCode();

                    await HttpCacheUtility.CreateCacheFileAsync(
                        result,
                        request.Uri,
                        throttledResponse.Response,
                        request.CacheContext,
                        request.EnsureValidContents,
                        lockedToken);

                    return new HttpSourceResult(
                        HttpSourceResultStatus.OpenedFromDisk,
                        result.CacheFile,
                        result.Stream);
                }
            },
                       token : token));
        }