// Read the source's end point to get the index json. // An exception will be thrown on failure. private async Task <ServiceIndexResourceV3> GetServiceIndexResourceV3( SourceRepository source, DateTime utcNow, ILogger log, CancellationToken token) { var url = source.PackageSource.Source; var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token); var client = httpSourceResource.HttpSource; const int maxRetries = 3; for (var retry = 1; retry <= maxRetries; retry++) { using (var sourceCacheContext = new SourceCacheContext()) { var cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, isFirstAttempt: retry == 1); try { return(await client.GetAsync( new HttpSourceCachedRequest( url, "service_index", cacheContext) { EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(url, stream), MaxTries = 1, IsRetry = retry > 1, IsLastAttempt = retry == maxRetries }, async httpSourceResult => { var result = await ConsumeServiceIndexStreamAsync(httpSourceResult.Stream, utcNow, token); return result; }, log, token)); } catch (Exception ex) when(retry < maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingServiceIndex, url) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); log.LogMinimal(message); } catch (Exception ex) when(retry == maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, url); throw new FatalProtocolException(message, ex); } } } return(null); }
private async Task <SortedDictionary <NuGetVersion, PackageInfo> > FindPackagesByIdAsync(string id, CancellationToken cancellationToken) { for (var retry = 0; retry != 3; ++retry) { var baseUri = _baseUris[retry % _baseUris.Count].OriginalString; var uri = baseUri + id.ToLowerInvariant() + "/index.json"; try { using (var result = await _httpSource.GetAsync( new HttpSourceCachedRequest( uri, $"list_{id}", CreateCacheContext(retry)) { IgnoreNotFounds = true, EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(uri, stream) }, Logger, cancellationToken)) { if (result.Status == HttpSourceResultStatus.NotFound) { return(new SortedDictionary <NuGetVersion, PackageInfo>()); } try { return(ConsumeFlatContainerIndex(result.Stream, id, baseUri)); } catch { Logger.LogWarning(string.Format(CultureInfo.CurrentCulture, Strings.Log_FileIsCorrupt, result.CacheFileName)); throw; } } } catch (Exception ex) when(retry < 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsync), uri) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); Logger.LogMinimal(message); } catch (Exception ex) when(retry == 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToRetrievePackage, uri); Logger.LogError(message + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex)); throw new FatalProtocolException(message, ex); } } return(null); }
private async Task <RepositorySignatureResource> GetRepositorySignatureResourceAsync( SourceRepository source, string repoSignUrl, ILogger log, CancellationToken token) { var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token); var client = httpSourceResource.HttpSource; for (var retry = 0; retry < 3; retry++) { using (var sourecCacheContext = new SourceCacheContext()) { var cacheContext = HttpSourceCacheContext.Create(sourecCacheContext, retry); try { return(await client.GetAsync( new HttpSourceCachedRequest( repoSignUrl, "repository_signature", cacheContext) { EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(repoSignUrl, stream), MaxTries = 1 }, async httpSourceResult => { var json = await httpSourceResult.Stream.AsJObjectAsync(); return new RepositorySignatureResource(json, source); }, log, token)); } catch (Exception ex) when(retry < 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingRepositorySignature, repoSignUrl) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); log.LogMinimal(message); } catch (Exception ex) when(retry == 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadRepositorySignature, repoSignUrl); throw new FatalProtocolException(message, ex); } } } return(null); }
// Read the source's end point to get the index json. // An exception will be thrown on failure. private async Task <ServiceIndexResourceV3> GetServiceIndexResourceV3( SourceRepository source, DateTime utcNow, ILogger log, CancellationToken token) { var url = source.PackageSource.Source; var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token); var client = httpSourceResource.HttpSource; for (var retry = 0; retry < 3; retry++) { var cacheContext = HttpSourceCacheContext.CreateCacheContext(new SourceCacheContext(), retry); try { using (var sourceResponse = await client.GetAsync( new HttpSourceCachedRequest( url, "service_index", cacheContext) { EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(url, stream) }, log, token)) { return(ConsumeServiceIndexStream(sourceResponse.Stream, utcNow)); } } catch (Exception ex) when(retry < 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingServiceIndex, url) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); log.LogMinimal(message); } catch (Exception ex) when(retry == 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, url); log.LogError(message + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex)); throw new FatalProtocolException(message, ex); } } return(null); }
private async Task <RepositorySignatureResource> GetRepositorySignatureResourceAsync( SourceRepository source, ServiceIndexEntry serviceEntry, ILogger log, CancellationToken token) { var repositorySignaturesResourceUri = serviceEntry.Uri; if (repositorySignaturesResourceUri == null || !string.Equals(repositorySignaturesResourceUri.Scheme, "https", StringComparison.OrdinalIgnoreCase)) { throw new FatalProtocolException(string.Format(CultureInfo.CurrentCulture, Strings.RepositorySignaturesResourceMustBeHttps, source.PackageSource.Source)); } var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token); var client = httpSourceResource.HttpSource; var cacheKey = GenerateCacheKey(serviceEntry); const int maxRetries = 3; for (var retry = 1; retry <= maxRetries; retry++) { using (var sourceCacheContext = new SourceCacheContext()) { var cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, isFirstAttempt: retry == 1); try { return(await client.GetAsync( new HttpSourceCachedRequest( serviceEntry.Uri.AbsoluteUri, cacheKey, cacheContext) { EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(repositorySignaturesResourceUri.AbsoluteUri, stream), MaxTries = 1, IsRetry = retry > 1, IsLastAttempt = retry == maxRetries }, async httpSourceResult => { var json = await httpSourceResult.Stream.AsJObjectAsync(token); return new RepositorySignatureResource(json, source); }, log, token)); } catch (Exception ex) when(retry < maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingRepositorySignature, repositorySignaturesResourceUri.AbsoluteUri) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); log.LogMinimal(message); } catch (Exception ex) when(retry == maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadRepositorySignature, repositorySignaturesResourceUri.AbsoluteUri); throw new FatalProtocolException(message, ex); } } } return(null); }
private async Task <SortedDictionary <NuGetVersion, PackageInfo> > FindPackagesByIdAsync( string id, SourceCacheContext cacheContext, ILogger logger, CancellationToken cancellationToken) { // Try each base URI 3 times. var maxTries = 3 * _baseUris.Count; var packageIdLowerCase = id.ToLowerInvariant(); for (var retry = 0; retry < maxTries; ++retry) { var baseUri = _baseUris[retry % _baseUris.Count].OriginalString; var uri = baseUri + packageIdLowerCase + "/index.json"; var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, retry); try { return(await _httpSource.GetAsync( new HttpSourceCachedRequest( uri, $"list_{packageIdLowerCase}", httpSourceCacheContext) { IgnoreNotFounds = true, EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(uri, stream), MaxTries = 1 }, async httpSourceResult => { var result = new SortedDictionary <NuGetVersion, PackageInfo>(); if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromDisk) { try { result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken); } catch { logger.LogWarning(string.Format(CultureInfo.CurrentCulture, Strings.Log_FileIsCorrupt, httpSourceResult.CacheFile)); throw; } } else if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromNetwork) { result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken); } return result; }, logger, cancellationToken)); } catch (Exception ex) when(retry < 2) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsync), uri) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); logger.LogMinimal(message); } catch (Exception ex) when(retry == 2) { var message = string.Format( CultureInfo.CurrentCulture, Strings.Log_FailedToRetrievePackage, id, uri); throw new FatalProtocolException(message, ex); } } return(null); }
/// <summary> /// Read the source's end point to get the index json. /// Retries are logged to any provided <paramref name="log"/> as LogMinimal. /// </summary> /// <param name="source"></param> /// <param name="utcNow"></param> /// <param name="log"></param> /// <param name="token"></param> /// <exception cref="OperationCanceledException">Logged to any provided <paramref name="log"/> as LogMinimal prior to throwing.</exception> /// <exception cref="FatalProtocolException">Encapsulates all other exceptions.</exception> /// <returns></returns> private async Task <ServiceIndexResourceV3> GetServiceIndexResourceV3( SourceRepository source, DateTime utcNow, ILogger log, CancellationToken token) { var url = source.PackageSource.Source; var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token); var client = httpSourceResource.HttpSource; int maxRetries = _enhancedHttpRetryHelper.IsEnabled ? _enhancedHttpRetryHelper.RetryCount : 3; for (var retry = 1; retry <= maxRetries; retry++) { using (var sourceCacheContext = new SourceCacheContext()) { var cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, isFirstAttempt: retry == 1); try { return(await client.GetAsync( new HttpSourceCachedRequest( url, "service_index", cacheContext) { EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(url, stream), MaxTries = 1, IsRetry = retry > 1, IsLastAttempt = retry == maxRetries }, async httpSourceResult => { var result = await ConsumeServiceIndexStreamAsync(httpSourceResult.Stream, utcNow, token); return result; }, log, token)); } catch (OperationCanceledException ex) { var message = ExceptionUtilities.DisplayMessage(ex); log.LogMinimal(message); throw; } catch (Exception ex) when(retry < maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingServiceIndex, url) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); log.LogMinimal(message); if (_enhancedHttpRetryHelper.IsEnabled && ex.InnerException != null && ex.InnerException is IOException && ex.InnerException.InnerException != null && ex.InnerException.InnerException is System.Net.Sockets.SocketException) { // An IO Exception with inner SocketException indicates server hangup ("Connection reset by peer"). // Azure DevOps feeds sporadically do this due to mandatory connection cycling. // Stalling an extra <ExperimentalRetryDelayMilliseconds> gives Azure more of a chance to recover. log.LogVerbose("Enhanced retry: Encountered SocketException, delaying between tries to allow recovery"); await Task.Delay(TimeSpan.FromMilliseconds(_enhancedHttpRetryHelper.DelayInMilliseconds), token); } } catch (Exception ex) when(retry == maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, url); throw new FatalProtocolException(message, ex); } } } return(null); }
private async Task <SortedDictionary <NuGetVersion, PackageInfo> > FindPackagesByIdAsync( string id, SourceCacheContext cacheContext, ILogger logger, CancellationToken cancellationToken) { // Try each base URI _maxRetries times. var maxTries = _maxRetries * _baseUris.Count; var packageIdLowerCase = id.ToLowerInvariant(); for (var retry = 1; retry <= maxTries; ++retry) { var baseUri = _baseUris[retry % _baseUris.Count].OriginalString; var uri = baseUri + packageIdLowerCase + "/index.json"; var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, isFirstAttempt: retry == 1); try { return(await _httpSource.GetAsync( new HttpSourceCachedRequest( uri, $"list_{packageIdLowerCase}", httpSourceCacheContext) { IgnoreNotFounds = true, EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(uri, stream), MaxTries = 1, IsRetry = retry > 1, IsLastAttempt = retry == maxTries }, async httpSourceResult => { var result = new SortedDictionary <NuGetVersion, PackageInfo>(); if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromDisk) { try { result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken); } catch { logger.LogWarning(string.Format(CultureInfo.CurrentCulture, Strings.Log_FileIsCorrupt, httpSourceResult.CacheFile)); throw; } } else if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromNetwork) { result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken); } return result; }, logger, cancellationToken)); } catch (Exception ex) when(retry < _maxRetries) { var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsync), uri) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); logger.LogMinimal(message); if (_enhancedHttpRetryHelper.IsEnabled && ex.InnerException != null && ex.InnerException is IOException && ex.InnerException.InnerException != null && ex.InnerException.InnerException is System.Net.Sockets.SocketException) { // An IO Exception with inner SocketException indicates server hangup ("Connection reset by peer"). // Azure DevOps feeds sporadically do this due to mandatory connection cycling. // Stalling an extra <ExperimentalRetryDelayMilliseconds> gives Azure more of a chance to recover. logger.LogVerbose("Enhanced retry: Encountered SocketException, delaying between tries to allow recovery"); await Task.Delay(TimeSpan.FromMilliseconds(_enhancedHttpRetryHelper.DelayInMilliseconds), cancellationToken); } } catch (Exception ex) when(retry == _maxRetries) { var message = string.Format( CultureInfo.CurrentCulture, Strings.Log_FailedToRetrievePackage, id, uri); throw new FatalProtocolException(message, ex); } } return(null); }