public static HttpCacheResult InitializeHttpCacheResult(
string httpCacheDirectory,
Uri baseUri,
string cacheKey,
HttpSourceCacheContext context)
{
// When the MaxAge is TimeSpan.Zero, this means the caller is passing in a folder different than
// the global HTTP cache used by default. Additionally, creating and cleaning up the directory is
// all the responsibility of the caller.
var maxAge = context.MaxAge;
string newFile;
string cacheFile;
if (!maxAge.Equals(TimeSpan.Zero))
{
var baseFolderName = RemoveInvalidFileNameChars(ComputeHash(baseUri.OriginalString));
var baseFileName = RemoveInvalidFileNameChars(cacheKey) + ".dat";
var cacheFolder = Path.Combine(httpCacheDirectory, baseFolderName);
cacheFile = Path.Combine(cacheFolder, baseFileName);
newFile = cacheFile + "-new";
}
else
{
cacheFile = Path.Combine(context.RootTempFolder, Path.GetRandomFileName());
newFile = Path.Combine(context.RootTempFolder, Path.GetRandomFileName());
}
return(new HttpCacheResult(maxAge, newFile, cacheFile));
}
// Read the source's end point to get the index json.
// An exception will be thrown on failure.
private async Task <ServiceIndexResourceV3> GetServiceIndexResourceV3(
SourceRepository source,
DateTime utcNow,
ILogger log,
CancellationToken token)
{
var url = source.PackageSource.Source;
var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);
var client = httpSourceResource.HttpSource;
const int maxRetries = 3;
for (var retry = 1; retry <= maxRetries; retry++)
{
using (var sourceCacheContext = new SourceCacheContext())
{
var cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, isFirstAttempt: retry == 1);
try
{
return(await client.GetAsync(
new HttpSourceCachedRequest(
url,
"service_index",
cacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(url, stream),
MaxTries = 1,
IsRetry = retry > 1,
IsLastAttempt = retry == maxRetries
},
async httpSourceResult =>
{
var result = await ConsumeServiceIndexStreamAsync(httpSourceResult.Stream, utcNow, token);
return result;
},
log,
token));
}
catch (Exception ex) when(retry < maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingServiceIndex, url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
log.LogMinimal(message);
}
catch (Exception ex) when(retry == maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, url);
throw new FatalProtocolException(message, ex);
}
}
}
return(null);
}
/// <summary>
/// Query RegistrationIndex from nuget server for Package Manager UI. This implementation optimized for performance so instead of keeping giant JObject in memory we use strong types.
/// </summary>
/// <param name="httpSource">Httpsource instance</param>
/// <param name="registrationUri">Package registration url</param>
/// <param name="packageId">PackageId for package we're looking.</param>
/// <param name="cacheContext">CacheContext for cache.</param>
/// <param name="processAsync">Func expression used for HttpSource.cs</param>
/// <param name="log">Logger Instance.</param>
/// <param name="token">Cancellation token.</param>
/// <returns></returns>
private async Task <ValueTuple <RegistrationIndex, HttpSourceCacheContext> > LoadRegistrationIndexAsync(
HttpSource httpSource,
Uri registrationUri,
string packageId,
SourceCacheContext cacheContext,
Func <HttpSourceResult, Task <RegistrationIndex> > processAsync,
ILogger log,
CancellationToken token)
{
var packageIdLowerCase = packageId.ToLowerInvariant();
var retryCount = 0;
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, retryCount);
var index = await httpSource.GetAsync(
new HttpSourceCachedRequest(
registrationUri.OriginalString,
$"list_{packageIdLowerCase}_index",
httpSourceCacheContext)
{
IgnoreNotFounds = true,
},
async httpSourceResult => await processAsync(httpSourceResult),
log,
token);
return(new ValueTuple <RegistrationIndex, HttpSourceCacheContext>(index, httpSourceCacheContext));
}
public static HttpCacheResult InitializeHttpCacheResult(
string httpCacheDirectory,
Uri sourceUri,
string cacheKey,
HttpSourceCacheContext context)
{
// When the MaxAge is TimeSpan.Zero, this means the caller is passing is using a temporary directory for
// cache files, instead of the global HTTP cache used by default. Additionally, the cleaning up of this
// directory is the responsibility of the caller.
if (context.MaxAge > TimeSpan.Zero)
{
var baseFolderName = RemoveInvalidFileNameChars(ComputeHash(sourceUri.OriginalString));
var baseFileName = RemoveInvalidFileNameChars(cacheKey) + ".dat";
var cacheFolder = Path.Combine(httpCacheDirectory, baseFolderName);
var cacheFile = Path.Combine(cacheFolder, baseFileName);
var newCacheFile = cacheFile + "-new";
return(new HttpCacheResult(
context.MaxAge,
newCacheFile,
cacheFile));
}
else
{
var temporaryFile = Path.Combine(context.RootTempFolder, Path.GetRandomFileName());
var newTemporaryFile = Path.Combine(context.RootTempFolder, Path.GetRandomFileName());
return(new HttpCacheResult(
context.MaxAge,
newTemporaryFile,
temporaryFile));
}
}
public async Task WriteToCacheAsync(string cacheKey, string content)
{
var response = new HttpResponseMessage
{
Content = new StringContent(content, Encoding.UTF8)
};
using (response)
using (var sourceCacheContext = new SourceCacheContext())
{
var httpSourceCacheContext = HttpSourceCacheContext.Create(sourceCacheContext, retryCount: 0);
var result = HttpCacheUtility.InitializeHttpCacheResult(
TestDirectory,
new Uri(FakeSource),
cacheKey,
httpSourceCacheContext);
await HttpCacheUtility.CreateCacheFileAsync(
result,
response,
stream => { },
CancellationToken.None);
result.Stream.Dispose();
}
}
public static async Task CreateCacheFileAsync(
HttpCacheResult result,
string uri,
HttpResponseMessage response,
HttpSourceCacheContext context,
Action <Stream> ensureValidContents,
CancellationToken cancellationToken)
{
// The update of a cached file is divided into two steps:
// 1) Delete the old file.
// 2) Create a new file with the same name.
using (var fileStream = new FileStream(
result.NewCacheFile,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
BufferSize,
useAsync: true))
{
using (var networkStream = await response.Content.ReadAsStreamAsync())
{
await networkStream.CopyToAsync(fileStream, BufferSize, cancellationToken);
}
// Validate the content before putting it into the cache.
fileStream.Seek(0, SeekOrigin.Begin);
ensureValidContents?.Invoke(fileStream);
}
if (File.Exists(result.CacheFile))
{
// Process B can perform deletion on an opened file if the file is opened by process A
// with FileShare.Delete flag. However, the file won't be actually deleted until A close it.
// This special feature can cause race condition, so we never delete an opened file.
if (!IsFileAlreadyOpen(result.CacheFile))
{
File.Delete(result.CacheFile);
}
}
// If the destination file doesn't exist, we can safely perform moving operation.
// Otherwise, moving operation will fail.
if (!File.Exists(result.CacheFile))
{
File.Move(
result.NewCacheFile,
result.CacheFile);
}
// Even the file deletion operation above succeeds but the file is not actually deleted,
// we can still safely read it because it means that some other process just updated it
// and we don't need to update it with the same content again.
result.Stream = new FileStream(
result.CacheFile,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete,
BufferSize,
useAsync: true);
}
/// <summary>
/// Process RegistrationIndex
/// </summary>
/// <param name="httpSource">Httpsource instance.</param>
/// <param name="rangeUri">Paged registration index url address.</param>
/// <param name="packageId">PackageId for package we're checking.</param>
/// <param name="lower">Lower bound of nuget package.</param>
/// <param name="upper">Upper bound of nuget package.</param>
/// <param name="httpSourceCacheContext">SourceCacheContext for cache.</param>
/// <param name="log">Logger Instance.</param>
/// <param name="token">Cancellation token.</param>
/// <returns></returns>
private Task <RegistrationPage> GetRegistratioIndexPageAsync(
HttpSource httpSource,
string rangeUri,
string packageId,
NuGetVersion lower,
NuGetVersion upper,
HttpSourceCacheContext httpSourceCacheContext,
ILogger log,
CancellationToken token)
{
var packageIdLowerCase = packageId.ToLowerInvariant();
var registrationPage = httpSource.GetAsync(
new HttpSourceCachedRequest(
rangeUri,
$"list_{packageIdLowerCase}_range_{lower.ToNormalizedString()}-{upper.ToNormalizedString()}",
httpSourceCacheContext)
{
IgnoreNotFounds = true,
},
httpSourceResult => DeserializeStreamDataAsync <RegistrationPage>(httpSourceResult.Stream, token),
log,
token);
return(registrationPage);
}
private HttpCacheResult InitializeHttpCacheResult(PackageIdentity identity, SourceCacheContext sourceCacheContext)
{
return(HttpCacheUtility.InitializeHttpCacheResult(
NuGetExe.GetHttpCachePath(this),
new Uri(CurrentSource),
$"nupkg_{identity.Id}.{identity.Version}",
HttpSourceCacheContext.Create(sourceCacheContext, retryCount: 0)));
}
private async Task <RepositorySignatureResource> GetRepositorySignatureResourceAsync(
SourceRepository source,
string repoSignUrl,
ILogger log,
CancellationToken token)
{
var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);
var client = httpSourceResource.HttpSource;
for (var retry = 0; retry < 3; retry++)
{
using (var sourecCacheContext = new SourceCacheContext())
{
var cacheContext = HttpSourceCacheContext.Create(sourecCacheContext, retry);
try
{
return(await client.GetAsync(
new HttpSourceCachedRequest(
repoSignUrl,
"repository_signature",
cacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(repoSignUrl, stream),
MaxTries = 1
},
async httpSourceResult =>
{
var json = await httpSourceResult.Stream.AsJObjectAsync();
return new RepositorySignatureResource(json, source);
},
log,
token));
}
catch (Exception ex) when(retry < 2)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingRepositorySignature, repoSignUrl)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
log.LogMinimal(message);
}
catch (Exception ex) when(retry == 2)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadRepositorySignature, repoSignUrl);
throw new FatalProtocolException(message, ex);
}
}
}
return(null);
}
public HttpSourceCacheContext GetHttpCacheContext(int retryCount, bool directDownload = false)
{
// create http cache context from source cache instance
var baseCache = _sourceContext;
if (directDownload)
{
baseCache = _sourceContext.Clone();
baseCache.DirectDownload = directDownload;
}
return(HttpSourceCacheContext.Create(baseCache, retryCount));
}
// Read the source's end point to get the index json.
// An exception will be thrown on failure.
private async Task <ServiceIndexResourceV3> GetServiceIndexResourceV3(
SourceRepository source,
DateTime utcNow,
ILogger log,
CancellationToken token)
{
var url = source.PackageSource.Source;
var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);
var client = httpSourceResource.HttpSource;
for (var retry = 0; retry < 3; retry++)
{
var cacheContext = HttpSourceCacheContext.CreateCacheContext(new SourceCacheContext(), retry);
try
{
using (var sourceResponse = await client.GetAsync(
new HttpSourceCachedRequest(
url,
"service_index",
cacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(url, stream)
},
log,
token))
{
return(ConsumeServiceIndexStream(sourceResponse.Stream, utcNow));
}
}
catch (Exception ex) when(retry < 2)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingServiceIndex, url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
log.LogMinimal(message);
}
catch (Exception ex) when(retry == 2)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, url);
log.LogError(message + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex));
throw new FatalProtocolException(message, ex);
}
}
return(null);
}
public TestContext(TestDirectory testDirectory)
{
// data
var source = FakeSource;
CacheValidationException = new Exception();
NetworkValidationException = new Exception();
CacheContent = "cache";
NetworkContent = "network";
CacheKey = "CacheKey";
Url = "https://fake.server/foo/bar/something.json";
Credentials = new NetworkCredential("foo", "bar");
Throttle = new Mock <IThrottle>();
if (!RuntimeEnvironmentHelper.IsWindows)
{
_cacheDirectory = "c810bdb33f8c56015efcaf435f94766aa0c4748c$https:_fake.server_users.json";
}
else
{
// colon is not valid path character on Windows
_cacheDirectory = "c810bdb33f8c56015efcaf435f94766aa0c4748c$https_fake.server_users.json";
}
// dependencies
var packageSource = new PackageSource(source);
var networkResponses = new Dictionary <string, string> {
{ Url, NetworkContent }
};
var messageHandler = new TestMessageHandler(networkResponses, string.Empty);
var handlerResource = new TestHttpHandler(messageHandler);
CacheContext = new HttpSourceCacheContext();
Logger = new TestLogger();
TestDirectory = testDirectory;
RetryHandlerMock = new Mock <IHttpRetryHandler>();
// target
HttpSource = new HttpSource(packageSource, () => Task.FromResult((HttpHandlerResource)handlerResource), Throttle.Object)
{
HttpCacheDirectory = TestDirectory
};
}
/// <summary>
/// Caching Get request.
/// </summary>
public Task <HttpSourceResult> GetAsync(
string uri,
string cacheKey,
HttpSourceCacheContext cacheContext,
ILogger log,
bool ignoreNotFounds,
Action <Stream> ensureValidContents,
CancellationToken cancellationToken)
{
return(GetAsync(
uri,
new MediaTypeWithQualityHeaderValue[0],
cacheKey,
cacheContext,
log,
ignoreNotFounds,
ensureValidContents,
cancellationToken));
}
public HttpSourceCachedRequest(string uri, string cacheKey, HttpSourceCacheContext cacheContext)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (cacheKey == null)
{
throw new ArgumentNullException(nameof(cacheKey));
}
if (cacheContext == null)
{
throw new ArgumentNullException(nameof(cacheContext));
}
Uri = uri;
CacheKey = cacheKey;
CacheContext = cacheContext;
}
/// <summary>
/// CatalogReader
/// </summary>
/// <param name="indexUri">URI of the catalog service or the feed service index.</param>
/// <param name="httpSource">Custom HttpSource.</param>
public CatalogReader(Uri indexUri, HttpSource httpSource, SourceCacheContext cacheContext, TimeSpan cacheTimeout, ILogger log)
{
_indexUri = indexUri ?? throw new ArgumentNullException(nameof(indexUri));
_log = log ?? NullLogger.Instance;
_sourceCacheContext = cacheContext ?? new SourceCacheContext();
_httpSource = httpSource;
// TODO: what should retry be?
_cacheContext = HttpSourceCacheContext.Create(_sourceCacheContext, 5);
if (_sourceCacheContext == null)
{
var sourceCacheContext = new SourceCacheContext()
{
MaxAge = DateTimeOffset.UtcNow.Subtract(cacheTimeout),
};
_cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, 5);
}
}
internal static Task <NuspecReader> GetNuspecAsync(this HttpSource source, Uri uri, HttpSourceCacheContext cacheContext, ILogger log, CancellationToken token)
{
var cacheKey = GetHashKey(uri);
var request = new HttpSourceCachedRequest(uri.AbsoluteUri, cacheKey, cacheContext)
{
IgnoreNotFounds = false,
EnsureValidContents = stream =>
{
using (var reader = new StreamReader(stream, Encoding.UTF8, false, 8192, true))
{
XDocument.Load(reader);
}
}
};
return(source.GetAsync(request, ProcessNuspec, log, token));
}
internal static Task <JObject> GetJObjectAsync(this HttpSource source, Uri uri, HttpSourceCacheContext cacheContext, ILogger log, CancellationToken token)
{
var cacheKey = GetHashKey(uri);
var request = new HttpSourceCachedRequest(uri.AbsoluteUri, cacheKey, cacheContext)
{
EnsureValidContents = stream => CatalogReaderUtility.LoadJson(stream, true),
IgnoreNotFounds = false
};
return(source.GetAsync(request, ProcessJson, log, token));
}
internal async Task <XDocument> LoadXmlAsync(
string uri,
string cacheKey,
bool ignoreNotFounds,
SourceCacheContext sourceCacheContext,
ILogger log,
CancellationToken token)
{
if (cacheKey != null && sourceCacheContext != null)
{
var httpSourceCacheContext = HttpSourceCacheContext.Create(sourceCacheContext, 0);
try
{
return(await _httpSource.GetAsync(
new HttpSourceCachedRequest(
uri,
cacheKey,
httpSourceCacheContext)
{
AcceptHeaderValues =
{
new MediaTypeWithQualityHeaderValue("application/atom+xml"),
new MediaTypeWithQualityHeaderValue("application/xml")
},
EnsureValidContents = stream => HttpStreamValidation.ValidateXml(uri, stream),
MaxTries = 1,
IgnoreNotFounds = ignoreNotFounds
},
async response =>
{
if (ignoreNotFounds && response.Status == HttpSourceResultStatus.NotFound)
{
// Treat "404 Not Found" as an empty response.
return null;
}
else if (response.Status == HttpSourceResultStatus.NoContent)
{
// Always treat "204 No Content" as exactly that.
return null;
}
else
{
return await LoadXmlAsync(response.Stream, token);
}
},
log,
token));
}
catch (Exception ex)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToFetchV2FeedHttp,
uri,
ex.Message);
throw new FatalProtocolException(message, ex);
}
}
else
{
// return results without httpCache
return(await _httpSource.ProcessResponseAsync(
new HttpSourceRequest(
() =>
{
var request = HttpRequestMessageFactory.Create(HttpMethod.Get, uri, log);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/atom+xml"));
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
return request;
})
{
IsRetry = true
},
async response =>
{
if (response.StatusCode == HttpStatusCode.OK)
{
var networkStream = await response.Content.ReadAsStreamAsync();
return await LoadXmlAsync(networkStream, token);
}
else if (ignoreNotFounds && response.StatusCode == HttpStatusCode.NotFound)
{
// Treat "404 Not Found" as an empty response.
return null;
}
else if (response.StatusCode == HttpStatusCode.NoContent)
{
// Always treat "204 No Content" as exactly that.
return null;
}
else
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToFetchV2Feed,
uri,
(int)response.StatusCode,
response.ReasonPhrase));
}
},
sourceCacheContext,
log,
token));
}
}
private async Task <T> ProcessHttpSourceResultAsync <T>(
PackageIdentity identity,
string url,
Func <HttpSourceResult, Task <T> > processAsync,
SourceCacheContext cacheContext,
ILogger logger,
CancellationToken token)
{
for (var retry = 0; retry < 3; ++retry)
{
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, retry);
try
{
return(await _httpSource.GetAsync(
new HttpSourceCachedRequest(
url,
"nupkg_" + identity.Id.ToLowerInvariant() + "." + identity.Version.ToNormalizedString(),
httpSourceCacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateNupkg(url, stream),
IgnoreNotFounds = true,
MaxTries = 1
},
async httpSourceResult => await processAsync(httpSourceResult),
logger,
token));
}
catch (TaskCanceledException) when(retry < 2)
{
// Requests can get cancelled if we got the data from elsewhere, no reason to warn.
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_CanceledNupkgDownload, url);
logger.LogMinimal(message);
}
catch (Exception ex) when(retry < 2)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToDownloadPackage,
identity,
url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogMinimal(message);
}
catch (Exception ex) when(retry == 2)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToDownloadPackage,
identity,
url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogError(message);
}
}
return(await processAsync(null));
}
private async Task <SortedDictionary <NuGetVersion, PackageInfo> > FindPackagesByIdAsync(
string id,
SourceCacheContext cacheContext,
ILogger logger,
CancellationToken cancellationToken)
{
// Try each base URI 3 times.
var maxTries = 3 * _baseUris.Count;
var packageIdLowerCase = id.ToLowerInvariant();
for (var retry = 0; retry < maxTries; ++retry)
{
var baseUri = _baseUris[retry % _baseUris.Count].OriginalString;
var uri = baseUri + packageIdLowerCase + "/index.json";
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, retry);
try
{
return(await _httpSource.GetAsync(
new HttpSourceCachedRequest(
uri,
$"list_{packageIdLowerCase}",
httpSourceCacheContext)
{
IgnoreNotFounds = true,
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(uri, stream),
MaxTries = 1
},
async httpSourceResult =>
{
var result = new SortedDictionary <NuGetVersion, PackageInfo>();
if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromDisk)
{
try
{
result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken);
}
catch
{
logger.LogWarning(string.Format(CultureInfo.CurrentCulture, Strings.Log_FileIsCorrupt, httpSourceResult.CacheFile));
throw;
}
}
else if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromNetwork)
{
result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken);
}
return result;
},
logger,
cancellationToken));
}
catch (Exception ex) when(retry < 2)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsync), uri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogMinimal(message);
}
catch (Exception ex) when(retry == 2)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToRetrievePackage,
id,
uri);
throw new FatalProtocolException(message, ex);
}
}
return(null);
}
private async Task <IEnumerable <PackageInfo> > FindPackagesByIdAsyncCore(
string id,
SourceCacheContext cacheContext,
ILogger logger,
CancellationToken cancellationToken)
{
for (var retry = 0; retry < 3; ++retry)
{
var uri = _baseUri + "FindPackagesById()?id='" + id + "'";
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, retry);
try
{
var results = new List <PackageInfo>();
var uris = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
uris.Add(uri);
var page = 1;
var paging = true;
while (paging)
{
// TODO: Pages for a package ID are cached separately.
// So we will get inaccurate data when a page shrinks.
// However, (1) In most cases the pages grow rather than shrink;
// (2) cache for pages is valid for only 30 min.
// So we decide to leave current logic and observe.
paging = await _httpSource.GetAsync(
new HttpSourceCachedRequest(
uri,
$"list_{id.ToLowerInvariant()}_page{page}",
httpSourceCacheContext)
{
AcceptHeaderValues =
{
new MediaTypeWithQualityHeaderValue("application/atom+xml"),
new MediaTypeWithQualityHeaderValue("application/xml")
},
EnsureValidContents = stream => HttpStreamValidation.ValidateXml(uri, stream),
MaxTries = 1
},
async httpSourceResult =>
{
if (httpSourceResult.Status == HttpSourceResultStatus.NoContent)
{
// Team city returns 204 when no versions of the package exist
// This should result in an empty list and we should not try to
// read the stream as xml.
return(false);
}
var doc = await V2FeedParser.LoadXmlAsync(httpSourceResult.Stream);
var result = doc.Root
.Elements(_xnameEntry)
.Select(x => BuildModel(id, x))
.Where(x => x != null);
results.AddRange(result);
// Find the next url for continuation
var nextUri = V2FeedParser.GetNextUrl(doc);
// Stop if there's nothing else to GET
if (string.IsNullOrEmpty(nextUri))
{
return(false);
}
// check for any duplicate url and error out
if (!uris.Add(nextUri))
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.Protocol_duplicateUri,
nextUri));
}
uri = nextUri;
page++;
return(true);
},
logger,
cancellationToken);
}
return(results);
}
catch (Exception ex) when(retry < 2)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsyncCore), uri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogMinimal(message);
}
catch (Exception ex) when(retry == 2)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToRetrievePackage,
id,
uri);
throw new FatalProtocolException(message, ex);
}
}
return(null);
}
internal static Task <HttpSourceResult> GetNupkgAsync(this HttpSource source, Uri uri, HttpSourceCacheContext cacheContext, ILogger log, CancellationToken token)
{
var cacheKey = GetHashKey(uri);
var request = new HttpSourceCachedRequest(uri.AbsoluteUri, cacheKey, cacheContext)
{
IgnoreNotFounds = false,
EnsureValidContents = stream =>
{
using (var reader = new PackageArchiveReader(stream, leaveStreamOpen: true))
{
reader.NuspecReader.GetIdentity();
}
}
};
return(source.GetAsync(request, ProcessResult, log, token));
}
private async Task <T> ProcessHttpSourceResultAsync <T>(
PackageIdentity identity,
string url,
Func <HttpSourceResult, Task <T> > processAsync,
SourceCacheContext cacheContext,
ILogger logger,
CancellationToken token)
{
int maxRetries = _enhancedHttpRetryHelper.EnhancedHttpRetryEnabled ? _enhancedHttpRetryHelper.ExperimentalMaxNetworkTryCount : 3;
for (var retry = 1; retry <= maxRetries; ++retry)
{
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, isFirstAttempt: retry == 1);
try
{
return(await _httpSource.GetAsync(
new HttpSourceCachedRequest(
url,
"nupkg_" + identity.Id.ToLowerInvariant() + "." + identity.Version.ToNormalizedString(),
httpSourceCacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateNupkg(url, stream),
IgnoreNotFounds = true,
MaxTries = 1,
IsRetry = retry > 1,
IsLastAttempt = retry == maxRetries
},
async httpSourceResult => await processAsync(httpSourceResult),
logger,
token));
}
catch (TaskCanceledException) when(retry < maxRetries)
{
// Requests can get cancelled if we got the data from elsewhere, no reason to warn.
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_CanceledNupkgDownload, url);
logger.LogMinimal(message);
}
catch (Exception ex) when(retry < maxRetries)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToDownloadPackage,
identity,
url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogMinimal(message);
if (_enhancedHttpRetryHelper.EnhancedHttpRetryEnabled &&
ex.InnerException != null &&
ex.InnerException is IOException &&
ex.InnerException.InnerException != null &&
ex.InnerException.InnerException is System.Net.Sockets.SocketException)
{
// An IO Exception with inner SocketException indicates server hangup ("Connection reset by peer").
// Azure DevOps feeds sporadically do this due to mandatory connection cycling.
// Stalling an extra <ExperimentalRetryDelayMilliseconds> gives Azure more of a chance to recover.
logger.LogVerbose("Enhanced retry: Encountered SocketException, delaying between tries to allow recovery");
await Task.Delay(TimeSpan.FromMilliseconds(_enhancedHttpRetryHelper.ExperimentalRetryDelayMilliseconds));
}
}
catch (Exception ex) when(retry == maxRetries)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToDownloadPackage,
identity,
url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogError(message);
}
}
return(await processAsync(null));
}
private async Task <SortedDictionary <NuGetVersion, PackageInfo> > FindPackagesByIdAsync(
string id,
SourceCacheContext cacheContext,
ILogger logger,
CancellationToken cancellationToken)
{
// Try each base URI _maxRetries times.
var maxTries = _maxRetries * _baseUris.Count;
var packageIdLowerCase = id.ToLowerInvariant();
for (var retry = 1; retry <= maxTries; ++retry)
{
var baseUri = _baseUris[retry % _baseUris.Count].OriginalString;
var uri = baseUri + packageIdLowerCase + "/index.json";
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, isFirstAttempt: retry == 1);
try
{
return(await _httpSource.GetAsync(
new HttpSourceCachedRequest(
uri,
$"list_{packageIdLowerCase}",
httpSourceCacheContext)
{
IgnoreNotFounds = true,
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(uri, stream),
MaxTries = 1,
IsRetry = retry > 1,
IsLastAttempt = retry == maxTries
},
async httpSourceResult =>
{
var result = new SortedDictionary <NuGetVersion, PackageInfo>();
if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromDisk)
{
try
{
result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken);
}
catch
{
logger.LogWarning(string.Format(CultureInfo.CurrentCulture, Strings.Log_FileIsCorrupt, httpSourceResult.CacheFile));
throw;
}
}
else if (httpSourceResult.Status == HttpSourceResultStatus.OpenedFromNetwork)
{
result = await ConsumeFlatContainerIndexAsync(httpSourceResult.Stream, id, baseUri, cancellationToken);
}
return result;
},
logger,
cancellationToken));
}
catch (Exception ex) when(retry < _maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsync), uri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
logger.LogMinimal(message);
if (_enhancedHttpRetryHelper.IsEnabled &&
ex.InnerException != null &&
ex.InnerException is IOException &&
ex.InnerException.InnerException != null &&
ex.InnerException.InnerException is System.Net.Sockets.SocketException)
{
// An IO Exception with inner SocketException indicates server hangup ("Connection reset by peer").
// Azure DevOps feeds sporadically do this due to mandatory connection cycling.
// Stalling an extra <ExperimentalRetryDelayMilliseconds> gives Azure more of a chance to recover.
logger.LogVerbose("Enhanced retry: Encountered SocketException, delaying between tries to allow recovery");
await Task.Delay(TimeSpan.FromMilliseconds(_enhancedHttpRetryHelper.DelayInMilliseconds), cancellationToken);
}
}
catch (Exception ex) when(retry == _maxRetries)
{
var message = string.Format(
CultureInfo.CurrentCulture,
Strings.Log_FailedToRetrievePackage,
id,
uri);
throw new FatalProtocolException(message, ex);
}
}
return(null);
}
public HttpCacheResult InitializeHttpCacheResult(string httpCacheDirectory, Uri sourceUri, string cacheKey, HttpSourceCacheContext context)
{
return(HttpCacheUtility.InitializeHttpCacheResult(
httpCacheDirectory,
sourceUri,
cacheKey,
context));
}
public async static Task <IEnumerable <JObject> > LoadRanges(
HttpSource httpSource,
Uri registrationUri,
string packageId,
VersionRange range,
SourceCacheContext cacheContext,
ILogger log,
CancellationToken token)
{
var packageIdLowerCase = packageId.ToLowerInvariant();
var httpSourceCacheContext = HttpSourceCacheContext.Create(cacheContext, 0);
var index = await httpSource.GetAsync(
new HttpSourceCachedRequest(
registrationUri.OriginalString,
$"list_{packageIdLowerCase}_index",
httpSourceCacheContext)
{
IgnoreNotFounds = true,
},
async httpSourceResult =>
{
return(await httpSourceResult.Stream.AsJObjectAsync(token));
},
log,
token);
if (index == null)
{
// The server returned a 404, the package does not exist
return(Enumerable.Empty <JObject>());
}
IList <Task <JObject> > rangeTasks = new List <Task <JObject> >();
foreach (JObject item in index["items"])
{
var lower = NuGetVersion.Parse(item["lower"].ToString());
var upper = NuGetVersion.Parse(item["upper"].ToString());
if (range.DoesRangeSatisfy(lower, upper))
{
JToken items;
if (!item.TryGetValue("items", out items))
{
var rangeUri = item["@id"].ToString();
rangeTasks.Add(httpSource.GetAsync(
new HttpSourceCachedRequest(
rangeUri,
$"list_{packageIdLowerCase}_range_{lower.ToNormalizedString()}-{upper.ToNormalizedString()}",
httpSourceCacheContext)
{
IgnoreNotFounds = true,
},
async httpSourceResult =>
{
return(await httpSourceResult.Stream.AsJObjectAsync(token));
},
log,
token));
}
else
{
rangeTasks.Add(Task.FromResult(item));
}
}
}
await Task.WhenAll(rangeTasks.ToArray());
return(rangeTasks.Select((t) => t.Result));
}
/// <summary>
/// Read the source's end point to get the index json.
/// Retries are logged to any provided <paramref name="log"/> as LogMinimal.
/// </summary>
/// <param name="source"></param>
/// <param name="utcNow"></param>
/// <param name="log"></param>
/// <param name="token"></param>
/// <exception cref="OperationCanceledException">Logged to any provided <paramref name="log"/> as LogMinimal prior to throwing.</exception>
/// <exception cref="FatalProtocolException">Encapsulates all other exceptions.</exception>
/// <returns></returns>
private async Task <ServiceIndexResourceV3> GetServiceIndexResourceV3(
SourceRepository source,
DateTime utcNow,
ILogger log,
CancellationToken token)
{
var url = source.PackageSource.Source;
var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);
var client = httpSourceResource.HttpSource;
int maxRetries = _enhancedHttpRetryHelper.IsEnabled ? _enhancedHttpRetryHelper.RetryCount : 3;
for (var retry = 1; retry <= maxRetries; retry++)
{
using (var sourceCacheContext = new SourceCacheContext())
{
var cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, isFirstAttempt: retry == 1);
try
{
return(await client.GetAsync(
new HttpSourceCachedRequest(
url,
"service_index",
cacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(url, stream),
MaxTries = 1,
IsRetry = retry > 1,
IsLastAttempt = retry == maxRetries
},
async httpSourceResult =>
{
var result = await ConsumeServiceIndexStreamAsync(httpSourceResult.Stream, utcNow, token);
return result;
},
log,
token));
}
catch (OperationCanceledException ex)
{
var message = ExceptionUtilities.DisplayMessage(ex);
log.LogMinimal(message);
throw;
}
catch (Exception ex) when(retry < maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingServiceIndex, url)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
log.LogMinimal(message);
if (_enhancedHttpRetryHelper.IsEnabled &&
ex.InnerException != null &&
ex.InnerException is IOException &&
ex.InnerException.InnerException != null &&
ex.InnerException.InnerException is System.Net.Sockets.SocketException)
{
// An IO Exception with inner SocketException indicates server hangup ("Connection reset by peer").
// Azure DevOps feeds sporadically do this due to mandatory connection cycling.
// Stalling an extra <ExperimentalRetryDelayMilliseconds> gives Azure more of a chance to recover.
log.LogVerbose("Enhanced retry: Encountered SocketException, delaying between tries to allow recovery");
await Task.Delay(TimeSpan.FromMilliseconds(_enhancedHttpRetryHelper.DelayInMilliseconds), token);
}
}
catch (Exception ex) when(retry == maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadServiceIndex, url);
throw new FatalProtocolException(message, ex);
}
}
}
return(null);
}
private async Task <RepositorySignatureResource> GetRepositorySignatureResourceAsync(
SourceRepository source,
ServiceIndexEntry serviceEntry,
ILogger log,
CancellationToken token)
{
var repositorySignaturesResourceUri = serviceEntry.Uri;
if (repositorySignaturesResourceUri == null || !string.Equals(repositorySignaturesResourceUri.Scheme, "https", StringComparison.OrdinalIgnoreCase))
{
throw new FatalProtocolException(string.Format(CultureInfo.CurrentCulture, Strings.RepositorySignaturesResourceMustBeHttps, source.PackageSource.Source));
}
var httpSourceResource = await source.GetResourceAsync <HttpSourceResource>(token);
var client = httpSourceResource.HttpSource;
var cacheKey = GenerateCacheKey(serviceEntry);
const int maxRetries = 3;
for (var retry = 1; retry <= maxRetries; retry++)
{
using (var sourceCacheContext = new SourceCacheContext())
{
var cacheContext = HttpSourceCacheContext.Create(sourceCacheContext, isFirstAttempt: retry == 1);
try
{
return(await client.GetAsync(
new HttpSourceCachedRequest(
serviceEntry.Uri.AbsoluteUri,
cacheKey,
cacheContext)
{
EnsureValidContents = stream => HttpStreamValidation.ValidateJObject(repositorySignaturesResourceUri.AbsoluteUri, stream),
MaxTries = 1,
IsRetry = retry > 1,
IsLastAttempt = retry == maxRetries
},
async httpSourceResult =>
{
var json = await httpSourceResult.Stream.AsJObjectAsync(token);
return new RepositorySignatureResource(json, source);
},
log,
token));
}
catch (Exception ex) when(retry < maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingRepositorySignature, repositorySignaturesResourceUri.AbsoluteUri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(ex);
log.LogMinimal(message);
}
catch (Exception ex) when(retry == maxRetries)
{
var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_FailedToReadRepositorySignature, repositorySignaturesResourceUri.AbsoluteUri);
throw new FatalProtocolException(message, ex);
}
}
}
return(null);
}
/// <summary>
/// Caching Get request.
/// </summary>
public async Task <HttpSourceResult> GetAsync(
string uri,
MediaTypeWithQualityHeaderValue[] accept,
string cacheKey,
HttpSourceCacheContext cacheContext,
ILogger log,
bool ignoreNotFounds,
Action <Stream> ensureValidContents,
CancellationToken cancellationToken)
{
var result = InitializeHttpCacheResult(cacheKey, cacheContext);
return(await ConcurrencyUtilities.ExecuteWithFileLockedAsync(
result.CacheFile,
action : async token =>
{
result.Stream = TryReadCacheFile(uri, result.MaxAge, result.CacheFile);
if (result.Stream != null)
{
log.LogInformation(string.Format(CultureInfo.InvariantCulture, " " + Strings.Http_RequestLog, "CACHE", uri));
// Validate the content fetched from the cache.
try
{
ensureValidContents?.Invoke(result.Stream);
result.Stream.Seek(0, SeekOrigin.Begin);
return new HttpSourceResult(
HttpSourceResultStatus.OpenedFromDisk,
result.CacheFile,
result.Stream);
}
catch (Exception e)
{
result.Stream.Dispose();
result.Stream = null;
string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_InvalidCacheEntry, uri)
+ Environment.NewLine
+ ExceptionUtilities.DisplayMessage(e);
log.LogWarning(message);
}
}
Func <HttpRequestMessage> requestFactory = () =>
{
var request = new HttpRequestMessage(HttpMethod.Get, uri);
foreach (var a in accept)
{
request.Headers.Accept.Add(a);
}
return request;
};
// Read the response headers before reading the entire stream to avoid timeouts from large packages.
Func <Task <HttpResponseMessage> > httpRequest = () => SendWithCredentialSupportAsync(
requestFactory,
log,
token);
using (var response = await httpRequest())
{
if (ignoreNotFounds && response.StatusCode == HttpStatusCode.NotFound)
{
return new HttpSourceResult(HttpSourceResultStatus.NotFound);
}
if (response.StatusCode == HttpStatusCode.NoContent)
{
// Ignore reading and caching the empty stream.
return new HttpSourceResult(HttpSourceResultStatus.NoContent);
}
response.EnsureSuccessStatusCode();
await CreateCacheFileAsync(result, uri, response, cacheContext, ensureValidContents, token);
return new HttpSourceResult(
HttpSourceResultStatus.OpenedFromDisk,
result.CacheFile,
result.Stream);
}
},
token : cancellationToken));
}