/// <summary>
/// Retrieve the <see cref="RemoteSourceDependencyInfo" /> for a registration.
/// </summary>
/// <returns>Returns an empty sequence if the package does not exist.</returns>
public static async Task <IEnumerable <RemoteSourceDependencyInfo> > GetDependencies(
HttpSource httpClient,
Uri registrationUri,
VersionRange range,
ILogger log,
CancellationToken token)
{
var ranges = await Utils.LoadRanges(httpClient, registrationUri, range, log, token);
var results = new HashSet <RemoteSourceDependencyInfo>();
foreach (var rangeObj in ranges)
{
if (rangeObj == null)
{
throw new InvalidDataException(registrationUri.AbsoluteUri);
}
foreach (JObject packageObj in rangeObj["items"])
{
var catalogEntry = (JObject)packageObj["catalogEntry"];
var version = NuGetVersion.Parse(catalogEntry["version"].ToString());
if (range.Satisfies(version))
{
results.Add(ProcessPackageVersion(packageObj, version));
}
}
}
return(results);
}
public static async Task <bool> UrlExistsAsync(this HttpSource httpSource, string url, ILogger logger)
{
var nuGetLogger = logger.ToNuGetLogger();
return(await httpSource.ProcessResponseAsync(
new HttpSourceRequest(() => HttpRequestMessageFactory.Create(HttpMethod.Head, url, nuGetLogger))
{
IgnoreNotFounds = true,
},
response =>
{
if (response.StatusCode == HttpStatusCode.OK)
{
return Task.FromResult(true);
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
return Task.FromResult(false);
}
throw new HttpRequestException(
$"The request to {url} return HTTP {(int)response.StatusCode} {response.ReasonPhrase}.");
},
nuGetLogger,
CancellationToken.None));
}
public async Task V2FeedParser_DownloadFromIdentity()
{
// Arrange
var repo = Repository.Factory.GetCoreV3(TestServers.NuGetV2);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, TestServers.NuGetV2);
// Act & Assert
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
using (var downloadResult = await parser.DownloadFromIdentity(
new PackageIdentity("WindowsAzure.Storage", new NuGetVersion("6.2.0")),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(11, files.Count());
}
}
}
public NuGetPackagePusher(string pushUrl, string destinationApiKey, HttpSource httpSource, ILogger logger)
{
_pushUrl = pushUrl;
_destinationApiKey = destinationApiKey;
_httpSource = httpSource;
_logger = logger;
}
public NuGetPackagePusher(string pushUrl, string destinationApiKey, HttpSource httpSource, ILogger logger)
{
_pushUrl = pushUrl;
_destinationApiKey = destinationApiKey;
_httpSource = httpSource;
_logger = logger;
}
public NugetFeedReader(HttpSource httpSource, string baseAddress, string source)
{
_httpSource = httpSource ?? throw new ArgumentNullException(nameof(httpSource));
_baseAddress = baseAddress?.Trim('/') ?? throw new ArgumentNullException(nameof(baseAddress));
_queryBuilder = new V2FeedQueryBuilder();
Source = source ?? throw new ArgumentNullException(nameof(source));
}
public ParamWithId GetParameter(object templateParameter, string identificationCode)
{
WebSeed seed = JsonConvert.DeserializeObject <WebSeed>(templateParameter.ToString());
HttpSource httpSource = new HttpSource()
{
GenName = GenType,
Accept = seed.Accept,
ContentType = seed.ContentType,
Encoding = seed.Encoding,
Header = seed.Header,
Method = seed.Method,
Referer = seed.Referer,
Url = seed.SeedUrl,
UserAgent = seed.UserAgent,
Layer = seed.Depth
};
string hashid = (seed.SeedUrl + this.GenType + identificationCode).ToMD5Hex();
string recommendLoacation = $"level{seed.Depth}/" + hashid;
return(new ParamWithId()
{
Parameter = httpSource,
Id = hashid,
RecommendLocation = recommendLoacation,
SourceType = "Http"
});
}
public static async Task <T> DeserializeUrlAsync <T>(
this HttpSource httpSource,
string url,
bool ignoreNotFounds,
int maxTries,
ILogger logger)
{
var nuGetLogger = logger.ToNuGetLogger();
return(await httpSource.ProcessStreamAsync(
new HttpSourceRequest(url, nuGetLogger)
{
IgnoreNotFounds = ignoreNotFounds,
MaxTries = maxTries,
RequestTimeout = TimeSpan.FromSeconds(30),
},
stream =>
{
if (stream == null)
{
return Task.FromResult(default(T));
}
using (var textReader = new StreamReader(stream))
using (var jsonReader = new JsonTextReader(textReader))
{
var result = Serializer.Deserialize <T>(jsonReader);
return Task.FromResult(result);
}
},
nuGetLogger,
CancellationToken.None));
}
public async Task V2FeedParser_DownloadFromInvalidUrl()
{
// Arrange
var randomName = Guid.NewGuid().ToString();
var repo = Repository.Factory.GetCoreV3(TestServers.NuGetV2);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, TestServers.NuGetV2);
// Act
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
Exception ex = await Assert.ThrowsAsync <FatalProtocolException>(
async() => await parser.DownloadFromUrl(
new PackageIdentity("not-found", new NuGetVersion("6.2.0")),
new Uri($"https://www.{randomName}.org/api/v2/"),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None));
// Assert
Assert.NotNull(ex);
Assert.Equal($"Error downloading 'not-found.6.2.0' from 'https://www.{randomName}.org/api/v2/'.", ex.Message);
}
}
public async Task V2FeedParser_DownloadFromIdentityFromDifferentCredentialServer(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
// Act & Assert
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
using (var downloadResult = await parser.DownloadFromIdentity(
new PackageIdentity("newtonsoft.json", new NuGetVersion("8.0.3")),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(15, files.Count());
}
}
}
public TestContext(TestDirectory testDirectory)
{
// data
var source = FakeSource;
CacheValidationException = new Exception();
NetworkValidationException = new Exception();
CacheContent = "cache";
NetworkContent = "network";
CacheKey = "CacheKey";
Url = "https://fake.server/foo/bar/something.json";
Credentials = new NetworkCredential("foo", "bar");
Throttle = new Mock <IThrottle>();
// dependencies
var packageSource = new PackageSource(source);
var networkResponses = new Dictionary <string, string> {
{ Url, NetworkContent }
};
var messageHandler = new TestMessageHandler(networkResponses, string.Empty);
var handlerResource = new TestHttpHandler(messageHandler);
Logger = new TestLogger();
TestDirectory = testDirectory;
RetryHandlerMock = new Mock <IHttpRetryHandler>();
// target
HttpSource = new HttpSource(packageSource, () => Task.FromResult((HttpHandlerResource)handlerResource), Throttle.Object)
{
HttpCacheDirectory = TestDirectory
};
}
public async Task V2FeedParser_SearchFromCredentialServer(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
var searchFilter = new SearchFilter(includePrerelease: false)
{
SupportedFrameworks = new string[] { "net45" }
};
// Act
var packages = await parser.Search("nunit", searchFilter, 0, 1, NullLogger.Instance, CancellationToken.None);
var package = packages.FirstOrDefault();
// Assert
Assert.Equal("NUnit", package.Id);
}
public PackageRegistrationMetadataResourceV3(
RegistrationResourceV3 registration,
HttpSource client)
{
_registration = registration;
_client = client;
}
public async Task V2FeedParser_SearchWithPrereleaseCredentialServer(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
var searchFilter = new SearchFilter(includePrerelease: true)
{
SupportedFrameworks = new string[] { "net" }
};
// Act
var packages = await parser.Search("entityframework", searchFilter, 0, 3, NullLogger.Instance, CancellationToken.None);
var package = packages.Where(p => p.Id == "EntityFramework" && p.Version.ToString() == "7.0.0-beta4").FirstOrDefault();
// Assert
Assert.NotNull(package);
}
public async Task V2FeedParser_DownloadFromUrlInvalidId()
{
// Arrange
var repo = Repository.Factory.GetCoreV3(TestServers.NuGetV2);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, TestServers.NuGetV2);
// Act
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
var actual = await parser.DownloadFromUrl(
new PackageIdentity("not-found", new NuGetVersion("6.2.0")),
new Uri($@"{TestServers.NuGetV2}/package/not-found/6.2.0"),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None);
// Assert
Assert.NotNull(actual);
Assert.Equal(DownloadResourceResultStatus.NotFound, actual.Status);
}
}
private DownloadResourceV3(HttpSource client)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
_client = client;
}
public SearchClient(
HttpSource httpSource,
ISearchServiceUrlCacheInvalidator invalidator,
ILogger <SearchClient> logger)
{
_httpSource = httpSource;
_invalidator = invalidator;
_logger = logger;
}
/// <summary>
/// Download packages using the package base address container resource.
/// </summary>
public DownloadResourceV3(HttpSource client, string packageBaseAddress)
: this(client)
{
if (packageBaseAddress == null)
{
throw new ArgumentNullException(nameof(packageBaseAddress));
}
_packageBaseAddressUrl = packageBaseAddress.TrimEnd('/');
}
/// <summary>
/// Download packages using the download url found in the registration resource.
/// </summary>
public DownloadResourceV3(HttpSource client, RegistrationResourceV3 regResource)
: this(client)
{
if (regResource == null)
{
throw new ArgumentNullException(nameof(regResource));
}
_regResource = regResource;
}
public static Task <int> MainCore(string[] args, HttpSource httpSource, ILogger log)
{
#if DEBUG
if (args.Contains("--debug"))
{
args = args.Skip(1).ToArray();
while (!Debugger.IsAttached)
{
}
Debugger.Break();
}
#endif
var assemblyVersion = NuGetVersion.Parse(typeof(Program).GetTypeInfo().Assembly.GetName().Version.ToString());
var app = new CommandLineApplication()
{
Name = "NuGet.CatalogValidator",
FullName = "nuget mirror"
};
app.HelpOption(Constants.HelpOption);
app.VersionOption("--version", assemblyVersion.ToNormalizedString());
app.Description = "Validate a nuget v3 feed.";
Configure();
ValidateCommand.Register(app, httpSource, log);
app.OnExecute(() =>
{
app.ShowHelp();
return(0);
});
var exitCode = 0;
try
{
exitCode = app.Execute(args);
}
catch (CommandParsingException ex)
{
ex.Command.ShowHelp();
}
catch (Exception ex)
{
exitCode = 1;
LogException(ex, log);
}
return(Task.FromResult(exitCode));
}
/// <summary>
/// Retrieve a registration blob
/// </summary>
/// <returns>Returns Null if the package does not exist</returns>
public static async Task <RegistrationInfo> GetRegistrationInfo(
HttpSource httpClient,
Uri registrationUri,
VersionRange range,
NuGetFramework projectTargetFramework,
ILogger log,
CancellationToken token)
{
var frameworkComparer = new NuGetFrameworkFullComparer();
var frameworkReducer = new FrameworkReducer();
var dependencies = await GetDependencies(httpClient, registrationUri, range, log, token);
var result = new HashSet <RegistrationInfo>();
var registrationInfo = new RegistrationInfo();
registrationInfo.IncludePrerelease = true;
foreach (var item in dependencies)
{
var packageInfo = new PackageInfo
{
Listed = item.Listed,
Version = item.Identity.Version,
PackageContent = new Uri(item.ContentUri)
};
// only one target framework group will be used at install time, which means
// we can filter down to that group now by using the project target framework
var depFrameworks = item.DependencyGroups.Select(e => e.TargetFramework);
var targetFramework = frameworkReducer.GetNearest(projectTargetFramework, depFrameworks);
// If no frameworks are compatible we just ignore them - Should this be an exception?
if (targetFramework != null)
{
var dependencyGroup = item.DependencyGroups.FirstOrDefault(d => frameworkComparer.Equals(targetFramework, d.TargetFramework));
if (dependencyGroup != null)
{
foreach (var dependency in dependencyGroup.Packages)
{
var dependencyInfo = new DependencyInfo
{
Id = dependency.Id,
Range = dependency.VersionRange
};
packageInfo.Dependencies.Add(dependencyInfo);
}
}
}
registrationInfo.Add(packageInfo);
registrationInfo.Id = item.Identity.Id;
}
return(registrationInfo);
}
public static Task <T> DeserializeUrlAsync <T>(
this HttpSource httpSource,
string url,
bool ignoreNotFounds,
ILogger logger)
{
return(httpSource.DeserializeUrlAsync <T>(
url,
ignoreNotFounds,
maxTries: 3,
logger: logger));
}
internal static Task <JObject> GetJObjectAsync(this HttpSource source, Uri uri, HttpSourceCacheContext cacheContext, ILogger log, CancellationToken token)
{
var cacheKey = GetHashKey(uri);
var request = new HttpSourceCachedRequest(uri.AbsoluteUri, cacheKey, cacheContext)
{
EnsureValidContents = stream => CatalogReaderUtility.LoadJson(stream, true),
IgnoreNotFounds = false
};
return(source.GetAsync(request, ProcessJson, log, token));
}
private FindPackageByIdResource GetFindPackageByIdResource(string source)
{
if (source.Contains("api/v2"))
{
var repository = factory.GetCoreV2(new PackageSource(source));
return(new RemoteV2FindPackageByIdResource(repository.PackageSource, HttpSource.Create(repository)));
}
else
{
var repository = factory.GetCoreV3(source);
return(new RemoteV3FindPackageByIdResource(repository, HttpSource.Create(repository)));
}
}
public RemoteCursorService(
ServiceIndexCache serviceIndexCache,
HttpSource httpSource,
SearchServiceCursorReader searchServiceCursorReader,
CursorService cursorService,
ILogger <RemoteCursorService> logger)
{
_serviceIndexCache = serviceIndexCache;
_httpSource = httpSource;
_searchServiceCursorReader = searchServiceCursorReader;
_cursorService = cursorService;
_logger = logger;
}
public NuspecStore(
IFileStorageService fileStorageService,
ServiceIndexCache serviceIndexCache,
FlatContainerClient flatContainerClient,
HttpSource httpSource,
ILogger <NuspecStore> logger)
{
_fileStorageService = fileStorageService;
_serviceIndexCache = serviceIndexCache;
_flatContainerClient = flatContainerClient;
_httpSource = httpSource;
_logger = logger;
}
public override Task <Tuple <bool, INuGetResource> > TryCreate(SourceRepository source, CancellationToken token)
{
DownloadResource resource = null;
if ((FeedTypeUtility.GetFeedType(source.PackageSource) & FeedType.HttpV2) != FeedType.None)
{
var httpSource = HttpSource.Create(source);
var parser = new V2FeedParser(httpSource, source.PackageSource);
resource = new DownloadResourceV2FeedPrivate(parser, httpSource);
}
return(Task.FromResult(new Tuple <bool, INuGetResource>(resource != null, resource)));
}
public RegistrationResourceV3(HttpSource client, Uri baseUrl)
{
if (client == null)
{
throw new ArgumentNullException("client");
}
if (baseUrl == null)
{
throw new ArgumentNullException("baseUrl");
}
_client = client;
BaseUri = baseUrl;
}
public async Task V2FeedParser_NormalizedVersion(string packageSource)
{
// Arrange
var repo = Repository.Factory.GetCoreV3(packageSource);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
// Act
var package = await parser.GetPackage(new PackageIdentity("owin", new NuGetVersion("1.0")), NullLogger.Instance, CancellationToken.None);
// Assert
Assert.Equal("Owin", package.Id);
Assert.Equal("1.0", package.Version.ToString());
}
public void HttpSourceTest()
{
ScrapySource scrapySource = JsonConvert.DeserializeObject <ScrapySource>(httpSourceDemoString);
HttpSource httpSource = JsonConvert.DeserializeObject <HttpSource>(scrapySource.Source.Parameters.ToString());
Assert.NotNull(httpSource);
Assert.Equal("http://www.sina.com.cn", httpSource.Referer);
Assert.Equal("https://news.sina.com.cn/c/2019-10-27/doc-iicezzrr5215576.shtml?cre=tianyi&mod=pchp&loc=10&r=0&rfunc=91&tj=none&tr=12", httpSource.Url);
Assert.Equal("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", httpSource.Accept);
Assert.Equal("Chrome_1", httpSource.UserAgent);
Assert.Equal("text/html; charset=utf-8", httpSource.ContentType);
Assert.Equal("utf-8", httpSource.Encoding);
Assert.Equal("GET", httpSource.Method);
Assert.Equal(2, httpSource.Header.Count);
}
public RawSearchResourceV3(HttpSource client, IEnumerable <Uri> searchEndpoints)
: base()
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
if (searchEndpoints == null)
{
throw new ArgumentNullException("searchEndpoints");
}
_client = client;
_searchEndpoints = searchEndpoints.ToArray();
}
public VsixPackageEnumerator(string source, HttpSource httpSource, ILogger logger)
{
_source = source;
_httpSource = httpSource;
_logger = logger;
}
