public async Task V2FeedParser_DownloadFromIdentityFromDifferentCredentialServer(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
// Act & Assert
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
using (var downloadResult = await parser.DownloadFromIdentity(
new PackageIdentity("newtonsoft.json", new NuGetVersion("8.0.3")),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(15, files.Count());
}
}
}
public async Task V2FeedParser_DownloadFromInvalidUrl()
{
// Arrange
var randomName = Guid.NewGuid().ToString();
var repo = Repository.Factory.GetCoreV3(TestServers.NuGetV2);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, TestServers.NuGetV2);
// Act
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
Exception ex = await Assert.ThrowsAsync <FatalProtocolException>(
async() => await parser.DownloadFromUrl(
new PackageIdentity("not-found", new NuGetVersion("6.2.0")),
new Uri($"https://www.{randomName}.org/api/v2/"),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None));
// Assert
Assert.NotNull(ex);
Assert.Equal($"Error downloading 'not-found.6.2.0' from 'https://www.{randomName}.org/api/v2/'.", ex.Message);
}
}
public async Task V2FeedParser_SearchWithPrereleaseCredentialServer(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
var searchFilter = new SearchFilter(includePrerelease: true)
{
SupportedFrameworks = new string[] { "net" }
};
// Act
var packages = await parser.Search("entityframework", searchFilter, 0, 3, NullLogger.Instance, CancellationToken.None);
var package = packages.Where(p => p.Id == "EntityFramework" && p.Version.ToString() == "7.0.0-beta4").FirstOrDefault();
// Assert
Assert.NotNull(package);
}
public async Task V2FeedParser_SearchFromCredentialServer(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
var searchFilter = new SearchFilter(includePrerelease: false)
{
SupportedFrameworks = new string[] { "net45" }
};
// Act
var packages = await parser.Search("nunit", searchFilter, 0, 1, NullLogger.Instance, CancellationToken.None);
var package = packages.FirstOrDefault();
// Assert
Assert.Equal("NUnit", package.Id);
}
public async Task V2FeedParser_DownloadFromUrlInvalidId()
{
// Arrange
var repo = Repository.Factory.GetCoreV3(TestServers.NuGetV2);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, TestServers.NuGetV2);
// Act
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
var actual = await parser.DownloadFromUrl(
new PackageIdentity("not-found", new NuGetVersion("6.2.0")),
new Uri($@"{TestServers.NuGetV2}/package/not-found/6.2.0"),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None);
// Assert
Assert.NotNull(actual);
Assert.Equal(DownloadResourceResultStatus.NotFound, actual.Status);
}
}
public async Task V2FeedParser_DownloadFromIdentity()
{
// Arrange
var repo = Repository.Factory.GetCoreV3(TestServers.NuGetV2);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, TestServers.NuGetV2);
// Act & Assert
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
using (var downloadResult = await parser.DownloadFromIdentity(
new PackageIdentity("WindowsAzure.Storage", new NuGetVersion("6.2.0")),
new PackageDownloadContext(cacheContext),
packagesFolder,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(11, files.Count());
}
}
}
private FindPackageByIdResource GetFindPackageByIdResource(string source)
{
if (source.Contains("api/v2"))
{
var repository = factory.GetCoreV2(new PackageSource(source));
return(new RemoteV2FindPackageByIdResource(repository.PackageSource, HttpSource.Create(repository)));
}
else
{
var repository = factory.GetCoreV3(source);
return(new RemoteV3FindPackageByIdResource(repository, HttpSource.Create(repository)));
}
}
public override Task <Tuple <bool, INuGetResource> > TryCreate(SourceRepository source, CancellationToken token)
{
DownloadResource resource = null;
if ((FeedTypeUtility.GetFeedType(source.PackageSource) & FeedType.HttpV2) != FeedType.None)
{
var httpSource = HttpSource.Create(source);
var parser = new V2FeedParser(httpSource, source.PackageSource);
resource = new DownloadResourceV2FeedPrivate(parser, httpSource);
}
return(Task.FromResult(new Tuple <bool, INuGetResource>(resource != null, resource)));
}
public async Task V2FeedParser_NormalizedVersion(string packageSource)
{
// Arrange
var repo = Repository.Factory.GetCoreV3(packageSource);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
// Act
var package = await parser.GetPackage(new PackageIdentity("owin", new NuGetVersion("1.0")), NullLogger.Instance, CancellationToken.None);
// Assert
Assert.Equal("Owin", package.Id);
Assert.Equal("1.0", package.Version.ToString());
}
public override Task <Tuple <bool, INuGetResource> > TryCreate(SourceRepository source, CancellationToken token)
{
HttpSourceResource curResource = null;
if (source.PackageSource.IsHttp)
{
curResource = _cache.GetOrAdd(
source.PackageSource,
packageSource => new HttpSourceResource(HttpSource.Create(
source,
NullThrottle.Instance,
_httpCacheUtility,
_concurrencyUtility)));
}
return(Task.FromResult(new Tuple <bool, INuGetResource>(curResource != null, curResource)));
}
public async Task V2FeedParser_CredentialNormalizedVersion(string packageSource, string feedName)
{
// Arrange
var credential = Utility.ReadCredential(feedName);
var source = new PackageSource(packageSource);
var sourceCredential = new PackageSourceCredential(packageSource, credential.Item1, credential.Item2, true);
source.Credentials = sourceCredential;
var repo = Repository.Factory.GetCoreV2(source);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
// Act
var package = await parser.GetPackage(new PackageIdentity("owin", new NuGetVersion("1.0")), NullLogger.Instance, CancellationToken.None);
// Assert
Assert.Equal("Owin", package.Id);
Assert.Equal("1.0", package.Version.ToString());
}
public async Task V2FeedParser_DownloadFromIdentityFromDifferentServer(string packageSource)
{
// Arrange
var repo = Repository.Factory.GetCoreV3(packageSource);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
// Act & Assert
using (var downloadResult = await parser.DownloadFromIdentity(new PackageIdentity("newtonsoft.json", new NuGetVersion("8.0.3")),
Configuration.NullSettings.Instance,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(15, files.Count());
}
}
public async Task V2FeedParser_DownloadFromUrl()
{
// Arrange
var repo = Repository.Factory.GetCoreV3("https://www.nuget.org/api/v2/");
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, "https://www.nuget.org/api/v2/");
// Act & Assert
using (var downloadResult = await parser.DownloadFromUrl(new PackageIdentity("WindowsAzure.Storage", new NuGetVersion("6.2.0")),
new Uri("https://www.nuget.org/api/v2/package/WindowsAzure.Storage/6.2.0"),
Configuration.NullSettings.Instance,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(11, files.Count());
}
}
public async Task V2FeedParser_Search(string packageSource)
{
// Arrange
var repo = Repository.Factory.GetCoreV3(packageSource);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
var searchFilter = new SearchFilter(includePrerelease: false)
{
SupportedFrameworks = new string[] { "net45" }
};
// Act
var packages = await parser.Search("nunit", searchFilter, 0, 1, NullLogger.Instance, CancellationToken.None);
var package = packages.FirstOrDefault();
// Assert
Assert.Equal("NUnit", package.Id);
}
public async Task V2FeedParser_DownloadFromIdentity()
{
// Arrange
var repo = Repository.Factory.GetCoreV3(TestSources.NuGetV2Uri);
var httpSource = HttpSource.Create(repo);
var parser = new V2FeedParser(httpSource, TestSources.NuGetV2Uri);
// Act & Assert
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
var downloadContext = new PackageDownloadContext(cacheContext)
{
ExtractionContext = new PackageExtractionContext(
PackageSaveMode.Defaultv3,
PackageExtractionBehavior.XmlDocFileSaveMode,
NullLogger.Instance,
signedPackageVerifier: null,
signedPackageVerifierSettings: null)
};
using (var downloadResult = await parser.DownloadFromIdentity(
new PackageIdentity("WindowsAzure.Storage", new NuGetVersion("6.2.0")),
downloadContext,
packagesFolder,
cacheContext,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(13, files.Count());
}
}
}
public async Task V2FeedParser_SearchWithPrerelease(string packageSource)
{
// Arrange
var repo = Repository.Factory.GetCoreV3(packageSource);
var httpSource = HttpSource.Create(repo);
V2FeedParser parser = new V2FeedParser(httpSource, packageSource);
var searchFilter = new SearchFilter()
{
IncludePrerelease = true,
SupportedFrameworks = new string[] { "net" }
};
// Act
var packages = await parser.Search("entityframework", searchFilter, 0, 3, NullLogger.Instance, CancellationToken.None);
var package = packages.Where(p => p.Id == "EntityFramework" && p.Version.ToString() == "7.0.0-beta4").FirstOrDefault();
// Assert
Assert.NotNull(package);
}
public async Task V2FeedParser_DownloadFromUrl()
{
// Arrange
var repo = Repository.Factory.GetCoreV3("https://www.nuget.org/api/v2/");
var httpSource = HttpSource.Create(repo);
var parser = new V2FeedParser(httpSource, "https://www.nuget.org/api/v2/");
// Act & Assert
using (var packagesFolder = TestDirectory.Create())
using (var cacheContext = new SourceCacheContext())
{
var downloadContext = new PackageDownloadContext(cacheContext)
{
ExtractionContext = new PackageExtractionContext(
PackageSaveMode.Defaultv3,
PackageExtractionBehavior.XmlDocFileSaveMode,
clientPolicyContext: null,
logger: NullLogger.Instance)
};
using (var downloadResult = await parser.DownloadFromUrl(
new PackageIdentity("WindowsAzure.Storage", new NuGetVersion("6.2.0")),
new Uri("https://www.nuget.org/api/v2/package/WindowsAzure.Storage/6.2.0"),
downloadContext,
packagesFolder,
NullLogger.Instance,
CancellationToken.None))
{
var packageReader = downloadResult.PackageReader;
var files = packageReader.GetFiles();
Assert.Equal(13, files.Count());
}
}
}
// Usage nuget-mirror -package {package} -source {sourceName} -destination {destinationName} -apikey {apikey}
static async Task Main(string[] args)
{
try
{
var commandLineParser = new SimpleCommandLineParser();
commandLineParser.Parse(args);
var package = commandLineParser.Arguments["package"][0];
var sourceFeedName = commandLineParser.Arguments["source"][0];
var destinationFeedName = commandLineParser.Arguments["destination"][0];
var apiKey = commandLineParser.Arguments["apikey"][0];
// Get package sources from configs
var settings = Settings.LoadDefaultSettings(Assembly.GetEntryAssembly().Location);
var sourceRepositoryProvider = new SourceRepositoryProvider(settings, Repository.Provider.GetCoreV3());
var packageSources = sourceRepositoryProvider.PackageSourceProvider.LoadPackageSources().ToArray();
var sourcePackageSource = packageSources.Single(s => s.Name == sourceFeedName);
var destinationPackageSource = packageSources.Single(s => s.Name == destinationFeedName);
// Source feed setup
var sourceRepository = sourceRepositoryProvider.CreateRepository(sourcePackageSource, FeedType.HttpV3);
var sourceHttpSource = HttpSource.Create(sourceRepository);
var sourcePackagesFind = new RemoteV3FindPackageByIdResource(sourceRepository, sourceHttpSource);
// Destination feed setup
var destinationRepository = sourceRepositoryProvider.CreateRepository(destinationPackageSource, FeedType.HttpV3);
var destinationHttpSource = HttpSource.Create(destinationRepository);
var destinationPackages =
new RemoteV3FindPackageByIdResource(destinationRepository, destinationHttpSource);
var updateResource = await destinationRepository.GetResourceAsync <PackageUpdateResource>();
var logger = new NullLogger();
using (var sourceCacheContext = new SourceCacheContext())
using (var destinationCacheContext = new SourceCacheContext())
{
// List all versions from source
var sourcePackageVersions = (await sourcePackagesFind
.GetAllVersionsAsync(package, sourceCacheContext, logger,
CancellationToken.None))
.Select(p => p.OriginalVersion);
// List all versions from destination
var destinationPackageVersions = (await destinationPackages
.GetAllVersionsAsync(package, destinationCacheContext, logger,
CancellationToken.None))
.Select(p => p.OriginalVersion);
// See what versions are missing
var missingVersions = sourcePackageVersions
.Where(version => !destinationPackageVersions.Contains(version))
.ToArray();
// Push missing versions
var tempPath = Path.GetTempPath();
foreach (var version in missingVersions)
{
Console.WriteLine($"Mirroring {package}.{version}...");
// download
var packageDownloader = await sourcePackagesFind.GetPackageDownloaderAsync(
new PackageIdentity(package, new NuGetVersion(version)),
sourceCacheContext, logger, CancellationToken.None);
var destinationFilePath = Path.Combine(tempPath, $"OwinHttpMessageHandler.{version}.nupkg");
await packageDownloader.CopyNupkgFileToAsync(destinationFilePath, CancellationToken.None);
// push
await updateResource.Push(destinationFilePath, null, 600, false, s => apiKey, _ => null,
logger);
}
}
Console.WriteLine("Complete.");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
private FindPackageByIdResource GetV2Resource(string source)
{
var repository = factory.GetCoreV2(new PackageSource(source));
return(new RemoteV2FindPackageByIdResource(repository.PackageSource, HttpSource.Create(repository)));
}
private FindPackageByIdResource GetV3Resource(string source)
{
var repository = factory.GetCoreV3(source);
return(new RemoteV3FindPackageByIdResource(repository, HttpSource.Create(repository)));
}
//// args[0] = Dump path
//// args[1] = Number of packages to fetch
//// args[2] = Dump partition size
static void Main(string[] args)
{
var repo = Repository.Factory.GetCoreV3(NugetODataFeedUrl);
var httpSource = HttpSource.Create(repo);
var reader = new NugetFeedReader(httpSource, NugetODataFeedUrl);
var logger = NullLogger.Instance;
var totalPackageCount = reader.GetCountAsync(logger, CancellationToken.None).Result;
var dumpPath = args.Length > 0 && !string.IsNullOrEmpty(args[0]) ? args[0] : NuSearchConfiguration.PackagePath;
var numberOfPackages = args.Length > 1 && int.TryParse(args[1], out int n) ? n : totalPackageCount;
var partitionSize = args.Length > 2 && int.TryParse(args[2], out int p) ? p : 1000;
Console.WriteLine($"Downloading packages from {NugetODataFeedUrl} to {dumpPath}");
var startTime = DateTime.Now;
var take = Math.Min(100, numberOfPackages);
var numberOfPages = (int)Math.Ceiling((double)numberOfPackages / take);
Console.WriteLine($"Total {numberOfPages} pages to download");
var sync = new object();
var packages = new List <FeedPackage>(partitionSize);
int page = 0, partition = 0;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = 4
};
var searchFilter = new SearchFilter(true, null)
{
OrderBy = SearchOrderBy.Id
};
Parallel.For(0, numberOfPages, parallelOptions, (i, state) =>
{
var foundPackages = reader.GetPackagesAsync(null, searchFilter, i * take, take, logger, CancellationToken.None).Result;
lock (sync)
{
packages.AddRange(foundPackages);
if (packages.Count >= partitionSize)
{
WritePackages(packages, dumpPath, partition);
partition++;
packages = new List <FeedPackage>(partitionSize);
}
}
Interlocked.Increment(ref page);
Console.WriteLine($"Downloaded {page}/{numberOfPages} pages, written {partition} files");
});
if (packages.Count > 0)
{
WritePackages(packages, dumpPath, partition);
partition++;
Console.WriteLine($"Downloaded {page}/{numberOfPages} pages, written {partition} files");
}
var span = DateTime.Now - startTime;
Console.WriteLine($"Harvesting completed in: {span}");
Console.WriteLine("Press Enter to continue");
Console.Read();
}
