public IEnumerable <LocalPackageInfo> GetPackages()
{
var physicalFs = (PhysicalFileSystem)_fileSystem;
string root = physicalFs.Root;
return(LocalFolderUtility.GetPackagesV3(root, _logAdapter));
}
public IEnumerable <LocalPackageInfo> FindPackagesById(string packageId)
{
//TODO resign from IFileSystem
var physicalFs = (PhysicalFileSystem)_fileSystem;
string root = physicalFs.Root;
return(LocalFolderUtility.GetPackagesV3(root, packageId.ToLowerInvariant(), _logAdapter));
}
public async Task RestorePackagesConfig_WithExistingLockFile_LockedMode_Succeeds()
{
// Arrange
using (var pathContext = new SimpleTestPathContext())
{
// Set up solution, project, and packages
var solution = new SimpleTestSolutionContext(pathContext.SolutionRoot);
var net461 = NuGetFramework.Parse("net461");
var projectA = SimpleTestProjectContext.CreateLegacyPackageReference(
"a",
pathContext.SolutionRoot,
net461);
var packageX = new SimpleTestPackageContext()
{
Id = "x",
Version = "1.0.0"
};
packageX.Files.Clear();
packageX.AddFile("lib/net461/x.dll");
solution.Projects.Add(projectA);
solution.Create(pathContext.SolutionRoot);
Util.CreateFile(Path.GetDirectoryName(projectA.ProjectPath), "packages.config",
@"<packages>
<package id=""x"" version=""1.0.0"" targetFramework=""net461"" />
</packages>");
await SimpleTestPackageUtility.CreateFolderFeedV3Async(
pathContext.PackageSource,
PackageSaveMode.Defaultv3,
packageX);
// Preconditions, regular restore
var result = RunRestore(pathContext, _successExitCode);
result.Success.Should().BeTrue(because: result.AllOutput);
new FileInfo(projectA.NuGetLockFileOutputPath).Exists.Should().BeFalse();
// Write expected lock file
var packagePath = LocalFolderUtility.GetPackagesV3(pathContext.PackageSource, NullLogger.Instance).Single().Path;
string contentHash = null;
using (var reader = new PackageArchiveReader(packagePath))
{
contentHash = reader.GetContentHash(CancellationToken.None);
}
var expectedLockFile = GetResource("NuGet.CommandLine.FuncTest.compiler.resources.pc.packages.lock.json").Replace("TEMPLATE", contentHash);
File.WriteAllText(projectA.NuGetLockFileOutputPath, expectedLockFile);
// Run lockedmode restore.
result = RunRestore(pathContext, _successExitCode, "-LockedMode");
result.Success.Should().BeTrue(because: result.AllOutput);
new FileInfo(projectA.NuGetLockFileOutputPath).Exists.Should().BeTrue();
}
}
private static void VerifyPackagesIntegrity(string folderRoot, string fileName)
{
var sb = new StringBuilder();
if (!Directory.Exists(folderRoot))
{
Console.WriteLine($"Could not find folder: {folderRoot}");
return;
}
var packageInfos = LocalFolderUtility.GetPackagesV3(folderRoot, NullLogger.Instance);
var numIncorrectHash = 0;
foreach (var packageInfo in packageInfos)
{
if (File.Exists(packageInfo.Path))
{
var packageHash = string.Empty;
using (var stream = new FileStream(
packageInfo.Path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 4096,
useAsync: true))
{
var bytes = new CryptoHashProvider("SHA512").CalculateHash(stream);
packageHash = Convert.ToBase64String(bytes);
}
var sha512File = packageInfo.Path + ".sha512";
if (File.Exists(sha512File))
{
var existingHash = File.ReadAllText(sha512File);
var flag = string.Equals(existingHash, packageHash);
if (!flag)
{
numIncorrectHash++;
}
sb.AppendLine(string.Join(',',
packageInfo.Identity.Id,
packageInfo.Identity.Version.ToNormalizedString(),
existingHash,
packageHash,
flag));
}
}
}
Console.WriteLine($"Total number of incorrect hashed packages in {folderRoot}: {numIncorrectHash}");
File.WriteAllText(fileName, sb.ToString());
}
public async Task GivenALargeNumberOfPackagesVerifyAllAreDownloaded()
{
// Arrange
using (var cache = new LocalCache())
using (var cacheContext = new SourceCacheContext())
using (var workingDir = new TestFolder())
{
var catalogLog = new TestLogger();
var log = new TestLogger();
var baseUri = Sleet.UriUtility.CreateUri("https://localhost:8080/testFeed/");
var feedFolder = Path.Combine(workingDir, "feed");
var nupkgsFolder = Path.Combine(workingDir, "nupkgs");
var nupkgsOutFolder = Path.Combine(workingDir, "nupkgsout");
Directory.CreateDirectory(feedFolder);
Directory.CreateDirectory(nupkgsFolder);
Directory.CreateDirectory(nupkgsOutFolder);
var expected = new HashSet <PackageIdentity>();
for (var i = 0; i < 200; i++)
{
var identity = new PackageIdentity(Guid.NewGuid().ToString(), NuGetVersion.Parse($"{i}.0.0"));
if (expected.Add(identity))
{
var package = new TestNupkg(identity.Id, identity.Version.ToNormalizedString());
TestNupkg.Save(nupkgsFolder, package);
}
}
await CatalogReaderTestHelpers.CreateCatalogAsync(workingDir, feedFolder, nupkgsFolder, baseUri, catalogLog);
var feedUri = Sleet.UriUtility.CreateUri(baseUri.AbsoluteUri + "index.json");
var httpSource = CatalogReaderTestHelpers.GetHttpSource(cache, feedFolder, baseUri);
var args = new string[] { "nupkgs", "-o", nupkgsOutFolder, feedUri.AbsoluteUri, "--delay", "0" };
var exitCode = await NuGetMirror.Program.MainCore(args, httpSource, log);
var errors = log.GetMessages(LogLevel.Error);
exitCode.Should().Be(0, errors);
var results = LocalFolderUtility.GetPackagesV3(nupkgsOutFolder, catalogLog).ToList();
results.Select(e => e.Identity).ShouldBeEquivalentTo(expected);
var errorLog = Path.Combine(nupkgsOutFolder, "lastRunErrors.txt");
File.Exists(errorLog).Should().BeFalse();
}
}
private static IList <string> VerifyPackageFolderIntegrity(string folderRoot)
{
var listOfInvalidFolders = new List <string>();
if (!Directory.Exists(folderRoot))
{
Console.WriteLine($"Could not find folder: {folderRoot}");
return(listOfInvalidFolders);
}
var packageInfos = LocalFolderUtility.GetPackagesV3(folderRoot, NullLogger.Instance);
var numExtractedFolders = 0;
var numNotExtractedFolders = 0;
var numIncorrectExtractedFolders = 0;
foreach (var packageInfo in packageInfos)
{
var directory = Path.GetDirectoryName(packageInfo.Path);
var sha512 = packageInfo.Path + ".sha512";
if (File.Exists(sha512))
{
var packageReader = packageInfo.GetReader();
var filesInPackage = packageReader.GetFiles().Where(t => !IsNotExtractableFile(t));
foreach (var file in filesInPackage)
{
var fullPath = Path.GetFullPath(Path.Combine(directory, file));
if (!File.Exists(fullPath))
{
listOfInvalidFolders.Add(directory);
numIncorrectExtractedFolders++;
break;
}
}
numExtractedFolders++;
}
else
{
numNotExtractedFolders++;
// if no sha512 file exists, we are good.
continue;
}
}
Console.WriteLine($"Total number of extracted folders in {folderRoot}: {numExtractedFolders}");
Console.WriteLine($"Total number of incorrectly extracted folders in {folderRoot}: {numIncorrectExtractedFolders}");
Console.WriteLine($"Total number of unextracted folders in {folderRoot}: {numNotExtractedFolders}");
return(listOfInvalidFolders);
}
IEnumerable <LocalPackageInfo> GetAllPackagesInFolder(string rootPath, NuGetDirectoryStructure nuGetDirectoryStructure)
{
var logger = new Logger(NuGet.Common.LogLevel.Information);
switch (nuGetDirectoryStructure)
{
case NuGetDirectoryStructure.V2:
return(LocalFolderUtility.GetPackagesV2(root: rootPath, log: logger));
case NuGetDirectoryStructure.V3:
return(LocalFolderUtility.GetPackagesV3(root: rootPath, log: logger));
}
return(new List <LocalPackageInfo>());
}
public async Task GivenLatestOnlyOptionVerifyDownloadsOnlyLatest()
{
// Arrange
using (var cache = new LocalCache())
using (var cacheContext = new SourceCacheContext())
using (var workingDir = new TestFolder())
{
var beforeDate = DateTimeOffset.UtcNow;
var catalogLog = new TestLogger();
var log = new TestLogger();
var baseUri = Sleet.UriUtility.CreateUri("https://localhost:8080/testFeed/");
var feedFolder = Path.Combine(workingDir, "feed");
var nupkgsFolder = Path.Combine(workingDir, "nupkgs");
var nupkgsOutFolder = Path.Combine(workingDir, "nupkgsout");
Directory.CreateDirectory(feedFolder);
Directory.CreateDirectory(nupkgsFolder);
Directory.CreateDirectory(nupkgsOutFolder);
var packageA1 = new TestNupkg("a", "1.0.0");
TestNupkg.Save(nupkgsFolder, packageA1);
var packageA2 = new TestNupkg("a", "2.0.0");
TestNupkg.Save(nupkgsFolder, packageA2);
var packageB1 = new TestNupkg("b", "1.0.0");
TestNupkg.Save(nupkgsFolder, packageB1);
var packageB2 = new TestNupkg("b", "2.0.0");
TestNupkg.Save(nupkgsFolder, packageB2);
await CatalogReaderTestHelpers.CreateCatalogAsync(workingDir, feedFolder, nupkgsFolder, baseUri, catalogLog);
var feedUri = Sleet.UriUtility.CreateUri(baseUri.AbsoluteUri + "index.json");
var httpSource = CatalogReaderTestHelpers.GetHttpSource(cache, feedFolder, baseUri);
var args = new string[] { "nupkgs", "-o", nupkgsOutFolder, feedUri.AbsoluteUri, "--delay", "0", "--latest-only" };
var exitCode = await NuGetMirror.Program.MainCore(args, httpSource, log);
exitCode.Should().Be(0);
var results = LocalFolderUtility.GetPackagesV3(nupkgsOutFolder, catalogLog).ToList();
results.Select(e => e.Identity).ShouldBeEquivalentTo(
new[] {
new PackageIdentity("a", NuGetVersion.Parse("2.0.0")),
new PackageIdentity("b", NuGetVersion.Parse("2.0.0"))
});
}
}
public static IEnumerable <LocalPackageInfo> GetPackages(string packagesPath, ILogger log = null)
{
var logger = log ?? NullLogger.Instance;
var type = LocalFolderUtility.GetLocalFeedType(packagesPath, logger);
switch (type)
{
case FeedType.FileSystemV2:
return(LocalFolderUtility.GetPackagesV2(packagesPath, logger));
case FeedType.FileSystemV3:
return(LocalFolderUtility.GetPackagesV3(packagesPath, logger));
default:
break;
}
return(new List <LocalPackageInfo>());
}
public async Task VerifyPackagesAreDownloadedInV3Structure()
{
// Arrange
using (var cache = new LocalCache())
using (var cacheContext = new SourceCacheContext())
using (var workingDir = new TestFolder())
{
var beforeDate = DateTimeOffset.UtcNow;
var catalogLog = new TestLogger();
var log = new TestLogger();
var baseUri = Sleet.UriUtility.CreateUri("https://localhost:8080/testFeed/");
var feedFolder = Path.Combine(workingDir, "feed");
var nupkgsFolder = Path.Combine(workingDir, "nupkgs");
var nupkgsOutFolder = Path.Combine(workingDir, "nupkgsout");
Directory.CreateDirectory(feedFolder);
Directory.CreateDirectory(nupkgsFolder);
Directory.CreateDirectory(nupkgsOutFolder);
var packageA = new TestNupkg("a", "1.0.0");
TestNupkg.Save(nupkgsFolder, packageA);
await CatalogReaderTestHelpers.CreateCatalogAsync(workingDir, feedFolder, nupkgsFolder, baseUri, catalogLog);
var feedUri = Sleet.UriUtility.CreateUri(baseUri.AbsoluteUri + "index.json");
var httpSource = CatalogReaderTestHelpers.GetHttpSource(cache, feedFolder, baseUri);
var args = new string[] { "nupkgs", "-o", nupkgsOutFolder, feedUri.AbsoluteUri, "--delay", "0" };
var exitCode = await NuGetMirror.Program.MainCore(args, httpSource, log);
exitCode.Should().Be(0);
var results = LocalFolderUtility.GetPackagesV3(nupkgsOutFolder, catalogLog).ToList();
results.Select(e => e.Identity).ShouldBeEquivalentTo(new[] { new PackageIdentity("a", NuGetVersion.Parse("1.0.0")) });
var afterDate = DateTimeOffset.UtcNow;
var cursor = MirrorUtility.LoadCursor(new DirectoryInfo(nupkgsOutFolder));
(cursor <= afterDate && cursor >= beforeDate).Should().BeTrue("the cursor should match the catalog");
var errorLog = Path.Combine(nupkgsOutFolder, "lastRunErrors.txt");
File.Exists(errorLog).Should().BeFalse();
}
}
public async Task <ActionResult <RegistrationIndex> > GetRegistrations(string lowerId)
{
// Find all local packages
var localRepDir = Directory.GetCurrentDirectory() + "/packages/";
var packages = LocalFolderUtility.GetPackagesV3(localRepDir, lowerId, new CustomLogger()).ToList();
if (packages.Count < 1)
{
var officialNuget = Repository.Factory.GetCoreV3("https://api.nuget.org/v3/index.json", FeedType.HttpV3);
var metadataResource = officialNuget.GetResource <PackageMetadataResource>();
var packageMetadatas = await metadataResource.GetMetadataAsync(lowerId, true, true, caching, new CustomLogger(), CancellationToken.None);
packages = packageMetadatas.Select(_ => new LocalPackageInfo(
_.Identity,
"PATH",
new Lazy <NuspecReader>(() => new NuspecReader())
));
foreach (var packageData in packageMetadata)
{
System.Console.WriteLine($"PACKAGE {packageData.Identity}");
}
return(NotFound());
}
NuGetVersion lowerVersion = null;
NuGetVersion upperVersion = null;
var currentFullUrl = HttpContext.Request.GetDisplayUrl();
var currentPackageBaseUrl = currentFullUrl.Replace("/index.json", "/");
var contentUrl = currentFullUrl.Substring(0, currentFullUrl.IndexOf("/v3/registrations")) + "/v3/flatcontainer/";
var pageItems = packages.Select(package => {
lowerVersion = lowerVersion == null || lowerVersion > package.Identity.Version ? package.Identity.Version : lowerVersion;
upperVersion = upperVersion == null || upperVersion < package.Identity.Version ? package.Identity.Version : upperVersion;
var packageContentUrl = $"{contentUrl}{package.Identity.Id.ToLowerInvariant()}/{package.Identity.Version.ToNormalizedString()}/{package.Identity.ToString().ToLowerInvariant()}.nupkg";
return(new RegistrationIndexPageItem(
$"{currentPackageBaseUrl}{package.Identity.Version.ToNormalizedString()}.json",
new PackageMetadata(
"",
package.Nuspec.GetId(),
package.Nuspec.GetVersion(),
package.Nuspec.GetAuthors(),
package.Nuspec.GetDescription(),
0,
false,
package.Nuspec.GetIconUrl(),
package.Nuspec.GetLanguage(),
package.Nuspec.GetLicenseUrl(),
true,
package.Nuspec.GetMinClientVersion()?.ToNormalizedString() ?? "",
packageContentUrl,
package.Nuspec.GetProjectUrl(),
package.Nuspec.GetRepositoryMetadata().Url,
package.Nuspec.GetRepositoryMetadata().Type,
package.LastWriteTimeUtc,
package.Nuspec.GetRequireLicenseAcceptance(),
package.Nuspec.GetSummary(),
package.Nuspec.GetTags().Split(' '),
package.Nuspec.GetTitle(),
package.Nuspec.GetDependencyGroups().Select(group => new DependencyGroupItem(
$"",
group.TargetFramework.DotNetFrameworkName,
group.Packages.Select(dependency => new DependencyItem(
$"",
dependency.Id,
dependency.VersionRange.ToString()
)).ToList()
)).ToList()
),
packageContentUrl
));
}).ToList();
return(new RegistrationIndex(1, 0, new List <RegistrationIndexPage> {
new RegistrationIndexPage(
$"{HttpContext.Request.GetDisplayUrl()}#page",
pageItems.Count,
pageItems,
lowerVersion,
upperVersion
)
}));
}
private void EnumerateFilesV3Style(bool onlyLatest)
{
try
{
Stopwatch timer = new Stopwatch();
timer.Start();
var packageDirectories = Directory.EnumerateDirectories(_inputPath).ToArray();
long count = 0;
long errorCount = 0;
var logger = new CommandOutputLogger(NuGet.Common.LogLevel.Information);
ParallelOptions ops = new ParallelOptions {
MaxDegreeOfParallelism = _maxThreadCount
};
Parallel.ForEach(packageDirectories, ops, packageDirectory =>
{
count++;
if (count % 10000 == 0)
{
DisplayStats(count, errorCount, packageDirectories.Length, timer);
}
try
{
var id = Path.GetFileName(packageDirectory);
var idPackages = LocalFolderUtility.GetPackagesV3(_inputPath, id, logger);
if (onlyLatest)
{
var package = idPackages
.OrderByDescending(e => e.Identity.Version)
.Max();
ProcessArchiveForNuGetAPIsUsedInScripts(package);
}
else
{
foreach (var package in idPackages)
{
ProcessArchiveForNuGetAPIsUsedInScripts(package);
}
}
}
catch (Exception ex)
{
errorCount++;
logger.LogInformation("Exception while parsing " + packageDirectory);
logger.LogInformation(ex.Message);
var curroptFilePath = Path.Combine(@"f:\CurroptPackages", Path.GetFileName(packageDirectory));
File.Move(packageDirectory, curroptFilePath);
}
});
logger.LogInformation("Done with all the " + count + " packages");
}
catch (AggregateException ae)
{
// This is where you can choose which exceptions to handle.
foreach (var ex in ae.InnerExceptions)
{
Console.WriteLine(ex.Message);
}
}
}
private static IEnumerable <LocalPackageInfo> GetAllPackagesInFolder(string rootPath)
{
return(LocalFolderUtility.GetPackagesV3(root: rootPath, log: NullLogger.Instance));
}
