public IEnumerable <LocalPackageInfo> GetPackages() { var physicalFs = (PhysicalFileSystem)_fileSystem; string root = physicalFs.Root; return(LocalFolderUtility.GetPackagesV3(root, _logAdapter)); }
public IEnumerable <LocalPackageInfo> FindPackagesById(string packageId) { //TODO resign from IFileSystem var physicalFs = (PhysicalFileSystem)_fileSystem; string root = physicalFs.Root; return(LocalFolderUtility.GetPackagesV3(root, packageId.ToLowerInvariant(), _logAdapter)); }
public async Task RestorePackagesConfig_WithExistingLockFile_LockedMode_Succeeds() { // Arrange using (var pathContext = new SimpleTestPathContext()) { // Set up solution, project, and packages var solution = new SimpleTestSolutionContext(pathContext.SolutionRoot); var net461 = NuGetFramework.Parse("net461"); var projectA = SimpleTestProjectContext.CreateLegacyPackageReference( "a", pathContext.SolutionRoot, net461); var packageX = new SimpleTestPackageContext() { Id = "x", Version = "1.0.0" }; packageX.Files.Clear(); packageX.AddFile("lib/net461/x.dll"); solution.Projects.Add(projectA); solution.Create(pathContext.SolutionRoot); Util.CreateFile(Path.GetDirectoryName(projectA.ProjectPath), "packages.config", @"<packages> <package id=""x"" version=""1.0.0"" targetFramework=""net461"" /> </packages>"); await SimpleTestPackageUtility.CreateFolderFeedV3Async( pathContext.PackageSource, PackageSaveMode.Defaultv3, packageX); // Preconditions, regular restore var result = RunRestore(pathContext, _successExitCode); result.Success.Should().BeTrue(because: result.AllOutput); new FileInfo(projectA.NuGetLockFileOutputPath).Exists.Should().BeFalse(); // Write expected lock file var packagePath = LocalFolderUtility.GetPackagesV3(pathContext.PackageSource, NullLogger.Instance).Single().Path; string contentHash = null; using (var reader = new PackageArchiveReader(packagePath)) { contentHash = reader.GetContentHash(CancellationToken.None); } var expectedLockFile = GetResource("NuGet.CommandLine.FuncTest.compiler.resources.pc.packages.lock.json").Replace("TEMPLATE", contentHash); File.WriteAllText(projectA.NuGetLockFileOutputPath, expectedLockFile); // Run lockedmode restore. result = RunRestore(pathContext, _successExitCode, "-LockedMode"); result.Success.Should().BeTrue(because: result.AllOutput); new FileInfo(projectA.NuGetLockFileOutputPath).Exists.Should().BeTrue(); } }
private static void VerifyPackagesIntegrity(string folderRoot, string fileName) { var sb = new StringBuilder(); if (!Directory.Exists(folderRoot)) { Console.WriteLine($"Could not find folder: {folderRoot}"); return; } var packageInfos = LocalFolderUtility.GetPackagesV3(folderRoot, NullLogger.Instance); var numIncorrectHash = 0; foreach (var packageInfo in packageInfos) { if (File.Exists(packageInfo.Path)) { var packageHash = string.Empty; using (var stream = new FileStream( packageInfo.Path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, useAsync: true)) { var bytes = new CryptoHashProvider("SHA512").CalculateHash(stream); packageHash = Convert.ToBase64String(bytes); } var sha512File = packageInfo.Path + ".sha512"; if (File.Exists(sha512File)) { var existingHash = File.ReadAllText(sha512File); var flag = string.Equals(existingHash, packageHash); if (!flag) { numIncorrectHash++; } sb.AppendLine(string.Join(',', packageInfo.Identity.Id, packageInfo.Identity.Version.ToNormalizedString(), existingHash, packageHash, flag)); } } } Console.WriteLine($"Total number of incorrect hashed packages in {folderRoot}: {numIncorrectHash}"); File.WriteAllText(fileName, sb.ToString()); }
public async Task GivenALargeNumberOfPackagesVerifyAllAreDownloaded() { // Arrange using (var cache = new LocalCache()) using (var cacheContext = new SourceCacheContext()) using (var workingDir = new TestFolder()) { var catalogLog = new TestLogger(); var log = new TestLogger(); var baseUri = Sleet.UriUtility.CreateUri("https://localhost:8080/testFeed/"); var feedFolder = Path.Combine(workingDir, "feed"); var nupkgsFolder = Path.Combine(workingDir, "nupkgs"); var nupkgsOutFolder = Path.Combine(workingDir, "nupkgsout"); Directory.CreateDirectory(feedFolder); Directory.CreateDirectory(nupkgsFolder); Directory.CreateDirectory(nupkgsOutFolder); var expected = new HashSet <PackageIdentity>(); for (var i = 0; i < 200; i++) { var identity = new PackageIdentity(Guid.NewGuid().ToString(), NuGetVersion.Parse($"{i}.0.0")); if (expected.Add(identity)) { var package = new TestNupkg(identity.Id, identity.Version.ToNormalizedString()); TestNupkg.Save(nupkgsFolder, package); } } await CatalogReaderTestHelpers.CreateCatalogAsync(workingDir, feedFolder, nupkgsFolder, baseUri, catalogLog); var feedUri = Sleet.UriUtility.CreateUri(baseUri.AbsoluteUri + "index.json"); var httpSource = CatalogReaderTestHelpers.GetHttpSource(cache, feedFolder, baseUri); var args = new string[] { "nupkgs", "-o", nupkgsOutFolder, feedUri.AbsoluteUri, "--delay", "0" }; var exitCode = await NuGetMirror.Program.MainCore(args, httpSource, log); var errors = log.GetMessages(LogLevel.Error); exitCode.Should().Be(0, errors); var results = LocalFolderUtility.GetPackagesV3(nupkgsOutFolder, catalogLog).ToList(); results.Select(e => e.Identity).ShouldBeEquivalentTo(expected); var errorLog = Path.Combine(nupkgsOutFolder, "lastRunErrors.txt"); File.Exists(errorLog).Should().BeFalse(); } }
private static IList <string> VerifyPackageFolderIntegrity(string folderRoot) { var listOfInvalidFolders = new List <string>(); if (!Directory.Exists(folderRoot)) { Console.WriteLine($"Could not find folder: {folderRoot}"); return(listOfInvalidFolders); } var packageInfos = LocalFolderUtility.GetPackagesV3(folderRoot, NullLogger.Instance); var numExtractedFolders = 0; var numNotExtractedFolders = 0; var numIncorrectExtractedFolders = 0; foreach (var packageInfo in packageInfos) { var directory = Path.GetDirectoryName(packageInfo.Path); var sha512 = packageInfo.Path + ".sha512"; if (File.Exists(sha512)) { var packageReader = packageInfo.GetReader(); var filesInPackage = packageReader.GetFiles().Where(t => !IsNotExtractableFile(t)); foreach (var file in filesInPackage) { var fullPath = Path.GetFullPath(Path.Combine(directory, file)); if (!File.Exists(fullPath)) { listOfInvalidFolders.Add(directory); numIncorrectExtractedFolders++; break; } } numExtractedFolders++; } else { numNotExtractedFolders++; // if no sha512 file exists, we are good. continue; } } Console.WriteLine($"Total number of extracted folders in {folderRoot}: {numExtractedFolders}"); Console.WriteLine($"Total number of incorrectly extracted folders in {folderRoot}: {numIncorrectExtractedFolders}"); Console.WriteLine($"Total number of unextracted folders in {folderRoot}: {numNotExtractedFolders}"); return(listOfInvalidFolders); }
IEnumerable <LocalPackageInfo> GetAllPackagesInFolder(string rootPath, NuGetDirectoryStructure nuGetDirectoryStructure) { var logger = new Logger(NuGet.Common.LogLevel.Information); switch (nuGetDirectoryStructure) { case NuGetDirectoryStructure.V2: return(LocalFolderUtility.GetPackagesV2(root: rootPath, log: logger)); case NuGetDirectoryStructure.V3: return(LocalFolderUtility.GetPackagesV3(root: rootPath, log: logger)); } return(new List <LocalPackageInfo>()); }
public async Task GivenLatestOnlyOptionVerifyDownloadsOnlyLatest() { // Arrange using (var cache = new LocalCache()) using (var cacheContext = new SourceCacheContext()) using (var workingDir = new TestFolder()) { var beforeDate = DateTimeOffset.UtcNow; var catalogLog = new TestLogger(); var log = new TestLogger(); var baseUri = Sleet.UriUtility.CreateUri("https://localhost:8080/testFeed/"); var feedFolder = Path.Combine(workingDir, "feed"); var nupkgsFolder = Path.Combine(workingDir, "nupkgs"); var nupkgsOutFolder = Path.Combine(workingDir, "nupkgsout"); Directory.CreateDirectory(feedFolder); Directory.CreateDirectory(nupkgsFolder); Directory.CreateDirectory(nupkgsOutFolder); var packageA1 = new TestNupkg("a", "1.0.0"); TestNupkg.Save(nupkgsFolder, packageA1); var packageA2 = new TestNupkg("a", "2.0.0"); TestNupkg.Save(nupkgsFolder, packageA2); var packageB1 = new TestNupkg("b", "1.0.0"); TestNupkg.Save(nupkgsFolder, packageB1); var packageB2 = new TestNupkg("b", "2.0.0"); TestNupkg.Save(nupkgsFolder, packageB2); await CatalogReaderTestHelpers.CreateCatalogAsync(workingDir, feedFolder, nupkgsFolder, baseUri, catalogLog); var feedUri = Sleet.UriUtility.CreateUri(baseUri.AbsoluteUri + "index.json"); var httpSource = CatalogReaderTestHelpers.GetHttpSource(cache, feedFolder, baseUri); var args = new string[] { "nupkgs", "-o", nupkgsOutFolder, feedUri.AbsoluteUri, "--delay", "0", "--latest-only" }; var exitCode = await NuGetMirror.Program.MainCore(args, httpSource, log); exitCode.Should().Be(0); var results = LocalFolderUtility.GetPackagesV3(nupkgsOutFolder, catalogLog).ToList(); results.Select(e => e.Identity).ShouldBeEquivalentTo( new[] { new PackageIdentity("a", NuGetVersion.Parse("2.0.0")), new PackageIdentity("b", NuGetVersion.Parse("2.0.0")) }); } }
public static IEnumerable <LocalPackageInfo> GetPackages(string packagesPath, ILogger log = null) { var logger = log ?? NullLogger.Instance; var type = LocalFolderUtility.GetLocalFeedType(packagesPath, logger); switch (type) { case FeedType.FileSystemV2: return(LocalFolderUtility.GetPackagesV2(packagesPath, logger)); case FeedType.FileSystemV3: return(LocalFolderUtility.GetPackagesV3(packagesPath, logger)); default: break; } return(new List <LocalPackageInfo>()); }
public async Task VerifyPackagesAreDownloadedInV3Structure() { // Arrange using (var cache = new LocalCache()) using (var cacheContext = new SourceCacheContext()) using (var workingDir = new TestFolder()) { var beforeDate = DateTimeOffset.UtcNow; var catalogLog = new TestLogger(); var log = new TestLogger(); var baseUri = Sleet.UriUtility.CreateUri("https://localhost:8080/testFeed/"); var feedFolder = Path.Combine(workingDir, "feed"); var nupkgsFolder = Path.Combine(workingDir, "nupkgs"); var nupkgsOutFolder = Path.Combine(workingDir, "nupkgsout"); Directory.CreateDirectory(feedFolder); Directory.CreateDirectory(nupkgsFolder); Directory.CreateDirectory(nupkgsOutFolder); var packageA = new TestNupkg("a", "1.0.0"); TestNupkg.Save(nupkgsFolder, packageA); await CatalogReaderTestHelpers.CreateCatalogAsync(workingDir, feedFolder, nupkgsFolder, baseUri, catalogLog); var feedUri = Sleet.UriUtility.CreateUri(baseUri.AbsoluteUri + "index.json"); var httpSource = CatalogReaderTestHelpers.GetHttpSource(cache, feedFolder, baseUri); var args = new string[] { "nupkgs", "-o", nupkgsOutFolder, feedUri.AbsoluteUri, "--delay", "0" }; var exitCode = await NuGetMirror.Program.MainCore(args, httpSource, log); exitCode.Should().Be(0); var results = LocalFolderUtility.GetPackagesV3(nupkgsOutFolder, catalogLog).ToList(); results.Select(e => e.Identity).ShouldBeEquivalentTo(new[] { new PackageIdentity("a", NuGetVersion.Parse("1.0.0")) }); var afterDate = DateTimeOffset.UtcNow; var cursor = MirrorUtility.LoadCursor(new DirectoryInfo(nupkgsOutFolder)); (cursor <= afterDate && cursor >= beforeDate).Should().BeTrue("the cursor should match the catalog"); var errorLog = Path.Combine(nupkgsOutFolder, "lastRunErrors.txt"); File.Exists(errorLog).Should().BeFalse(); } }
public async Task <ActionResult <RegistrationIndex> > GetRegistrations(string lowerId) { // Find all local packages var localRepDir = Directory.GetCurrentDirectory() + "/packages/"; var packages = LocalFolderUtility.GetPackagesV3(localRepDir, lowerId, new CustomLogger()).ToList(); if (packages.Count < 1) { var officialNuget = Repository.Factory.GetCoreV3("https://api.nuget.org/v3/index.json", FeedType.HttpV3); var metadataResource = officialNuget.GetResource <PackageMetadataResource>(); var packageMetadatas = await metadataResource.GetMetadataAsync(lowerId, true, true, caching, new CustomLogger(), CancellationToken.None); packages = packageMetadatas.Select(_ => new LocalPackageInfo( _.Identity, "PATH", new Lazy <NuspecReader>(() => new NuspecReader()) )); foreach (var packageData in packageMetadata) { System.Console.WriteLine($"PACKAGE {packageData.Identity}"); } return(NotFound()); } NuGetVersion lowerVersion = null; NuGetVersion upperVersion = null; var currentFullUrl = HttpContext.Request.GetDisplayUrl(); var currentPackageBaseUrl = currentFullUrl.Replace("/index.json", "/"); var contentUrl = currentFullUrl.Substring(0, currentFullUrl.IndexOf("/v3/registrations")) + "/v3/flatcontainer/"; var pageItems = packages.Select(package => { lowerVersion = lowerVersion == null || lowerVersion > package.Identity.Version ? package.Identity.Version : lowerVersion; upperVersion = upperVersion == null || upperVersion < package.Identity.Version ? package.Identity.Version : upperVersion; var packageContentUrl = $"{contentUrl}{package.Identity.Id.ToLowerInvariant()}/{package.Identity.Version.ToNormalizedString()}/{package.Identity.ToString().ToLowerInvariant()}.nupkg"; return(new RegistrationIndexPageItem( $"{currentPackageBaseUrl}{package.Identity.Version.ToNormalizedString()}.json", new PackageMetadata( "", package.Nuspec.GetId(), package.Nuspec.GetVersion(), package.Nuspec.GetAuthors(), package.Nuspec.GetDescription(), 0, false, package.Nuspec.GetIconUrl(), package.Nuspec.GetLanguage(), package.Nuspec.GetLicenseUrl(), true, package.Nuspec.GetMinClientVersion()?.ToNormalizedString() ?? "", packageContentUrl, package.Nuspec.GetProjectUrl(), package.Nuspec.GetRepositoryMetadata().Url, package.Nuspec.GetRepositoryMetadata().Type, package.LastWriteTimeUtc, package.Nuspec.GetRequireLicenseAcceptance(), package.Nuspec.GetSummary(), package.Nuspec.GetTags().Split(' '), package.Nuspec.GetTitle(), package.Nuspec.GetDependencyGroups().Select(group => new DependencyGroupItem( $"", group.TargetFramework.DotNetFrameworkName, group.Packages.Select(dependency => new DependencyItem( $"", dependency.Id, dependency.VersionRange.ToString() )).ToList() )).ToList() ), packageContentUrl )); }).ToList(); return(new RegistrationIndex(1, 0, new List <RegistrationIndexPage> { new RegistrationIndexPage( $"{HttpContext.Request.GetDisplayUrl()}#page", pageItems.Count, pageItems, lowerVersion, upperVersion ) })); }
private void EnumerateFilesV3Style(bool onlyLatest) { try { Stopwatch timer = new Stopwatch(); timer.Start(); var packageDirectories = Directory.EnumerateDirectories(_inputPath).ToArray(); long count = 0; long errorCount = 0; var logger = new CommandOutputLogger(NuGet.Common.LogLevel.Information); ParallelOptions ops = new ParallelOptions { MaxDegreeOfParallelism = _maxThreadCount }; Parallel.ForEach(packageDirectories, ops, packageDirectory => { count++; if (count % 10000 == 0) { DisplayStats(count, errorCount, packageDirectories.Length, timer); } try { var id = Path.GetFileName(packageDirectory); var idPackages = LocalFolderUtility.GetPackagesV3(_inputPath, id, logger); if (onlyLatest) { var package = idPackages .OrderByDescending(e => e.Identity.Version) .Max(); ProcessArchiveForNuGetAPIsUsedInScripts(package); } else { foreach (var package in idPackages) { ProcessArchiveForNuGetAPIsUsedInScripts(package); } } } catch (Exception ex) { errorCount++; logger.LogInformation("Exception while parsing " + packageDirectory); logger.LogInformation(ex.Message); var curroptFilePath = Path.Combine(@"f:\CurroptPackages", Path.GetFileName(packageDirectory)); File.Move(packageDirectory, curroptFilePath); } }); logger.LogInformation("Done with all the " + count + " packages"); } catch (AggregateException ae) { // This is where you can choose which exceptions to handle. foreach (var ex in ae.InnerExceptions) { Console.WriteLine(ex.Message); } } }
private static IEnumerable <LocalPackageInfo> GetAllPackagesInFolder(string rootPath) { return(LocalFolderUtility.GetPackagesV3(root: rootPath, log: NullLogger.Instance)); }