public Facts() { Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >(); Config = new Catalog2RegistrationConfiguration { LegacyBaseUrl = "https://example/reg/", GzippedBaseUrl = "https://example/reg-gz/", SemVer2BaseUrl = "https://example/reg-gz-semver2/", GalleryBaseUrl = "https://example-gallery/", FlatContainerBaseUrl = "https://example/fc/", }; Options.Setup(x => x.Value).Returns(() => Config); LeafItem = new RegistrationLeafItem(); Page = new RegistrationPage(); Index = new RegistrationIndex(); Hive = HiveType.Legacy; Id = V3Data.PackageId; PackageDetails = new PackageDetailsCatalogLeaf { PackageVersion = V3Data.NormalizedVersion, }; Commit = new CatalogCommit(V3Data.CommitId, V3Data.CommitTimestamp); SerializerSettings = NuGetJsonSerialization.Settings; SerializerSettings.Formatting = Formatting.Indented; }
public void AddsEmptyStringForDependencyVersionAllRange() { var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, DependencyGroups = new List <PackageDependencyGroup> { new PackageDependencyGroup { TargetFramework = "net40", Dependencies = new List <Protocol.Catalog.PackageDependency> { new Protocol.Catalog.PackageDependency { Id = "NuGet.Versioning", Range = "(, )" } }, }, }, }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal("NuGet.Versioning::net40", full.FlattenedDependencies); }
/// <summary> /// This method produces output for official client. The implementation on the client side, at one point of time /// was: /// https://github.com/NuGet/NuGet.Client/blob/b404acf6eb88c2b6086a9cbb5106104534de2428/src/NuGet.Core/NuGet.Protocol/LegacyFeed/V2FeedPackageInfo.cs#L228-L308 /// /// The output is a string where each dependency is separated by a "|" character and information about each /// dependency is separated by a ":" character. Per dependency, the colon-seperated data is a pair or triple. /// The three fields in order are: dependency package ID, version range, and target framework. If third value /// (the framework that the dependency targets) is an empty string or excluded, this means the dependency /// applies to any framework. If the package ID and range and empty strings but the framework is included, this /// means that the package supports the specified framework but has no dependencies specific to that framework. /// /// Output: /// [DEPENDENCY[|DEPENDENCY][|...][|DEPENDENCY]] /// Each dependency: /// [PACKAGE_ID]:[VERSION_RANGE][:TARGET_FRAMEWORK] /// /// Example A (no target frameworks): /// Microsoft.Data.OData:5.0.2|Microsoft.WindowsAzure.ConfigurationManager:1.8.0 /// Example B (target frameworks): /// NETStandard.Library:1.6.1:netstandard1.0|Newtonsoft.Json:10.0.2:netstandard1.0 /// Example D (empty target framework): /// Microsoft.Data.OData:5.0.2: /// Example D (just target framework): /// ::net20|::net35|::net40|::net45|NETStandard.Library:1.6.1:netstandard1.0 /// </summary> private static string GetFlattenedDependencies(PackageDetailsCatalogLeaf leaf) { if (leaf.DependencyGroups == null) { return(null); } var builder = new StringBuilder(); foreach (var dependencyGroup in leaf.DependencyGroups) { var targetFramework = dependencyGroup.ParseTargetFramework(); if (dependencyGroup.Dependencies != null && dependencyGroup.Dependencies.Any()) { foreach (var packageDependency in dependencyGroup.Dependencies) { AddFlattenedPackageDependency(targetFramework, packageDependency, builder); } } else { if (builder.Length > 0) { builder.Append("|"); } builder.Append(":"); AddFlattenedFrameworkDependency(targetFramework, builder); } } return(builder.Length > 0 ? builder.ToString() : null); }
public Task <bool> ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf) { Console.WriteLine( $"{leaf.CommitTimestamp:O}: Found package details leaf for {leaf.PackageId} {leaf.PackageVersion}."); return(GetResultAsync(leaf)); }
protected void SetVersion(string version) { var parsedVersion = NuGetVersion.Parse(version); _packageVersion = version; _commitItem = new CatalogCommitItem( new Uri("https://example/uri"), "29e5c582-c1ef-4a5c-a053-d86c7381466b", new DateTime(2018, 11, 1), new List <string> { Schema.DataTypes.PackageDetails.AbsoluteUri }, new List <Uri> { Schema.DataTypes.PackageDetails }, new PackageIdentity(_packageId, parsedVersion)); _leaf = new PackageDetailsCatalogLeaf { PackageId = _packageId, PackageVersion = _commitItem.PackageIdentity.Version.ToFullString(), VerbatimVersion = _commitItem.PackageIdentity.Version.OriginalVersion, IsPrerelease = parsedVersion.IsPrerelease, Listed = true, }; _latestEntries = new List <CatalogCommitItem> { _commitItem }; _entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>( ReferenceEqualityComparer <CatalogCommitItem> .Default) { { _commitItem, _leaf }, }; }
public async Task DowngradeToUnlist() { var existingVersion = "0.0.1"; var existingLeaf = new PackageDetailsCatalogLeaf { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/0.0.1", PackageId = _packageId, VerbatimVersion = existingVersion, PackageVersion = existingVersion, Listed = false, }; _leaf.Listed = false; _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, }), _versionListDataResult.AccessCondition); _latestCatalogLeaves = new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse(existingVersion), existingLeaf }, }); var indexActions = await _target.AddCatalogEntriesAsync( _packageId, _latestEntries, _entryToLeaf); Assert.Equal(4, indexActions.Search.Count); Assert.All(indexActions.Search, x => Assert.IsType <KeyedDocument>(x.Document)); Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Delete, x.ActionType)); Assert.Equal(2, indexActions.Hijack.Count); var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); Assert.IsType <HijackDocument.Full>(existing.Document); Assert.Equal(IndexActionType.MergeOrUpload, existing.ActionType); var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); Assert.IsType <HijackDocument.Full>(added.Document); Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); var properties = indexActions.VersionListDataResult.Result.VersionProperties; Assert.Equal( new[] { existingVersion, _packageVersion }, properties.Keys.ToArray()); Assert.False(properties[existingVersion].Listed); Assert.False(properties[existingVersion].SemVer2); Assert.False(properties[_packageVersion].Listed); Assert.False(properties[_packageVersion].SemVer2); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>()), Times.Never); }
public SearchDocument.UpdateLatest UpdateLatestFromCatalog( SearchFilters searchFilters, string[] versions, bool isLatestStable, bool isLatest, string normalizedVersion, string fullVersion, PackageDetailsCatalogLeaf leaf, string[] owners) { var document = new SearchDocument.UpdateLatest(); // Determine if we have packageTypes to forward. // Otherwise, we need to let the system know that there were no explicit package types var packageTypes = leaf.PackageTypes != null && leaf.PackageTypes.Count > 0 ? leaf.PackageTypes.Select(pt => pt.Name).ToArray() : null; PopulateUpdateLatest( document, leaf.PackageId, searchFilters, lastUpdatedFromCatalog: true, lastCommitTimestamp: leaf.CommitTimestamp, lastCommitId: leaf.CommitId, versions: versions, isLatestStable: isLatestStable, isLatest: isLatest, fullVersion: fullVersion, owners: owners, packageTypes: packageTypes); _baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf); return(document); }
public void AddShortFormOfDependencyVersionRange(string input, string expected) { var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, DependencyGroups = new List <PackageDependencyGroup> { new PackageDependencyGroup { TargetFramework = "net40", Dependencies = new List <Protocol.Catalog.PackageDependency> { new Protocol.Catalog.PackageDependency { Id = "NuGet.Versioning", Range = input } }, }, }, }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal("NuGet.Versioning:" + expected + ":net40", full.FlattenedDependencies); }
public void DoesNotIncludeDependencyVersionSpecialFrameworks(string framework) { var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, DependencyGroups = new List <PackageDependencyGroup> { new PackageDependencyGroup { TargetFramework = framework, Dependencies = new List <Protocol.Catalog.PackageDependency> { new Protocol.Catalog.PackageDependency { Id = "NuGet.Versioning", Range = "2.0.0", }, new Protocol.Catalog.PackageDependency { Id = "NuGet.Frameworks", Range = "3.0.0", }, }, }, }, }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal("NuGet.Versioning:2.0.0|NuGet.Frameworks:3.0.0", full.FlattenedDependencies); }
private async Task IndexPackageAsync( PackageDetailsCatalogLeaf catalogLeaf, CancellationToken cancellationToken) { var packageId = catalogLeaf.PackageId; var packageVersion = catalogLeaf.ParsePackageVersion(); Package package; Stream packageStream = null; Stream readmeStream = null; try { var contentClient = _clientFactory.CreatePackageContentClient(); using (var stream = await contentClient.GetPackageContentStreamOrNullAsync(packageId, packageVersion, cancellationToken)) { packageStream = await stream.AsTemporaryFileStreamAsync(cancellationToken); } _logger.LogInformation( "Downloaded package {PackageId} {PackageVersion}, building metadata...", packageId, packageVersion); using (var reader = new PackageArchiveReader(packageStream, leaveStreamOpen: true)) { package = reader.GetPackageMetadata(); package.Listed = catalogLeaf.IsListed(); package.Published = catalogLeaf.Published.UtcDateTime; if (package.HasReadme) { using (var stream = await reader.GetReadmeAsync(cancellationToken)) { readmeStream = await stream.AsTemporaryFileStreamAsync(cancellationToken); } } await IndexPackageAsync(package, readmeStream, cancellationToken); } } catch (Exception e) { _logger.LogError( e, "Failed to process package {PackageId} {PackageVersion}", packageId, packageVersion); throw; } finally { packageStream?.Dispose(); readmeStream?.Dispose(); } }
public async Task PushOperationLogsCorrectLag() { var currentTime = DateTimeOffset.UtcNow; PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf { PackageId = "Test", PackageVersion = "1.0.0", Created = currentTime + TimeSpan.FromSeconds(50), LastEdited = currentTime + TimeSpan.FromSeconds(100), Listed = true }; var oldSearchResponse = TestHelpers.GetEmptyTestSearchResponse(currentTime); var newTime = currentTime + TimeSpan.FromSeconds(200); var newCreatedTime = listPackageLeaf.Created; var newLastEditedTime = listPackageLeaf.LastEdited; var expectedLag = newTime - listPackageLeaf.Created; var newSearchResponse = TestHelpers.GetTestSearchResponse(newTime, newCreatedTime, newLastEditedTime); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(newSearchResponse)); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(newSearchResponse)); _searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(newTime)); _telemetryService .Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Verifiable(); _telemetryService .Setup(ts => ts.TrackV3Lag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Verifiable(); try { var lag = await _target.ProcessPackageLagDetailsAsync(listPackageLeaf, listPackageLeaf.Created, listPackageLeaf.LastEdited, expectListed : true, isDelete : false); Assert.True(await _target.WaitForProcessing()); Assert.Equal(expectedLag, lag); _searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); _searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); _searchServiceClient.Verify(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); _telemetryService.Verify(); } catch (Exception e) { throw e; } }
private CatalogCommitItem GetPackageDetailsEntry(NuGetVersion version, PackageDetailsCatalogLeaf leaf) { return(new CatalogCommitItem( uri: new Uri(leaf.Url, UriKind.Absolute), commitId: leaf.CommitId, commitTimeStamp: leaf.CommitTimestamp.UtcDateTime, types: new string[0], typeUris: new[] { Schema.DataTypes.PackageDetails }, packageIdentity: new PackageIdentity(leaf.PackageId, version))); }
public async Task QueryAbandonedIfRetryLimitReached() { var currentTime = DateTimeOffset.UtcNow; PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf { PackageId = "Test", PackageVersion = "1.0.0", Created = currentTime, LastEdited = currentTime, Listed = true }; var oldSearchResponse = TestHelpers.GetEmptyTestSearchResponse(currentTime); var newTime = currentTime + TimeSpan.FromSeconds(200); var newSearchResponse = TestHelpers.GetTestSearchResponse(newTime, currentTime, currentTime); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(newSearchResponse)); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(newSearchResponse)); _searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>())) .Throws(new Exception("Unexpected call to get reload time")); _telemetryService .Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Throws(new Exception("Unexpected Logging")); _telemetryService .Setup(ts => ts.TrackV3Lag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Throws(new Exception("Unexpected Logging")); _target.RetryLimit = 2; try { var lag = await _target.ProcessPackageLagDetailsAsync(listPackageLeaf, listPackageLeaf.Created, listPackageLeaf.LastEdited, expectListed : true, isDelete : false); Assert.Null(lag); _searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(_target.RetryLimit)); _searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(_target.RetryLimit)); _telemetryService.Verify(); } catch (Exception e) { throw e; } }
public async Task ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf, CancellationToken token) { var exists = await _context.PackageDetailsCatalogLeafs.AnyAsync( p => p.PackageId == leaf.PackageId && p.PackageVersion == leaf.PackageVersion, token); if (!exists) { _context.PackageDetailsCatalogLeafs.Add(leaf); await Save(token); } }
public void AddPackageDetails(string version) { var parsedVersion = NuGetVersion.Parse(version); var catalogCommitItem = GetCatalogCommitItem(parsedVersion, Schema.DataTypes.PackageDetails); Entries.Add(catalogCommitItem); EntryToCatalogLeaf[catalogCommitItem] = new PackageDetailsCatalogLeaf { PackageVersion = version, }; }
private string VerifyConsistencyAndNormalizeVersion( Context context, PackageDetailsCatalogLeaf leaf) { if (!StringComparer.OrdinalIgnoreCase.Equals(context.PackageId, leaf.PackageId)) { const string message = "The package ID found in the catalog package does not match the catalog leaf."; _logger.LogError( message + " Page ID: {PagePackageId}, leaf ID: {LeafPackageId}, leaf URL: {Url}", context.PackageId, leaf.PackageId, leaf.Url); throw new InvalidOperationException(message); } var parsedPackageVersion = leaf.ParsePackageVersion(); var normalizedVersion = parsedPackageVersion.ToNormalizedString(); if (leaf.VerbatimVersion != null) { var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion); if (normalizedVersion != parsedVerbatimVersion.ToNormalizedString()) { const string message = "The normalized versions from the package version and the verbatim version do not match."; _logger.LogError( message + " ID: {PackageId}, version: {PackageVersion}, verbatim: {VerbatimVersion}, leaf URL: {Url}", leaf.PackageId, leaf.PackageVersion, leaf.VerbatimVersion, leaf.Url); throw new InvalidOperationException(message); } } if (parsedPackageVersion.IsPrerelease != leaf.IsPrerelease) { var message = $"The {nameof(PackageDetailsCatalogLeaf.IsPrerelease)} from the leaf does not match the version. " + $"Using the value from the parsed version. "; _logger.LogWarning( message + " ID: {PackageId}, version: {PackageVersion}, leaf is prerelease: {LeafIsPrerelease}, " + "parsed is prerelease: {ParsedIsPrerelease}, leaf URL: {Url}", leaf.PackageId, leaf.PackageVersion, leaf.IsPrerelease, parsedPackageVersion.IsPrerelease, leaf.Url); leaf.IsPrerelease = parsedPackageVersion.IsPrerelease; } return(normalizedVersion); }
public void PopulateMetadata( IBaseMetadataDocument document, string normalizedVersion, PackageDetailsCatalogLeaf leaf) { document.Authors = leaf.Authors; document.Copyright = leaf.Copyright; document.Created = leaf.Created; document.Description = leaf.Description; document.FileSize = leaf.PackageSize; document.FlattenedDependencies = GetFlattenedDependencies(leaf); document.Hash = leaf.PackageHash; document.HashAlgorithm = leaf.PackageHashAlgorithm; document.Language = leaf.Language; document.LastEdited = leaf.LastEdited; document.MinClientVersion = leaf.MinClientVersion; document.NormalizedVersion = normalizedVersion; document.OriginalVersion = leaf.VerbatimVersion; document.PackageId = leaf.PackageId; document.Prerelease = leaf.IsPrerelease; document.ProjectUrl = leaf.ProjectUrl; document.Published = leaf.Published; document.ReleaseNotes = leaf.ReleaseNotes; document.RequiresLicenseAcceptance = leaf.RequireLicenseAcceptance; document.SemVerLevel = leaf.IsSemVer2() ? SemVerLevelKey.SemVer2 : SemVerLevelKey.Unknown; document.SortableTitle = GetSortableTitle(leaf.Title, leaf.PackageId); document.Summary = leaf.Summary; document.Tags = leaf.Tags == null ? null : leaf.Tags.ToArray(); document.Title = GetTitle(leaf.Title, leaf.PackageId); document.TokenizedPackageId = leaf.PackageId; if (leaf.LicenseExpression != null || leaf.LicenseFile != null) { document.LicenseUrl = LicenseHelper.GetGalleryLicenseUrl( document.PackageId, normalizedVersion, _options.Value.ParseGalleryBaseUrl()); } else { document.LicenseUrl = leaf.LicenseUrl; } if (leaf.IconFile != null || (!string.IsNullOrWhiteSpace(leaf.IconUrl) && _options.Value.AllIconsInFlatContainer)) { SetIconUrlFromFlatContainer(document); } else { document.IconUrl = leaf.IconUrl; } }
/// <summary> /// Determines if the provided package details leaf represents a listed package. /// </summary> /// <param name="leaf">The catalog leaf.</param> /// <returns>True if the package is listed.</returns> public static bool IsListed(this PackageDetailsCatalogLeaf leaf) { if (leaf.Listed.HasValue) { return(leaf.Listed.Value); } // A published year of 1900 indicates that this package is unlisted, when the listed property itself is // not present (legacy behavior). // Example: https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json return(leaf.Published.Year != 1900); }
public void AllowNullDependencyGroups() { var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, DependencyGroups = null }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Null(full.FlattenedDependencies); }
public void IfLeafDoesNotHaveAnyIconFile_NoIconUrlIsSet() { Config.AllIconsInFlatContainer = true; var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Null(full.IconUrl); }
public void IfLeafDoesNotHaveIconFile_UsesIconUrl() { var iconUrl = "iconUrl"; var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, IconUrl = iconUrl }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal(iconUrl, full.IconUrl); }
public void IfLeafDoesNotHaveIconFileButHasUrl_UsesFlatContainerIfConfigured() { Config.AllIconsInFlatContainer = true; var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, IconUrl = "iconUrl" }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl); }
public void IfLeafHasIconFile_LinksToFlatContainer() { var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, IconFile = "iconFile", IconUrl = "iconUrl" }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl); }
private async Task <bool> ProcessPackageDetailsViaHttpAsync( HttpClient client, string id, string version, Uri sourceUri, PackageDetailsCatalogLeaf catalogLeaf, Dictionary <string, string> telemetryProperties, CancellationToken cancellationToken) { var packageDownloader = new PackageDownloader(client, _logger); var requestUri = Utilities.GetNugetCacheBustingUri(sourceUri); using (var stream = await packageDownloader.DownloadAsync(requestUri, cancellationToken)) { if (stream == null) { _logger.LogWarning("Package {Id}/{Version} not found.", id, version); return(false); } telemetryProperties[TelemetryConstants.SizeInBytes] = stream.Length.ToString(); var nuspec = GetNuspec(stream, id); if (nuspec == null) { _logger.LogWarning("No .nuspec available for {Id}/{Version}. Skipping.", id, version); return(false); } stream.Position = 0; await _dnxMaker.AddPackageAsync( stream, nuspec, id, version, catalogLeaf.IconFile, cancellationToken); } _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", id, version); return(true); }
public async Task ListOperationDoesNotLogCreationLag() { var currentTime = DateTimeOffset.UtcNow; PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf { PackageId = "Test", PackageVersion = "1.0.0", Created = currentTime, LastEdited = currentTime, Listed = true }; var oldSearchResponse = TestHelpers.GetTestSearchResponse(currentTime, currentTime - TimeSpan.FromSeconds(200), currentTime - TimeSpan.FromSeconds(200), false); var newTime = currentTime + TimeSpan.FromSeconds(200); var newSearchResponse = TestHelpers.GetTestSearchResponse(newTime, newTime, newTime); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(newSearchResponse)); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(oldSearchResponse)) .Returns(Task.FromResult(newSearchResponse)); _searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(newTime)); _telemetryService .Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Throws(new Exception("Unexpected Logging")); try { var success = await _target.ProcessPackageDetailsAsync(listPackageLeaf); Assert.True(await _target.WaitForProcessing()); _searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); _searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); _searchServiceClient.Verify(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()), Times.Exactly(2)); } catch (Exception e) { throw e; } }
/// <summary> /// Determines if the provied package details leaf represents a SemVer 2.0.0 package. A package is considered /// SemVer 2.0.0 if it's version is SemVer 2.0.0 or one of its dependency version ranges is SemVer 2.0.0. /// </summary> /// <param name="leaf">The catalog leaf.</param> /// <returns>True if the package is SemVer 2.0.0.</returns> public static bool IsSemVer2(this PackageDetailsCatalogLeaf leaf) { var parsedPackageVersion = leaf.ParsePackageVersion(); if (parsedPackageVersion.IsSemVer2) { return(true); } if (leaf.VerbatimVersion != null) { var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion); if (parsedVerbatimVersion.IsSemVer2) { return(true); } } if (leaf.DependencyGroups != null) { foreach (var dependencyGroup in leaf.DependencyGroups) { // Example: https://api.nuget.org/v3/catalog0/data/2018.10.28.07.42.42/mvcsitemapprovider.3.3.0-pre1.json if (dependencyGroup.Dependencies == null) { continue; } foreach (var dependency in dependencyGroup.Dependencies) { var versionRange = dependency.ParseRange(); if ((versionRange.MaxVersion != null && versionRange.MaxVersion.IsSemVer2) || (versionRange.MinVersion != null && versionRange.MinVersion.IsSemVer2)) { return(true); } } } } return(false); }
public HijackDocument.Full FullFromCatalog( string normalizedVersion, HijackDocumentChanges changes, PackageDetailsCatalogLeaf leaf) { var document = new HijackDocument.Full(); PopulateLatest( document, leaf.PackageId, normalizedVersion, lastUpdatedFromCatalog: true, lastCommitTimestamp: leaf.CommitTimestamp, lastCommitId: leaf.CommitId, changes: changes); _baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf); document.Listed = leaf.IsListed(); return(document); }
public void AddsEmptyDependencyGroup(string framework, string expected) { var leaf = new PackageDetailsCatalogLeaf { PackageId = Data.PackageId, PackageVersion = Data.NormalizedVersion, DependencyGroups = new List <PackageDependencyGroup> { new PackageDependencyGroup { TargetFramework = framework, }, }, }; var full = new HijackDocument.Full(); Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); Assert.Equal(expected, full.FlattenedDependencies); }
public async Task NoInstancesDoesNotLog() { var currentTime = DateTimeOffset.UtcNow; PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf { PackageId = "Test", PackageVersion = "1.0.0", Created = currentTime, LastEdited = currentTime, Listed = true }; var emptyInstances = new List <Instance>(); var newTarget = new PackageLagCatalogLeafProcessor(emptyInstances, _searchServiceClient.Object, _telemetryService.Object, _logger); _searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Throws(new Exception("Unexpectd call to get search result")); _searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>())) .Throws(new Exception("Unexpected call to get reload time")); _telemetryService .Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Throws(new Exception("Unexpected Logging")); _telemetryService .Setup(ts => ts.TrackV3Lag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>())) .Throws(new Exception("Unexpected Logging")); try { var lag = await _target.ProcessPackageLagDetailsAsync(listPackageLeaf, listPackageLeaf.Created, listPackageLeaf.LastEdited, expectListed : true, isDelete : false); Assert.Null(lag); } catch (Exception e) { throw e; } }
public async Task Search404MergeFailureIsApplicable() { ItemList.Add(new CatalogCommitItem( new Uri("https://example/catalog/0.json"), "commit-id-a", new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc), new string[0], new[] { Schema.DataTypes.PackageDetails }, new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("1.0.0")))); ItemList.Add(new CatalogCommitItem( new Uri("https://example/catalog/1.json"), "commit-id-a", new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc), new string[0], new[] { Schema.DataTypes.PackageDetails }, new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("0.9.0-beta.1")))); IndexingResults.Add(new IndexingResult(key: "search-doc", statusCode: 404)); AllIndexActions.Add(new IdAndValue <IndexActions>( "NuGet.Versioning", new IndexActions( search: new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument { Key = "search-doc" }), }, hijack: new List <IndexAction <KeyedDocument> >(), versionListDataResult: new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), Mock.Of <IAccessCondition>())))); VersionListClient .Setup(x => x.ReadAsync(It.IsAny <string>())) .ReturnsAsync(() => new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { "1.0.0", new VersionPropertiesData(listed: true, semVer2: false) }, }), Mock.Of <IAccessCondition>())); var leaf = new PackageDetailsCatalogLeaf { Url = "https://example/catalog/2.json", CommitId = "commit-id", CommitTimestamp = new DateTimeOffset(2020, 3, 17, 12, 5, 0, TimeSpan.Zero), Type = CatalogLeafType.PackageDetails, }; LeafFetcher .Setup(x => x.GetLatestLeavesAsync( It.IsAny <string>(), It.IsAny <IReadOnlyList <IReadOnlyList <NuGetVersion> > >())) .ReturnsAsync(() => new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse("1.0.0"), leaf }, })); var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); Assert.True(result.Applicable, "The fix up should be applicable."); Assert.Equal(3, result.ItemList.Count); Assert.Empty(ItemList.Except(result.ItemList)); var addedItem = Assert.Single(result.ItemList.Except(ItemList)); Assert.Equal(leaf.Url, addedItem.Uri.AbsoluteUri); Assert.Equal(leaf.CommitId, addedItem.CommitId); Assert.Equal(leaf.CommitTimestamp, addedItem.CommitTimeStamp); Assert.Empty(addedItem.Types); Assert.Equal(Schema.DataTypes.PackageDetails, Assert.Single(addedItem.TypeUris)); Assert.Equal(new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0")), addedItem.PackageIdentity); Assert.True(addedItem.IsPackageDetails, "The generated item should be a package details item."); Assert.False(addedItem.IsPackageDelete, "The generated item should not be a package delete item."); }