public Facts()
{
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Config = new Catalog2RegistrationConfiguration
{
LegacyBaseUrl = "https://example/reg/",
GzippedBaseUrl = "https://example/reg-gz/",
SemVer2BaseUrl = "https://example/reg-gz-semver2/",
GalleryBaseUrl = "https://example-gallery/",
FlatContainerBaseUrl = "https://example/fc/",
};
Options.Setup(x => x.Value).Returns(() => Config);
LeafItem = new RegistrationLeafItem();
Page = new RegistrationPage();
Index = new RegistrationIndex();
Hive = HiveType.Legacy;
Id = V3Data.PackageId;
PackageDetails = new PackageDetailsCatalogLeaf
{
PackageVersion = V3Data.NormalizedVersion,
};
Commit = new CatalogCommit(V3Data.CommitId, V3Data.CommitTimestamp);
SerializerSettings = NuGetJsonSerialization.Settings;
SerializerSettings.Formatting = Formatting.Indented;
}
public void AddsEmptyStringForDependencyVersionAllRange()
{
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
DependencyGroups = new List <PackageDependencyGroup>
{
new PackageDependencyGroup
{
TargetFramework = "net40",
Dependencies = new List <Protocol.Catalog.PackageDependency>
{
new Protocol.Catalog.PackageDependency
{
Id = "NuGet.Versioning",
Range = "(, )"
}
},
},
},
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal("NuGet.Versioning::net40", full.FlattenedDependencies);
}
/// <summary>
/// This method produces output for official client. The implementation on the client side, at one point of time
/// was:
/// https://github.com/NuGet/NuGet.Client/blob/b404acf6eb88c2b6086a9cbb5106104534de2428/src/NuGet.Core/NuGet.Protocol/LegacyFeed/V2FeedPackageInfo.cs#L228-L308
///
/// The output is a string where each dependency is separated by a "|" character and information about each
/// dependency is separated by a ":" character. Per dependency, the colon-seperated data is a pair or triple.
/// The three fields in order are: dependency package ID, version range, and target framework. If third value
/// (the framework that the dependency targets) is an empty string or excluded, this means the dependency
/// applies to any framework. If the package ID and range and empty strings but the framework is included, this
/// means that the package supports the specified framework but has no dependencies specific to that framework.
///
/// Output:
/// [DEPENDENCY[|DEPENDENCY][|...][|DEPENDENCY]]
/// Each dependency:
/// [PACKAGE_ID]:[VERSION_RANGE][:TARGET_FRAMEWORK]
///
/// Example A (no target frameworks):
/// Microsoft.Data.OData:5.0.2|Microsoft.WindowsAzure.ConfigurationManager:1.8.0
/// Example B (target frameworks):
/// NETStandard.Library:1.6.1:netstandard1.0|Newtonsoft.Json:10.0.2:netstandard1.0
/// Example D (empty target framework):
/// Microsoft.Data.OData:5.0.2:
/// Example D (just target framework):
/// ::net20|::net35|::net40|::net45|NETStandard.Library:1.6.1:netstandard1.0
/// </summary>
private static string GetFlattenedDependencies(PackageDetailsCatalogLeaf leaf)
{
if (leaf.DependencyGroups == null)
{
return(null);
}
var builder = new StringBuilder();
foreach (var dependencyGroup in leaf.DependencyGroups)
{
var targetFramework = dependencyGroup.ParseTargetFramework();
if (dependencyGroup.Dependencies != null && dependencyGroup.Dependencies.Any())
{
foreach (var packageDependency in dependencyGroup.Dependencies)
{
AddFlattenedPackageDependency(targetFramework, packageDependency, builder);
}
}
else
{
if (builder.Length > 0)
{
builder.Append("|");
}
builder.Append(":");
AddFlattenedFrameworkDependency(targetFramework, builder);
}
}
return(builder.Length > 0 ? builder.ToString() : null);
}
public Task <bool> ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf)
{
Console.WriteLine(
$"{leaf.CommitTimestamp:O}: Found package details leaf for {leaf.PackageId} {leaf.PackageVersion}.");
return(GetResultAsync(leaf));
}
protected void SetVersion(string version)
{
var parsedVersion = NuGetVersion.Parse(version);
_packageVersion = version;
_commitItem = new CatalogCommitItem(
new Uri("https://example/uri"),
"29e5c582-c1ef-4a5c-a053-d86c7381466b",
new DateTime(2018, 11, 1),
new List <string> {
Schema.DataTypes.PackageDetails.AbsoluteUri
},
new List <Uri> {
Schema.DataTypes.PackageDetails
},
new PackageIdentity(_packageId, parsedVersion));
_leaf = new PackageDetailsCatalogLeaf
{
PackageId = _packageId,
PackageVersion = _commitItem.PackageIdentity.Version.ToFullString(),
VerbatimVersion = _commitItem.PackageIdentity.Version.OriginalVersion,
IsPrerelease = parsedVersion.IsPrerelease,
Listed = true,
};
_latestEntries = new List <CatalogCommitItem> {
_commitItem
};
_entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default)
{
{ _commitItem, _leaf },
};
}
public async Task DowngradeToUnlist()
{
var existingVersion = "0.0.1";
var existingLeaf = new PackageDetailsCatalogLeaf
{
CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero),
Url = "http://example/leaf/0.0.1",
PackageId = _packageId,
VerbatimVersion = existingVersion,
PackageVersion = existingVersion,
Listed = false,
};
_leaf.Listed = false;
_versionListDataResult = new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>
{
{ existingVersion, new VersionPropertiesData(listed: true, semVer2: false) },
{ _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) },
}),
_versionListDataResult.AccessCondition);
_latestCatalogLeaves = new LatestCatalogLeaves(
new HashSet <NuGetVersion>(),
new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf>
{
{ NuGetVersion.Parse(existingVersion), existingLeaf },
});
var indexActions = await _target.AddCatalogEntriesAsync(
_packageId,
_latestEntries,
_entryToLeaf);
Assert.Equal(4, indexActions.Search.Count);
Assert.All(indexActions.Search, x => Assert.IsType <KeyedDocument>(x.Document));
Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Delete, x.ActionType));
Assert.Equal(2, indexActions.Hijack.Count);
var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion);
Assert.IsType <HijackDocument.Full>(existing.Document);
Assert.Equal(IndexActionType.MergeOrUpload, existing.ActionType);
var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion);
Assert.IsType <HijackDocument.Full>(added.Document);
Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType);
Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition);
var properties = indexActions.VersionListDataResult.Result.VersionProperties;
Assert.Equal(
new[] { existingVersion, _packageVersion },
properties.Keys.ToArray());
Assert.False(properties[existingVersion].Listed);
Assert.False(properties[existingVersion].SemVer2);
Assert.False(properties[_packageVersion].Listed);
Assert.False(properties[_packageVersion].SemVer2);
_ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>()), Times.Never);
}
public SearchDocument.UpdateLatest UpdateLatestFromCatalog(
SearchFilters searchFilters,
string[] versions,
bool isLatestStable,
bool isLatest,
string normalizedVersion,
string fullVersion,
PackageDetailsCatalogLeaf leaf,
string[] owners)
{
var document = new SearchDocument.UpdateLatest();
// Determine if we have packageTypes to forward.
// Otherwise, we need to let the system know that there were no explicit package types
var packageTypes = leaf.PackageTypes != null && leaf.PackageTypes.Count > 0 ?
leaf.PackageTypes.Select(pt => pt.Name).ToArray() :
null;
PopulateUpdateLatest(
document,
leaf.PackageId,
searchFilters,
lastUpdatedFromCatalog: true,
lastCommitTimestamp: leaf.CommitTimestamp,
lastCommitId: leaf.CommitId,
versions: versions,
isLatestStable: isLatestStable,
isLatest: isLatest,
fullVersion: fullVersion,
owners: owners,
packageTypes: packageTypes);
_baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf);
return(document);
}
public void AddShortFormOfDependencyVersionRange(string input, string expected)
{
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
DependencyGroups = new List <PackageDependencyGroup>
{
new PackageDependencyGroup
{
TargetFramework = "net40",
Dependencies = new List <Protocol.Catalog.PackageDependency>
{
new Protocol.Catalog.PackageDependency
{
Id = "NuGet.Versioning",
Range = input
}
},
},
},
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal("NuGet.Versioning:" + expected + ":net40", full.FlattenedDependencies);
}
public void DoesNotIncludeDependencyVersionSpecialFrameworks(string framework)
{
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
DependencyGroups = new List <PackageDependencyGroup>
{
new PackageDependencyGroup
{
TargetFramework = framework,
Dependencies = new List <Protocol.Catalog.PackageDependency>
{
new Protocol.Catalog.PackageDependency
{
Id = "NuGet.Versioning",
Range = "2.0.0",
},
new Protocol.Catalog.PackageDependency
{
Id = "NuGet.Frameworks",
Range = "3.0.0",
},
},
},
},
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal("NuGet.Versioning:2.0.0|NuGet.Frameworks:3.0.0", full.FlattenedDependencies);
}
private async Task IndexPackageAsync(
PackageDetailsCatalogLeaf catalogLeaf,
CancellationToken cancellationToken)
{
var packageId = catalogLeaf.PackageId;
var packageVersion = catalogLeaf.ParsePackageVersion();
Package package;
Stream packageStream = null;
Stream readmeStream = null;
try
{
var contentClient = _clientFactory.CreatePackageContentClient();
using (var stream = await contentClient.GetPackageContentStreamOrNullAsync(packageId, packageVersion, cancellationToken))
{
packageStream = await stream.AsTemporaryFileStreamAsync(cancellationToken);
}
_logger.LogInformation(
"Downloaded package {PackageId} {PackageVersion}, building metadata...",
packageId,
packageVersion);
using (var reader = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
{
package = reader.GetPackageMetadata();
package.Listed = catalogLeaf.IsListed();
package.Published = catalogLeaf.Published.UtcDateTime;
if (package.HasReadme)
{
using (var stream = await reader.GetReadmeAsync(cancellationToken))
{
readmeStream = await stream.AsTemporaryFileStreamAsync(cancellationToken);
}
}
await IndexPackageAsync(package, readmeStream, cancellationToken);
}
}
catch (Exception e)
{
_logger.LogError(
e,
"Failed to process package {PackageId} {PackageVersion}",
packageId,
packageVersion);
throw;
}
finally
{
packageStream?.Dispose();
readmeStream?.Dispose();
}
}
public async Task PushOperationLogsCorrectLag()
{
var currentTime = DateTimeOffset.UtcNow;
PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf
{
PackageId = "Test",
PackageVersion = "1.0.0",
Created = currentTime + TimeSpan.FromSeconds(50),
LastEdited = currentTime + TimeSpan.FromSeconds(100),
Listed = true
};
var oldSearchResponse = TestHelpers.GetEmptyTestSearchResponse(currentTime);
var newTime = currentTime + TimeSpan.FromSeconds(200);
var newCreatedTime = listPackageLeaf.Created;
var newLastEditedTime = listPackageLeaf.LastEdited;
var expectedLag = newTime - listPackageLeaf.Created;
var newSearchResponse = TestHelpers.GetTestSearchResponse(newTime, newCreatedTime, newLastEditedTime);
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(newSearchResponse));
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(newSearchResponse));
_searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(newTime));
_telemetryService
.Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Verifiable();
_telemetryService
.Setup(ts => ts.TrackV3Lag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Verifiable();
try
{
var lag = await _target.ProcessPackageLagDetailsAsync(listPackageLeaf, listPackageLeaf.Created, listPackageLeaf.LastEdited, expectListed : true, isDelete : false);
Assert.True(await _target.WaitForProcessing());
Assert.Equal(expectedLag, lag);
_searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2));
_searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2));
_searchServiceClient.Verify(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()), Times.Exactly(2));
_telemetryService.Verify();
}
catch (Exception e)
{
throw e;
}
}
private CatalogCommitItem GetPackageDetailsEntry(NuGetVersion version, PackageDetailsCatalogLeaf leaf)
{
return(new CatalogCommitItem(
uri: new Uri(leaf.Url, UriKind.Absolute),
commitId: leaf.CommitId,
commitTimeStamp: leaf.CommitTimestamp.UtcDateTime,
types: new string[0],
typeUris: new[] { Schema.DataTypes.PackageDetails },
packageIdentity: new PackageIdentity(leaf.PackageId, version)));
}
public async Task QueryAbandonedIfRetryLimitReached()
{
var currentTime = DateTimeOffset.UtcNow;
PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf
{
PackageId = "Test",
PackageVersion = "1.0.0",
Created = currentTime,
LastEdited = currentTime,
Listed = true
};
var oldSearchResponse = TestHelpers.GetEmptyTestSearchResponse(currentTime);
var newTime = currentTime + TimeSpan.FromSeconds(200);
var newSearchResponse = TestHelpers.GetTestSearchResponse(newTime, currentTime, currentTime);
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(newSearchResponse));
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(newSearchResponse));
_searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()))
.Throws(new Exception("Unexpected call to get reload time"));
_telemetryService
.Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Throws(new Exception("Unexpected Logging"));
_telemetryService
.Setup(ts => ts.TrackV3Lag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Throws(new Exception("Unexpected Logging"));
_target.RetryLimit = 2;
try
{
var lag = await _target.ProcessPackageLagDetailsAsync(listPackageLeaf, listPackageLeaf.Created, listPackageLeaf.LastEdited, expectListed : true, isDelete : false);
Assert.Null(lag);
_searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(_target.RetryLimit));
_searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(_target.RetryLimit));
_telemetryService.Verify();
}
catch (Exception e)
{
throw e;
}
}
public async Task ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf, CancellationToken token)
{
var exists = await _context.PackageDetailsCatalogLeafs.AnyAsync(
p => p.PackageId == leaf.PackageId && p.PackageVersion == leaf.PackageVersion, token);
if (!exists)
{
_context.PackageDetailsCatalogLeafs.Add(leaf);
await Save(token);
}
}
public void AddPackageDetails(string version)
{
var parsedVersion = NuGetVersion.Parse(version);
var catalogCommitItem = GetCatalogCommitItem(parsedVersion, Schema.DataTypes.PackageDetails);
Entries.Add(catalogCommitItem);
EntryToCatalogLeaf[catalogCommitItem] = new PackageDetailsCatalogLeaf
{
PackageVersion = version,
};
}
private string VerifyConsistencyAndNormalizeVersion(
Context context,
PackageDetailsCatalogLeaf leaf)
{
if (!StringComparer.OrdinalIgnoreCase.Equals(context.PackageId, leaf.PackageId))
{
const string message = "The package ID found in the catalog package does not match the catalog leaf.";
_logger.LogError(
message + " Page ID: {PagePackageId}, leaf ID: {LeafPackageId}, leaf URL: {Url}",
context.PackageId,
leaf.PackageId,
leaf.Url);
throw new InvalidOperationException(message);
}
var parsedPackageVersion = leaf.ParsePackageVersion();
var normalizedVersion = parsedPackageVersion.ToNormalizedString();
if (leaf.VerbatimVersion != null)
{
var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion);
if (normalizedVersion != parsedVerbatimVersion.ToNormalizedString())
{
const string message =
"The normalized versions from the package version and the verbatim version do not match.";
_logger.LogError(
message + " ID: {PackageId}, version: {PackageVersion}, verbatim: {VerbatimVersion}, leaf URL: {Url}",
leaf.PackageId,
leaf.PackageVersion,
leaf.VerbatimVersion,
leaf.Url);
throw new InvalidOperationException(message);
}
}
if (parsedPackageVersion.IsPrerelease != leaf.IsPrerelease)
{
var message =
$"The {nameof(PackageDetailsCatalogLeaf.IsPrerelease)} from the leaf does not match the version. " +
$"Using the value from the parsed version. ";
_logger.LogWarning(
message + " ID: {PackageId}, version: {PackageVersion}, leaf is prerelease: {LeafIsPrerelease}, " +
"parsed is prerelease: {ParsedIsPrerelease}, leaf URL: {Url}",
leaf.PackageId,
leaf.PackageVersion,
leaf.IsPrerelease,
parsedPackageVersion.IsPrerelease,
leaf.Url);
leaf.IsPrerelease = parsedPackageVersion.IsPrerelease;
}
return(normalizedVersion);
}
public void PopulateMetadata(
IBaseMetadataDocument document,
string normalizedVersion,
PackageDetailsCatalogLeaf leaf)
{
document.Authors = leaf.Authors;
document.Copyright = leaf.Copyright;
document.Created = leaf.Created;
document.Description = leaf.Description;
document.FileSize = leaf.PackageSize;
document.FlattenedDependencies = GetFlattenedDependencies(leaf);
document.Hash = leaf.PackageHash;
document.HashAlgorithm = leaf.PackageHashAlgorithm;
document.Language = leaf.Language;
document.LastEdited = leaf.LastEdited;
document.MinClientVersion = leaf.MinClientVersion;
document.NormalizedVersion = normalizedVersion;
document.OriginalVersion = leaf.VerbatimVersion;
document.PackageId = leaf.PackageId;
document.Prerelease = leaf.IsPrerelease;
document.ProjectUrl = leaf.ProjectUrl;
document.Published = leaf.Published;
document.ReleaseNotes = leaf.ReleaseNotes;
document.RequiresLicenseAcceptance = leaf.RequireLicenseAcceptance;
document.SemVerLevel = leaf.IsSemVer2() ? SemVerLevelKey.SemVer2 : SemVerLevelKey.Unknown;
document.SortableTitle = GetSortableTitle(leaf.Title, leaf.PackageId);
document.Summary = leaf.Summary;
document.Tags = leaf.Tags == null ? null : leaf.Tags.ToArray();
document.Title = GetTitle(leaf.Title, leaf.PackageId);
document.TokenizedPackageId = leaf.PackageId;
if (leaf.LicenseExpression != null || leaf.LicenseFile != null)
{
document.LicenseUrl = LicenseHelper.GetGalleryLicenseUrl(
document.PackageId,
normalizedVersion,
_options.Value.ParseGalleryBaseUrl());
}
else
{
document.LicenseUrl = leaf.LicenseUrl;
}
if (leaf.IconFile != null ||
(!string.IsNullOrWhiteSpace(leaf.IconUrl) && _options.Value.AllIconsInFlatContainer))
{
SetIconUrlFromFlatContainer(document);
}
else
{
document.IconUrl = leaf.IconUrl;
}
}
/// <summary>
/// Determines if the provided package details leaf represents a listed package.
/// </summary>
/// <param name="leaf">The catalog leaf.</param>
/// <returns>True if the package is listed.</returns>
public static bool IsListed(this PackageDetailsCatalogLeaf leaf)
{
if (leaf.Listed.HasValue)
{
return(leaf.Listed.Value);
}
// A published year of 1900 indicates that this package is unlisted, when the listed property itself is
// not present (legacy behavior).
// Example: https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json
return(leaf.Published.Year != 1900);
}
public void AllowNullDependencyGroups()
{
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
DependencyGroups = null
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Null(full.FlattenedDependencies);
}
public void IfLeafDoesNotHaveAnyIconFile_NoIconUrlIsSet()
{
Config.AllIconsInFlatContainer = true;
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Null(full.IconUrl);
}
public void IfLeafDoesNotHaveIconFile_UsesIconUrl()
{
var iconUrl = "iconUrl";
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
IconUrl = iconUrl
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal(iconUrl, full.IconUrl);
}
public void IfLeafDoesNotHaveIconFileButHasUrl_UsesFlatContainerIfConfigured()
{
Config.AllIconsInFlatContainer = true;
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
IconUrl = "iconUrl"
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl);
}
public void IfLeafHasIconFile_LinksToFlatContainer()
{
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
IconFile = "iconFile",
IconUrl = "iconUrl"
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl);
}
private async Task <bool> ProcessPackageDetailsViaHttpAsync(
HttpClient client,
string id,
string version,
Uri sourceUri,
PackageDetailsCatalogLeaf catalogLeaf,
Dictionary <string, string> telemetryProperties,
CancellationToken cancellationToken)
{
var packageDownloader = new PackageDownloader(client, _logger);
var requestUri = Utilities.GetNugetCacheBustingUri(sourceUri);
using (var stream = await packageDownloader.DownloadAsync(requestUri, cancellationToken))
{
if (stream == null)
{
_logger.LogWarning("Package {Id}/{Version} not found.", id, version);
return(false);
}
telemetryProperties[TelemetryConstants.SizeInBytes] = stream.Length.ToString();
var nuspec = GetNuspec(stream, id);
if (nuspec == null)
{
_logger.LogWarning("No .nuspec available for {Id}/{Version}. Skipping.", id, version);
return(false);
}
stream.Position = 0;
await _dnxMaker.AddPackageAsync(
stream,
nuspec,
id,
version,
catalogLeaf.IconFile,
cancellationToken);
}
_logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", id, version);
return(true);
}
public async Task ListOperationDoesNotLogCreationLag()
{
var currentTime = DateTimeOffset.UtcNow;
PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf
{
PackageId = "Test",
PackageVersion = "1.0.0",
Created = currentTime,
LastEdited = currentTime,
Listed = true
};
var oldSearchResponse = TestHelpers.GetTestSearchResponse(currentTime, currentTime - TimeSpan.FromSeconds(200), currentTime - TimeSpan.FromSeconds(200), false);
var newTime = currentTime + TimeSpan.FromSeconds(200);
var newSearchResponse = TestHelpers.GetTestSearchResponse(newTime, newTime, newTime);
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(newSearchResponse));
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldSearchResponse))
.Returns(Task.FromResult(newSearchResponse));
_searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(newTime));
_telemetryService
.Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Throws(new Exception("Unexpected Logging"));
try
{
var success = await _target.ProcessPackageDetailsAsync(listPackageLeaf);
Assert.True(await _target.WaitForProcessing());
_searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("801")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2));
_searchServiceClient.Verify(ssc => ssc.GetResultForPackageIdVersion(It.Is <Instance>(i => i.DiagUrl.Contains("802")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Exactly(2));
_searchServiceClient.Verify(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()), Times.Exactly(2));
}
catch (Exception e)
{
throw e;
}
}
/// <summary>
/// Determines if the provied package details leaf represents a SemVer 2.0.0 package. A package is considered
/// SemVer 2.0.0 if it's version is SemVer 2.0.0 or one of its dependency version ranges is SemVer 2.0.0.
/// </summary>
/// <param name="leaf">The catalog leaf.</param>
/// <returns>True if the package is SemVer 2.0.0.</returns>
public static bool IsSemVer2(this PackageDetailsCatalogLeaf leaf)
{
var parsedPackageVersion = leaf.ParsePackageVersion();
if (parsedPackageVersion.IsSemVer2)
{
return(true);
}
if (leaf.VerbatimVersion != null)
{
var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion);
if (parsedVerbatimVersion.IsSemVer2)
{
return(true);
}
}
if (leaf.DependencyGroups != null)
{
foreach (var dependencyGroup in leaf.DependencyGroups)
{
// Example: https://api.nuget.org/v3/catalog0/data/2018.10.28.07.42.42/mvcsitemapprovider.3.3.0-pre1.json
if (dependencyGroup.Dependencies == null)
{
continue;
}
foreach (var dependency in dependencyGroup.Dependencies)
{
var versionRange = dependency.ParseRange();
if ((versionRange.MaxVersion != null && versionRange.MaxVersion.IsSemVer2) ||
(versionRange.MinVersion != null && versionRange.MinVersion.IsSemVer2))
{
return(true);
}
}
}
}
return(false);
}
public HijackDocument.Full FullFromCatalog(
string normalizedVersion,
HijackDocumentChanges changes,
PackageDetailsCatalogLeaf leaf)
{
var document = new HijackDocument.Full();
PopulateLatest(
document,
leaf.PackageId,
normalizedVersion,
lastUpdatedFromCatalog: true,
lastCommitTimestamp: leaf.CommitTimestamp,
lastCommitId: leaf.CommitId,
changes: changes);
_baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf);
document.Listed = leaf.IsListed();
return(document);
}
public void AddsEmptyDependencyGroup(string framework, string expected)
{
var leaf = new PackageDetailsCatalogLeaf
{
PackageId = Data.PackageId,
PackageVersion = Data.NormalizedVersion,
DependencyGroups = new List <PackageDependencyGroup>
{
new PackageDependencyGroup
{
TargetFramework = framework,
},
},
};
var full = new HijackDocument.Full();
Target.PopulateMetadata(full, Data.NormalizedVersion, leaf);
Assert.Equal(expected, full.FlattenedDependencies);
}
public async Task NoInstancesDoesNotLog()
{
var currentTime = DateTimeOffset.UtcNow;
PackageDetailsCatalogLeaf listPackageLeaf = new PackageDetailsCatalogLeaf
{
PackageId = "Test",
PackageVersion = "1.0.0",
Created = currentTime,
LastEdited = currentTime,
Listed = true
};
var emptyInstances = new List <Instance>();
var newTarget = new PackageLagCatalogLeafProcessor(emptyInstances, _searchServiceClient.Object, _telemetryService.Object, _logger);
_searchServiceClient.SetupSequence(ssc => ssc.GetResultForPackageIdVersion(It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.Throws(new Exception("Unexpectd call to get search result"));
_searchServiceClient.Setup(ssc => ssc.GetIndexLastReloadTimeAsync(It.IsAny <Instance>(), It.IsAny <CancellationToken>()))
.Throws(new Exception("Unexpected call to get reload time"));
_telemetryService
.Setup(ts => ts.TrackPackageCreationLag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Throws(new Exception("Unexpected Logging"));
_telemetryService
.Setup(ts => ts.TrackV3Lag(It.IsAny <DateTimeOffset>(), It.IsAny <Instance>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TimeSpan>()))
.Throws(new Exception("Unexpected Logging"));
try
{
var lag = await _target.ProcessPackageLagDetailsAsync(listPackageLeaf, listPackageLeaf.Created, listPackageLeaf.LastEdited, expectListed : true, isDelete : false);
Assert.Null(lag);
}
catch (Exception e)
{
throw e;
}
}
public async Task Search404MergeFailureIsApplicable()
{
ItemList.Add(new CatalogCommitItem(
new Uri("https://example/catalog/0.json"),
"commit-id-a",
new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc),
new string[0],
new[] { Schema.DataTypes.PackageDetails },
new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("1.0.0"))));
ItemList.Add(new CatalogCommitItem(
new Uri("https://example/catalog/1.json"),
"commit-id-a",
new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc),
new string[0],
new[] { Schema.DataTypes.PackageDetails },
new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("0.9.0-beta.1"))));
IndexingResults.Add(new IndexingResult(key: "search-doc", statusCode: 404));
AllIndexActions.Add(new IdAndValue <IndexActions>(
"NuGet.Versioning",
new IndexActions(
search: new List <IndexAction <KeyedDocument> >
{
IndexAction.Merge(new KeyedDocument {
Key = "search-doc"
}),
},
hijack: new List <IndexAction <KeyedDocument> >(),
versionListDataResult: new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
Mock.Of <IAccessCondition>()))));
VersionListClient
.Setup(x => x.ReadAsync(It.IsAny <string>()))
.ReturnsAsync(() => new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>
{
{ "1.0.0", new VersionPropertiesData(listed: true, semVer2: false) },
}),
Mock.Of <IAccessCondition>()));
var leaf = new PackageDetailsCatalogLeaf
{
Url = "https://example/catalog/2.json",
CommitId = "commit-id",
CommitTimestamp = new DateTimeOffset(2020, 3, 17, 12, 5, 0, TimeSpan.Zero),
Type = CatalogLeafType.PackageDetails,
};
LeafFetcher
.Setup(x => x.GetLatestLeavesAsync(
It.IsAny <string>(),
It.IsAny <IReadOnlyList <IReadOnlyList <NuGetVersion> > >()))
.ReturnsAsync(() => new LatestCatalogLeaves(
new HashSet <NuGetVersion>(),
new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf>
{
{ NuGetVersion.Parse("1.0.0"), leaf },
}));
var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception);
Assert.True(result.Applicable, "The fix up should be applicable.");
Assert.Equal(3, result.ItemList.Count);
Assert.Empty(ItemList.Except(result.ItemList));
var addedItem = Assert.Single(result.ItemList.Except(ItemList));
Assert.Equal(leaf.Url, addedItem.Uri.AbsoluteUri);
Assert.Equal(leaf.CommitId, addedItem.CommitId);
Assert.Equal(leaf.CommitTimestamp, addedItem.CommitTimeStamp);
Assert.Empty(addedItem.Types);
Assert.Equal(Schema.DataTypes.PackageDetails, Assert.Single(addedItem.TypeUris));
Assert.Equal(new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0")), addedItem.PackageIdentity);
Assert.True(addedItem.IsPackageDetails, "The generated item should be a package details item.");
Assert.False(addedItem.IsPackageDelete, "The generated item should not be a package delete item.");
}
