public async Task <JObject> CreatePackageBlob(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, new string[] { "Package", "http://schema.nuget.org/catalog#Permalink" });
var packageDetailsFile = _context.Source.Get(packageInput.PackageDetailsUri);
if (!await packageDetailsFile.Exists(_context.Log, _context.Token))
{
throw new FileNotFoundException($"Unable to find {packageDetailsFile.EntityUri.AbsoluteUri}");
}
var detailsJson = await packageDetailsFile.GetJson(_context.Log, _context.Token);
json.Add("catalogEntry", packageInput.PackageDetailsUri.AbsoluteUri);
json.Add("packageContent", detailsJson["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"listed",
"published",
};
JsonUtility.CopyProperties(detailsJson, json, copyProperties, skipEmpty: true);
var context = JsonUtility.GetContext("Package");
json.Add("@context", context);
return(JsonLDTokenComparer.Format(json));
}
public async Task RemovePackageAsync(PackageIdentity packageIdentity)
{
var packageIndex = new PackageIndex(_context);
var allPackagesForId = await packageIndex.GetPackagesByIdAsync(packageIdentity.Id);
allPackagesForId.Remove(packageIdentity);
// Only remove the package if all versions have been removed
if (allPackagesForId.Count == 0)
{
var file = RootIndexFile;
var json = await file.GetJson(_context.Log, _context.Token);
var data = json["data"] as JArray;
var ids = await GetPackageIds();
if (ids.Remove(packageIdentity.Id))
{
data.Clear();
foreach (var id in ids.OrderBy(s => s, StringComparer.OrdinalIgnoreCase))
{
data.Add(id);
}
json["totalHits"] = ids.Count;
json = JsonLDTokenComparer.Format(json);
await file.Write(json, _context.Log, _context.Token);
}
}
}
/// <summary>
/// Create PackageDetails for a delete
/// </summary>
public static async Task <JObject> CreateDeleteDetailsAsync(PackageIdentity package, string reason, Uri catalogBaseURI, Guid commitId)
{
var now = DateTimeOffset.UtcNow;
var pageId = Guid.NewGuid().ToString().ToLowerInvariant();
var rootUri = UriUtility.GetPath(catalogBaseURI, $"data/{pageId}.json");
var json = JsonUtility.Create(rootUri, new List <string>()
{
"PackageDelete", "catalog:Permalink"
});
json.Add("commitId", commitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", now.GetDateString());
json.Add("sleet:operation", "remove");
var context = await JsonUtility.GetContextAsync("Catalog");
json.Add("@context", context);
json.Add("id", package.Id);
json.Add("version", package.Version.ToFullVersionString());
json.Add("created", DateTimeOffset.UtcNow.GetDateString());
json.Add("sleet:removeReason", reason);
json.Add("sleet:toolVersion", AssemblyVersionHelper.GetVersion().ToFullVersionString());
return(JsonLDTokenComparer.Format(json));
}
public async Task <JObject> CreateIndexAsync(Uri indexUri, List <JObject> packageDetails)
{
var json = JsonUtility.Create(indexUri,
new string[] {
"catalog:CatalogRoot",
"PackageRegistration",
"catalog:Permalink"
});
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
var itemsArray = new JArray();
json.Add("items", itemsArray);
json.Add("count", 1);
// Add everything to a single page
var pageJson = CreatePage(indexUri, packageDetails);
itemsArray.Add(pageJson);
var context = await JsonUtility.GetContextAsync("Registration");
json.Add("@context", context);
// Avoid formatting all package details again since this file could be very large.
return(JsonLDTokenComparer.Format(json, recurse: false));
}
public async Task <JObject> CreatePackageBlobAsync(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, new string[] { "Package", "http://schema.nuget.org/catalog#Permalink" });
json.Add("catalogEntry", packageInput.PackageDetails.GetIdUri().AbsoluteUri);
json.Add("packageContent", packageInput.PackageDetails["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"listed",
"published",
};
JsonUtility.CopyProperties(packageInput.PackageDetails, json, copyProperties, skipEmpty: true);
// Copy the catalog entry into the package blob. This allows the feed to
// save this info even if the catalog is disabled.
// Note that this is different from NuGet.org, so the sleet: namespace is used.
var catalogEntry = (JObject)packageInput.PackageDetails.DeepClone();
// Clear packageEntries, this can be very large in some cases.
catalogEntry.Remove("packageEntries");
json.Add("sleet:catalogEntry", catalogEntry);
var context = await JsonUtility.GetContextAsync("Package");
json.Add("@context", context);
return(JsonLDTokenComparer.Format(json));
}
public JObject CreateIndex(Uri indexUri, List <JObject> packageDetails)
{
var json = JsonUtility.Create(indexUri,
new string[] {
"catalog:CatalogRoot",
"PackageRegistration",
"catalog:Permalink"
});
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
var itemsArray = new JArray();
json.Add("items", itemsArray);
json.Add("count", 1);
// Add everything to a single page
var pageJson = CreatePage(indexUri, packageDetails);
itemsArray.Add(pageJson);
var context = JsonUtility.GetContext("Registration");
json.Add("@context", context);
return(JsonLDTokenComparer.Format(json));
}
public JObject CreatePage(Uri indexUri, List <JObject> packageDetails)
{
var versionSet = new HashSet <NuGetVersion>(packageDetails.Select(GetPackageVersion));
var lower = versionSet.Min().ToIdentityString().ToLowerInvariant();
var upper = versionSet.Max().ToIdentityString().ToLowerInvariant();
var json = JsonUtility.Create(indexUri, $"page/{lower}/{upper}", "catalog:CatalogPage");
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("count", packageDetails.Count);
json.Add("parent", indexUri.AbsoluteUri);
json.Add("lower", lower);
json.Add("upper", upper);
var itemsArray = new JArray();
json.Add("items", itemsArray);
// Order and add all items
foreach (var entry in packageDetails.OrderBy(GetPackageVersion))
{
itemsArray.Add(entry);
}
return(JsonLDTokenComparer.Format(json));
}
/// <summary>
/// Create PackageDetails for a delete
/// </summary>
public JObject CreateDeleteDetails(PackageIdentity package, string reason)
{
var now = DateTimeOffset.UtcNow;
var pageId = Guid.NewGuid().ToString().ToLowerInvariant();
var rootUri = UriUtility.CreateUri($"{_context.Source.BaseURI}catalog/data/{pageId}.json");
var json = JsonUtility.Create(rootUri, new List <string>()
{
"PackageDetails", "catalog:Permalink"
});
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", now.GetDateString());
json.Add("sleet:operation", "remove");
var context = JsonUtility.GetContext("Catalog");
json.Add("@context", context);
json.Add("id", package.Id);
json.Add("version", package.Version.ToFullVersionString());
json.Add("created", DateTimeOffset.UtcNow.GetDateString());
json.Add("sleet:removeReason", reason);
json.Add("sleet:toolVersion", Constants.SleetVersion.ToFullVersionString());
return(JsonLDTokenComparer.Format(json));
}
/// <summary>
/// Catalog index page.
/// </summary>
public static async Task <JObject> CreateCatalogPageAsync(Uri indexUri, Uri rootUri, List <JObject> packageDetails, Guid commitId)
{
var json = JsonUtility.Create(rootUri, "CatalogPage");
json.Add("commitId", commitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("count", packageDetails.Count);
json.Add("parent", indexUri.AbsoluteUri);
var itemArray = new JArray();
json.Add("items", itemArray);
foreach (var entry in packageDetails
.OrderBy(e => e["commitTimeStamp"].ToObject <DateTimeOffset>())
.ThenBy(e => e["@id"].ToString()))
{
itemArray.Add(entry);
}
var context = await JsonUtility.GetContextAsync("CatalogPage");
json.Add("@context", context);
return(JsonLDTokenComparer.Format(json));
}
private async Task <JObject> CreatePage(Dictionary <string, JObject> data)
{
var page = JObject.Parse(await TemplateUtility.LoadTemplate("Search", _context.OperationStart, _context.Source.BaseURI));
page["totalHits"] = data.Count;
page["data"] = new JArray(data.OrderBy(e => e.Key, StringComparer.OrdinalIgnoreCase).Select(e => e.Value));
return(JsonLDTokenComparer.Format(page, recurse: false));
}
/// <summary>
/// Create a new page entry in the catalog index.
/// </summary>
public static JObject CreateCatalogIndexPageEntry(Uri pageUri, Guid commitId)
{
var newPage = JsonUtility.Create(pageUri, "CatalogPage");
newPage["commitId"] = commitId.ToString().ToLowerInvariant();
newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
newPage["count"] = 0;
newPage = JsonLDTokenComparer.Format(newPage);
return(newPage);
}
/// <summary>
/// Add an entry to the catalog.
/// </summary>
private async Task AddCatalogCommits(IEnumerable <JObject> newPageCommits, string lastUpdatedPropertyName)
{
// Add catalog page entry
var catalogIndexJson = await RootIndexFile.GetJson(_context.Log, _context.Token);
catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
var pages = JsonUtility.GetItems(catalogIndexJson);
var currentPageUri = GetCurrentPage(catalogIndexJson);
var currentPageFile = _context.Source.Get(currentPageUri);
var pageCommits = new List <JObject>();
var currentPageJson = await currentPageFile.GetJsonOrNull(_context.Log, _context.Token);
if (currentPageJson != null)
{
pageCommits = JsonUtility.GetItems(currentPageJson);
}
else
{
var newPageEntry = CatalogUtility.CreateCatalogIndexPageEntry(currentPageUri, _context.CommitId);
var pageArray = (JArray)catalogIndexJson["items"];
pageArray.Add(newPageEntry);
// Update pages
pages = JsonUtility.GetItems(catalogIndexJson);
}
// Add all commits, this might go over the max catalog page size.
// This could be improved to create new pages once over the limit if needed.
pageCommits.AddRange(newPageCommits);
// Write catalog page
var pageJson = await CatalogUtility.CreateCatalogPageAsync(RootIndexFile.EntityUri, currentPageUri, pageCommits, _context.CommitId);
await currentPageFile.Write(pageJson, _context.Log, _context.Token);
// Update index
CatalogUtility.UpdatePageIndex(catalogIndexJson, pageJson, _context.CommitId);
catalogIndexJson[lastUpdatedPropertyName] = DateTimeOffset.UtcNow.GetDateString();
catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);
await RootIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
}
/// <summary>
/// Add an entry to the catalog.
/// </summary>
private async Task AddCatalogEntry(JObject pageCommit, string lastUpdatedPropertyName)
{
// Add catalog page entry
var catalogIndexJson = await RootIndexFile.GetJson(_context.Log, _context.Token);
catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
var pages = JsonUtility.GetItems(catalogIndexJson);
var currentPageUri = GetCurrentPage(catalogIndexJson);
var currentPageFile = _context.Source.Get(currentPageUri);
var pageCommits = new List <JObject>();
var currentPageJson = await currentPageFile.GetJsonOrNull(_context.Log, _context.Token);
if (currentPageJson != null)
{
pageCommits = JsonUtility.GetItems(currentPageJson);
}
else
{
var newPageEntry = CatalogUtility.CreateCatalogIndexPageEntry(currentPageUri, _context.CommitId);
var pageArray = (JArray)catalogIndexJson["items"];
pageArray.Add(newPageEntry);
// Update pages
pages = JsonUtility.GetItems(catalogIndexJson);
}
pageCommits.Add(pageCommit);
// Write catalog page
var pageJson = await CatalogUtility.CreateCatalogPageAsync(RootIndexFile.EntityUri, currentPageUri, pageCommits, _context.CommitId);
await currentPageFile.Write(pageJson, _context.Log, _context.Token);
// Update index
CatalogUtility.UpdatePageIndex(catalogIndexJson, pageJson, _context.CommitId);
catalogIndexJson[lastUpdatedPropertyName] = DateTimeOffset.UtcNow.GetDateString();
catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);
await RootIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
}
/// <summary>
/// Create the file directly without loading the previous file.
/// </summary>
public async Task CreateAsync(IEnumerable <string> packageIds)
{
var ids = new SortedSet <string>(packageIds, StringComparer.OrdinalIgnoreCase);
// Create a new file using the full set of package ids.
// There is no need to read the existing file.
var json = await GetEmptyTemplate();
var data = new JArray(ids.Take(MaxResults));
json["data"] = data;
json["totalHits"] = data.Count;
var formatted = JsonLDTokenComparer.Format(json, recurse: false);
await RootIndexFile.Write(formatted, _context.Log, _context.Token);
}
private JObject CreatePage(List <JObject> data)
{
var page = JObject.Parse(TemplateUtility.LoadTemplate("Search", _context.OperationStart, _context.Source.BaseURI));
page["totalHits"] = data.Count;
var dataArray = new JArray();
page["data"] = dataArray;
foreach (var entry in data.OrderBy(e => e.GetId(), StringComparer.OrdinalIgnoreCase))
{
dataArray.Add(entry);
}
return(JsonLDTokenComparer.Format(page));
}
/// <summary>
/// Create a package item entry.
/// </summary>
public async Task <JObject> CreateItem(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, "Package");
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
var packageDetailsFile = _context.Source.Get(packageInput.PackageDetailsUri);
var detailsJson = await packageDetailsFile.GetJson(_context.Log, _context.Token);
json.Add("packageContent", detailsJson["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"@id",
"@type",
"authors",
"dependencyGroups",
"description",
"iconUrl",
"id",
"language",
"licenseUrl",
"listed",
"minClientVersion",
"packageContent",
"projectUrl",
"published",
"requireLicenseAcceptance",
"summary",
"tags",
"title",
"version"
};
var catalogEntry = new JObject();
JsonUtility.CopyProperties(detailsJson, catalogEntry, copyProperties, skipEmpty: true);
json.Add("catalogEntry", catalogEntry);
return(JsonLDTokenComparer.Format(json));
}
/// <summary>
/// Create a package item entry.
/// </summary>
public JObject CreateItem(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, "Package");
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("packageContent", packageInput.PackageDetails["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"@id",
"@type",
"authors",
"dependencyGroups",
"description",
"iconUrl",
"id",
"language",
"licenseUrl",
"listed",
"minClientVersion",
"packageContent",
"projectUrl",
"published",
"requireLicenseAcceptance",
"summary",
"tags",
"title",
"version"
};
var catalogEntry = new JObject();
JsonUtility.CopyProperties(packageInput.PackageDetails, catalogEntry, copyProperties, skipEmpty: true);
json.Add("catalogEntry", catalogEntry);
// Format package details at creation time, and avoid doing it again later to improve perf.
return(JsonLDTokenComparer.Format(json));
}
public async Task AddPackageAsync(PackageInput packageInput)
{
var file = RootIndexFile;
var json = await file.GetJson(_context.Log, _context.Token);
var data = json["data"] as JArray;
var ids = await GetPackageIds();
ids.Add(packageInput.Identity.Id);
data.Clear();
foreach (var id in ids.OrderBy(s => s, StringComparer.OrdinalIgnoreCase))
{
data.Add(id);
}
json["totalHits"] = ids.Count;
json = JsonLDTokenComparer.Format(json);
await file.Write(json, _context.Log, _context.Token);
}
/// <summary>
/// Create a result containing all versions of the package. The passed in identity
/// may or may not be the latest one that is shown.
/// </summary>
private async Task <JObject> CreatePackageEntry(PackageIdentity package, bool add)
{
var packageIndex = new PackageIndex(_context);
var versions = await packageIndex.GetPackageVersions(package.Id);
if (add)
{
versions.Add(package.Version);
}
else
{
versions.Remove(package.Version);
}
var latest = versions.Max();
var latestIdentity = new PackageIdentity(package.Id, latest);
var packageUri = Registrations.GetPackageUri(_context.Source.BaseURI, latestIdentity);
var packageEntry = JsonUtility.Create(packageUri, "Package");
var registrationUri = Registrations.GetIndexUri(_context.Source.BaseURI, package.Id);
// Read the catalog entry from the package blob. The catalog may not be enabled.
var registrations = new Registrations(_context);
var catalogEntry = await registrations.GetCatalogEntryFromPackageBlob(latestIdentity);
Debug.Assert(catalogEntry != null);
packageEntry.Add("registration", registrationUri.AbsoluteUri);
var copyProperties = new[]
{
"id",
"version",
"description",
"summary",
"title",
"iconUrl",
"licenseUrl",
"projectUrl",
"tags"
};
JsonUtility.CopyProperties(catalogEntry, packageEntry, copyProperties, skipEmpty: false);
var copyPropertiesDelimited = new[]
{
"authors",
"owners"
};
JsonUtility.CopyDelimitedProperties(catalogEntry, packageEntry, copyPropertiesDelimited, ',');
JsonUtility.RequireArrayWithEmptyString(packageEntry, new[] { "tags", "authors" });
packageEntry.Add("totalDownloads", 0);
var versionsArray = new JArray();
packageEntry.Add("versions", versionsArray);
foreach (var version in versions.OrderBy(v => v))
{
var versionIdentity = new PackageIdentity(package.Id, version);
var versionUri = Registrations.GetPackageUri(_context.Source.BaseURI, versionIdentity);
var versionEntry = JsonUtility.Create(versionUri, "Package");
versionEntry.Add("downloads", 0);
versionEntry.Add("version", version.ToFullVersionString());
versionsArray.Add(versionEntry);
}
return(JsonLDTokenComparer.Format(packageEntry));
}
/// <summary>
/// Create a PackageDetails page that contains all the package information.
/// </summary>
public JObject CreatePackageDetails(PackageInput packageInput)
{
var now = DateTimeOffset.UtcNow;
var package = packageInput.Package;
var nuspec = XDocument.Load(package.GetNuspec());
var nuspecReader = new NuspecReader(nuspec);
var pageId = Guid.NewGuid().ToString().ToLowerInvariant();
var rootUri = UriUtility.CreateUri($"{_context.Source.BaseURI}catalog/data/{pageId}.json");
packageInput.PackageDetailsUri = rootUri;
var json = JsonUtility.Create(rootUri, new List <string>()
{
"PackageDetails", "catalog:Permalink"
});
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("sleet:operation", "add");
var context = JsonUtility.GetContext("Catalog");
json.Add("@context", context);
json.Add("id", packageInput.Identity.Id);
json.Add("version", packageInput.Identity.Version.ToFullVersionString());
json.Add("created", now.GetDateString());
json.Add("lastEdited", "0001-01-01T00:00:00Z");
var copyProperties = new List <string>()
{
"authors",
"copyright",
"description",
"iconUrl",
"projectUrl",
"licenseUrl",
"language",
"summary",
"owners",
"releaseNotes"
};
foreach (var propertyName in copyProperties)
{
json.Add(CreateProperty(propertyName, propertyName, nuspecReader));
}
json.Add("isPrerelease", packageInput.Identity.Version.IsPrerelease);
// Unused?
json.Add("licenseNames", string.Empty);
json.Add("licenseReportUrl", string.Empty);
// All packages are listed
json.Add("listed", true);
var titleValue = GetEntry(nuspecReader, "title");
if (!string.IsNullOrEmpty(titleValue))
{
json.Add("title", titleValue);
}
using (var stream = File.OpenRead(packageInput.PackagePath))
{
using (var sha512 = SHA512.Create())
{
var packageHash = Convert.ToBase64String(sha512.ComputeHash(stream));
json.Add("packageHash", packageHash);
json.Add("packageHashAlgorithm", "SHA512");
}
json.Add("packageSize", stream.Length);
}
json.Add("published", now.GetDateString());
json.Add("requireLicenseAcceptance", GetEntry(nuspecReader, "requireLicenseAcceptance").Equals("true", StringComparison.OrdinalIgnoreCase));
var minVersion = nuspecReader.GetMinClientVersion();
if (minVersion != null)
{
json.Add("minClientVersion", minVersion.ToIdentityString());
}
// Tags
var tagSet = new HashSet <string>(GetEntry(nuspecReader, "tags").Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase);
tagSet.Remove(string.Empty);
var tagArray = new JArray(tagSet);
json.Add("tags", tagArray);
// Framework assemblies
var fwrGroups = nuspecReader.GetFrameworkReferenceGroups();
var fwrArray = new JArray();
json.Add("frameworkAssemblyGroup", fwrArray);
foreach (var group in fwrGroups.OrderBy(e => e.TargetFramework.GetShortFolderName(), StringComparer.OrdinalIgnoreCase))
{
var groupTFM = group.TargetFramework.GetShortFolderName().ToLowerInvariant();
var groupNode = JsonUtility.Create(rootUri, $"frameworkassemblygroup/{groupTFM}".ToLowerInvariant(), "FrameworkAssemblyGroup");
// Leave the framework property out for the 'any' group
if (!group.TargetFramework.IsAny)
{
groupNode.Add("targetFramework", groupTFM);
}
fwrArray.Add(groupNode);
if (group.Items.Any())
{
var assemblyArray = new JArray();
groupNode.Add("assembly", assemblyArray);
foreach (var fwAssembly in group.Items.Distinct().OrderBy(e => e, StringComparer.OrdinalIgnoreCase))
{
assemblyArray.Add(fwAssembly);
}
}
}
// Dependencies
var dependencyGroups = nuspecReader.GetDependencyGroups();
var depArray = new JArray();
json.Add("dependencyGroups", depArray);
foreach (var group in dependencyGroups.OrderBy(e => e.TargetFramework.GetShortFolderName(), StringComparer.OrdinalIgnoreCase))
{
var groupTFM = group.TargetFramework.GetShortFolderName().ToLowerInvariant();
var groupNode = JsonUtility.Create(rootUri, $"dependencygroup/{groupTFM}".ToLowerInvariant(), "PackageDependencyGroup");
// Leave the framework property out for the 'any' group
if (!group.TargetFramework.IsAny)
{
groupNode.Add("targetFramework", groupTFM);
}
depArray.Add(groupNode);
if (group.Packages.Any())
{
var packageArray = new JArray();
groupNode.Add("dependencies", packageArray);
foreach (var depPackage in group.Packages.Distinct().OrderBy(e => e.Id, StringComparer.OrdinalIgnoreCase))
{
var packageNode = JsonUtility.Create(rootUri, $"dependencygroup/{groupTFM}/{depPackage.Id}".ToLowerInvariant(), "PackageDependency");
packageNode.Add("id", depPackage.Id);
packageNode.Add("range", depPackage.VersionRange.ToNormalizedString());
packageArray.Add(packageNode);
}
}
}
json.Add("packageContent", packageInput.NupkgUri.AbsoluteUri);
// add flatcontainer files
var packageEntriesArray = new JArray();
json.Add("packageEntries", packageEntriesArray);
var packageEntryIndex = 0;
foreach (var entry in packageInput.Zip.Entries.OrderBy(e => e.FullName, StringComparer.OrdinalIgnoreCase))
{
var fileEntry = JsonUtility.Create(rootUri, $"packageEntry/{packageEntryIndex}", "packageEntry");
fileEntry.Add("fullName", entry.FullName);
fileEntry.Add("length", entry.Length);
fileEntry.Add("lastWriteTime", entry.LastWriteTime.GetDateString());
packageEntriesArray.Add(fileEntry);
packageEntryIndex++;
}
json.Add("sleet:toolVersion", Constants.SleetVersion.ToFullVersionString());
return(JsonLDTokenComparer.Format(json));
}
/// <summary>
/// Add a package to the catalog.
/// </summary>
public async Task AddPackage(PackageInput packageInput)
{
// Create package details page
var packageDetails = CreatePackageDetails(packageInput);
var packageDetailsFile = _context.Source.Get(UriUtility.CreateUri(packageDetails["@id"].ToString()));
await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);
// Add catalog page entry
var catalogIndexUri = _context.Source.GetPath("/catalog/index.json");
var catalogIndexFile = _context.Source.Get(catalogIndexUri);
var catalogIndexJson = await catalogIndexFile.GetJson(_context.Log, _context.Token);
catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
var pages = GetItems(catalogIndexJson);
var currentPageUri = GetCurrentPage(catalogIndexJson);
var currentPageFile = _context.Source.Get(currentPageUri);
var pageCommits = new List <JObject>();
if (await currentPageFile.Exists(_context.Log, _context.Token))
{
var currentPageJson = await currentPageFile.GetJson(_context.Log, _context.Token);
pageCommits = GetItems(currentPageJson);
}
else
{
var newPage = JsonUtility.Create(currentPageFile.EntityUri, "CatalogPage");
newPage["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
newPage["count"] = 0;
newPage = JsonLDTokenComparer.Format(newPage);
var pageArray = (JArray)catalogIndexJson["items"];
pageArray.Add(newPage);
// Update pages
pages = GetItems(catalogIndexJson);
}
// Create commit
var pageCommit = JsonUtility.Create(packageDetailsFile.EntityUri, "nuget:PackageDetails");
pageCommit["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
pageCommit["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
pageCommit["nuget:id"] = packageInput.Identity.Id;
pageCommit["nuget:version"] = packageInput.Identity.Version.ToFullVersionString();
pageCommit["sleet:operation"] = "add";
pageCommits.Add(pageCommit);
// Write catalog page
var pageJson = CreateCatalogPage(catalogIndexUri, currentPageUri, pageCommits);
await currentPageFile.Write(pageJson, _context.Log, _context.Token);
// Update index
var pageEntry = pages.Where(e => e["@id"].ToString() == currentPageFile.EntityUri.AbsoluteUri).Single();
pageEntry["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
pageEntry["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
pageEntry["count"] = pageCommits.Count;
catalogIndexJson["count"] = pages.Count;
catalogIndexJson["nuget:lastCreated"] = DateTimeOffset.UtcNow.GetDateString();
catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);
await catalogIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
}
/// <summary>
/// Create a PackageDetails page that contains all the package information and an exact uri.
/// </summary>
public static async Task <JObject> CreatePackageDetailsWithExactUriAsync(PackageInput packageInput, Uri detailsUri, Uri nupkgUri, Guid commitId, bool writeFileList)
{
var now = DateTimeOffset.UtcNow;
var nuspecReader = packageInput.Nuspec;
var json = JsonUtility.Create(detailsUri, new List <string>()
{
"PackageDetails", "catalog:Permalink"
});
json.Add("commitId", commitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("sleet:operation", "add");
var context = await JsonUtility.GetContextAsync("Catalog");
json.Add("@context", context);
json.Add("id", packageInput.Identity.Id);
json.Add("version", packageInput.Identity.Version.ToFullVersionString());
json.Add("created", now.GetDateString());
json.Add("lastEdited", "0001-01-01T00:00:00Z");
var copyProperties = new List <string>()
{
"authors",
"copyright",
"description",
"iconUrl",
"projectUrl",
"licenseUrl",
"language",
"summary",
"owners",
"releaseNotes"
};
foreach (var propertyName in copyProperties)
{
json.Add(CreateProperty(propertyName, propertyName, nuspecReader));
}
json.Add("isPrerelease", packageInput.Identity.Version.IsPrerelease);
// Unused?
json.Add("licenseNames", string.Empty);
json.Add("licenseReportUrl", string.Empty);
// All packages are listed
json.Add("listed", true);
var titleValue = GetEntry(nuspecReader, "title");
if (!string.IsNullOrEmpty(titleValue))
{
json.Add("title", titleValue);
}
// Avoid hashing the package for virtual catalog pages, it takes
// a long time for machines with slow disks. It also isn't used
// anywhere except in the real catalog.
if (writeFileList)
{
using (var stream = File.OpenRead(packageInput.PackagePath))
{
using (var sha512 = SHA512.Create())
{
var packageHash = Convert.ToBase64String(sha512.ComputeHash(stream));
json.Add("packageHash", packageHash);
json.Add("packageHashAlgorithm", "SHA512");
}
json.Add("packageSize", stream.Length);
}
}
json.Add("published", now.GetDateString());
json.Add("requireLicenseAcceptance", GetEntry(nuspecReader, "requireLicenseAcceptance").Equals("true", StringComparison.OrdinalIgnoreCase));
var minVersion = nuspecReader.GetMinClientVersion();
if (minVersion != null)
{
json.Add("minClientVersion", minVersion.ToIdentityString());
}
// Tags
var tagSet = new HashSet <string>(GetEntry(nuspecReader, "tags").Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase);
tagSet.Remove(string.Empty);
var tagArray = new JArray(tagSet);
json.Add("tags", tagArray);
// Framework assemblies
var fwrGroups = nuspecReader.GetFrameworkAssemblyGroups();
var fwrArray = new JArray();
json.Add("frameworkAssemblyGroup", fwrArray);
foreach (var group in fwrGroups.OrderBy(e => e.TargetFramework.GetShortFolderName(), StringComparer.OrdinalIgnoreCase))
{
var groupTFM = group.TargetFramework.GetShortFolderName().ToLowerInvariant();
var groupNode = JsonUtility.Create(detailsUri, $"frameworkassemblygroup/{groupTFM}".ToLowerInvariant(), "FrameworkAssemblyGroup");
// Leave the framework property out for the 'any' group
if (!group.TargetFramework.IsAny)
{
groupNode.Add("targetFramework", groupTFM);
}
fwrArray.Add(groupNode);
if (group.Items.Any())
{
var assemblyArray = new JArray();
groupNode.Add("assembly", assemblyArray);
foreach (var fwAssembly in group.Items.Distinct().OrderBy(e => e, StringComparer.OrdinalIgnoreCase))
{
assemblyArray.Add(fwAssembly);
}
}
}
// Dependencies
var dependencyGroups = nuspecReader.GetDependencyGroups();
var depArray = new JArray();
json.Add("dependencyGroups", depArray);
foreach (var group in dependencyGroups.OrderBy(e => e.TargetFramework.GetShortFolderName(), StringComparer.OrdinalIgnoreCase))
{
var groupTFM = group.TargetFramework.GetShortFolderName().ToLowerInvariant();
var groupNode = JsonUtility.Create(detailsUri, $"dependencygroup/{groupTFM}".ToLowerInvariant(), "PackageDependencyGroup");
// Leave the framework property out for the 'any' group
if (!group.TargetFramework.IsAny)
{
groupNode.Add("targetFramework", groupTFM);
}
depArray.Add(groupNode);
if (group.Packages.Any())
{
var packageArray = new JArray();
groupNode.Add("dependencies", packageArray);
foreach (var depPackage in group.Packages.Distinct().OrderBy(e => e.Id, StringComparer.OrdinalIgnoreCase))
{
var packageNode = JsonUtility.Create(detailsUri, $"dependencygroup/{groupTFM}/{depPackage.Id}".ToLowerInvariant(), "PackageDependency");
packageNode.Add("id", depPackage.Id);
packageNode.Add("range", depPackage.VersionRange.ToNormalizedString());
packageArray.Add(packageNode);
}
}
}
json.Add("packageContent", nupkgUri.AbsoluteUri);
if (writeFileList)
{
// Write out all files contained in the package
var packageEntriesArray = new JArray();
json.Add("packageEntries", packageEntriesArray);
using (var zip = packageInput.CreateZip())
{
AddZipEntry(zip, detailsUri, packageEntriesArray);
}
}
json.Add("sleet:toolVersion", AssemblyVersionHelper.GetVersion().ToFullVersionString());
return(JsonLDTokenComparer.Format(json));
}
