public async Task <JObject> CreatePackageBlobAsync(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, new string[] { "Package", "http://schema.nuget.org/catalog#Permalink" });
json.Add("catalogEntry", packageInput.PackageDetails.GetIdUri().AbsoluteUri);
json.Add("packageContent", packageInput.PackageDetails["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"listed",
"published",
};
JsonUtility.CopyProperties(packageInput.PackageDetails, json, copyProperties, skipEmpty: true);
// Copy the catalog entry into the package blob. This allows the feed to
// save this info even if the catalog is disabled.
// Note that this is different from NuGet.org, so the sleet: namespace is used.
var catalogEntry = (JObject)packageInput.PackageDetails.DeepClone();
// Clear packageEntries, this can be very large in some cases.
catalogEntry.Remove("packageEntries");
json.Add("sleet:catalogEntry", catalogEntry);
var context = await JsonUtility.GetContextAsync("Package");
json.Add("@context", context);
return(JsonLDTokenComparer.Format(json));
}
/// <summary>
/// Create a PackageDetails page that contains all the package information.
/// </summary>
public static Task <JObject> CreatePackageDetailsAsync(PackageInput packageInput, Uri catalogBaseURI, Uri nupkgUri, Guid commitId, bool writeFileList)
{
var pageId = Guid.NewGuid().ToString().ToLowerInvariant();
var rootUri = UriUtility.GetPath(catalogBaseURI, $"data/{pageId}.json");
return(CreatePackageDetailsWithExactUriAsync(packageInput, rootUri, nupkgUri, commitId, writeFileList));
}
public async Task <JObject> CreatePackageBlob(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, new string[] { "Package", "http://schema.nuget.org/catalog#Permalink" });
var packageDetailsFile = _context.Source.Get(packageInput.PackageDetailsUri);
if (!await packageDetailsFile.Exists(_context.Log, _context.Token))
{
throw new FileNotFoundException($"Unable to find {packageDetailsFile.EntityUri.AbsoluteUri}");
}
var detailsJson = await packageDetailsFile.GetJson(_context.Log, _context.Token);
json.Add("catalogEntry", packageInput.PackageDetailsUri.AbsoluteUri);
json.Add("packageContent", detailsJson["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"listed",
"published",
};
JsonUtility.CopyProperties(detailsJson, json, copyProperties, skipEmpty: true);
var context = JsonUtility.GetContext("Package");
json.Add("@context", context);
return(JsonLDTokenComparer.Format(json));
}
public async Task AddPackageAsync(PackageInput packageInput)
{
// Create package details page
var packageDetails = await CatalogUtility.CreatePackageDetailsAsync(packageInput, CatalogBaseURI, _context.CommitId, writeFileList : false);
packageInput.PackageDetails = packageDetails;
}
/// <summary>
/// Add a package to the catalog.
/// </summary>
public async Task AddPackageAsync(PackageInput packageInput)
{
// Create package details page
var addFileList = _context.SourceSettings.CatalogEnabled;
var packageDetails = await CatalogUtility.CreatePackageDetailsAsync(packageInput, CatalogBaseURI, _context.CommitId, addFileList);
var packageDetailsUri = JsonUtility.GetIdUri(packageDetails);
// Add output to the package input for other services to use.
packageInput.PackageDetails = packageDetails;
var packageDetailsFile = _context.Source.Get(packageDetailsUri);
await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);
// Create commit
var pageCommit = CatalogUtility.CreatePageCommit(
packageInput.Identity,
packageDetailsUri,
_context.CommitId,
SleetOperation.Add,
"nuget:PackageDetails");
await AddCatalogEntry(pageCommit, "nuget:lastCreated");
}
private static Task<PackageInput> GetPackageInput(string file, ILogger log)
{
// Validate package
log.LogVerbose($"Reading {file}");
PackageInput packageInput = null;
try
{
// Read basic info from the package and verify that it isn't broken.
packageInput = PackageInput.Create(file);
}
catch
{
log.LogError($"Invalid package '{file}'.");
throw;
}
// Display a message for non-normalized packages
if (packageInput.Identity.Version.ToString() != packageInput.Identity.Version.ToNormalizedString())
{
var message = $"Package '{packageInput.PackagePath}' does not contain a normalized version. Normalized: '{packageInput.Identity.Version.ToNormalizedString()}' Nuspec version: '{packageInput.Identity.Version.ToString()}'. See https://semver.org/ for details.";
log.LogVerbose(message);
}
return Task.FromResult(packageInput);
}
private async Task CreateDetailsForAdd(PackageInput packageInput)
{
// Create a a details page and assign it to the input
var nupkgUri = packageInput.GetNupkgUri(_context);
var packageDetails = await CatalogUtility.CreatePackageDetailsAsync(packageInput, CatalogBaseURI, nupkgUri, _context.CommitId, writeFileList : false);
packageInput.PackageDetails = packageDetails;
}
private Task AddPackageAsync(PackageInput packageInput)
{
return(Task.WhenAll(new[]
{
AddNuspecAsync(packageInput),
AddNupkgAsync(packageInput)
}));
}
private static PackageInput GetPackageInput(string file, ILogger log)
{
// Validate package
log.LogInformation($"Reading {file}");
PackageIdentity identity = null;
var isSymbolsPackage = false;
var hasNuspec = false;
var hasMultipleNuspecs = false;
var nuspecName = string.Empty;
try
{
// Read basic info from the package and verify that it isn't broken.
using (var zip = new ZipArchive(File.OpenRead(file), ZipArchiveMode.Read, leaveOpen: false))
using (var package = new PackageArchiveReader(zip))
{
identity = package.GetIdentity();
isSymbolsPackage = SymbolsUtility.IsSymbolsPackage(zip, file);
// Check for correct nuspec name
nuspecName = identity.Id + ".nuspec";
var nuspecs = zip.Entries.Where(entry => entry.FullName.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)).ToList();
hasNuspec = nuspecs.Count == 1;
hasMultipleNuspecs = nuspecs.Count > 1;
}
}
catch
{
log.LogError($"Invalid package '{file}'.");
throw;
}
var packageInput = new PackageInput(file, identity, isSymbolsPackage);
// Display a message for non-normalized packages
if (packageInput.Identity.Version.ToString() != packageInput.Identity.Version.ToNormalizedString())
{
var message = $"Package '{packageInput.PackagePath}' does not contain a normalized version. Normalized: '{packageInput.Identity.Version.ToNormalizedString()}' Nuspec version: '{packageInput.Identity.Version.ToString()}'. See https://semver.org/ for details.";
log.LogVerbose(message);
}
// Check for correct nuspec name
if (!hasNuspec)
{
throw new InvalidDataException($"'{packageInput.PackagePath}' does not contain '{nuspecName}'.");
}
// Check for multiple nuspec files
if (hasMultipleNuspecs)
{
throw new InvalidDataException($"'{packageInput.PackagePath}' contains multiple nuspecs and cannot be consumed.");
}
return(packageInput);
}
/// <summary>
/// Add a package to all services.
/// </summary>
public static async Task AddPackage(SleetContext context, PackageInput package)
{
var services = GetServices(context);
foreach (var service in services)
{
await service.AddPackage(package);
}
}
private Task AddNupkgAsync(PackageInput packageInput)
{
// Add the nupkg by linking it instead of copying it to the local cache.
var nupkgFile = _context.Source.Get(GetNupkgPath(packageInput.Identity));
nupkgFile.Link(packageInput.PackagePath, _context.Log, _context.Token);
return(Task.FromResult(true));
}
private async Task AddAssemblyAsync(PackageFile assembly, PackageInput packageInput)
{
var file = _context.Source.Get(SymbolsIndexUtility.GetAssemblyFilePath(assembly.FileName, assembly.Hash));
if (await file.Exists(_context.Log, _context.Token) == false)
{
// Write assembly
await file.Write(assembly.Stream, _context.Log, _context.Token);
}
}
/// <summary>
/// Create a package details page for a package id/version.
/// </summary>
private async Task CreatePackagePageAsync(PackageInput package)
{
// Create package page
var packageUri = GetPackageUri(package.Identity);
var packageFile = _context.Source.Get(packageUri);
var packageJson = await CreatePackageBlobAsync(package);
// Write package page
await packageFile.Write(packageJson, _context.Log, _context.Token);
}
public static SleetOperations CreateDelete(PackageSets originalIndex, IEnumerable <PackageIdentity> packagesToRemove, IEnumerable <PackageIdentity> symbolsPackagesToRemove)
{
var toAdd = new List <PackageInput>();
var toRemove = new List <PackageInput>();
toRemove.AddRange(packagesToRemove.Select(e => PackageInput.CreateForDelete(e, isSymbols: false)));
toRemove.AddRange(symbolsPackagesToRemove.Select(e => PackageInput.CreateForDelete(e, isSymbols: true)));
return(Create(originalIndex, toAdd, toRemove));
}
public async Task AddSymbolsPackageAsync(PackageInput packageInput)
{
// Load existing index
var sets = await GetPackageSetsAsync();
// Add package
await sets.Symbols.AddPackageAsync(packageInput);
// Write file
await Save(sets);
}
private async Task AddNuspecAsync(PackageInput packageInput)
{
// Add nuspec
var nuspecPath = $"{packageInput.Identity.Id}.nuspec".ToLowerInvariant();
var entryFile = _context.Source.Get(GetZipFileUri(packageInput.Identity, nuspecPath));
using (var nuspecStream = packageInput.Nuspec.Xml.AsMemoryStream())
{
await entryFile.Write(nuspecStream, _context.Log, _context.Token);
}
}
private async Task AddAssemblyAsync(PackageFile assembly, PackageInput packageInput)
{
var file = _context.Source.Get(SymbolsIndexUtility.GetAssemblyFilePath(assembly.FileName, assembly.Hash));
if (await file.Exists(_context.Log, _context.Token) == false)
{
// Write assembly
using (var stream = await packageInput.GetEntryStreamWithLockAsync(assembly.ZipEntry))
{
await file.Write(stream, _context.Log, _context.Token);
}
}
}
/// <summary>
/// file/hash/index -> package
/// </summary>
private Task AddAssemblyToPackageIndexAsync(PackageInput package, ISleetFile indexFile, bool isSymbolsPackage)
{
var index = new PackageIndexFile(_context, indexFile, persistWhenEmpty: false);
if (isSymbolsPackage)
{
return(index.AddSymbolsPackageAsync(package));
}
else
{
return(index.AddPackageAsync(package));
}
}
private static void CheckForDuplicates(List<PackageInput> packages)
{
PackageInput lastPackage = null;
foreach (var package in packages.OrderBy(e => e))
{
if (package.Equals(lastPackage))
{
throw new InvalidOperationException($"Duplicate packages detected for '{package.Identity}'.");
}
lastPackage = package;
}
}
/// <summary>
/// Create a package details page for a package id/version.
/// </summary>
private async Task CreatePackagePageAsync(PackageInput package)
{
// Create package page
var packageUri = GetPackageUri(package.Identity);
var packageFile = _context.Source.Get(packageUri);
using (var timer = PerfEntryWrapper.CreateModifyTimer(packageFile, _context))
{
var packageJson = await CreatePackageBlobAsync(package);
// Write package page
await packageFile.Write(packageJson, _context.Log, _context.Token);
}
}
private async Task AddNupkgAsync(PackageInput packageInput)
{
// Add nupkg
var nupkgFile = _context.Source.Get(GetNupkgPath(packageInput.Identity));
await nupkgFile.Write(File.OpenRead(packageInput.PackagePath), _context.Log, _context.Token);
// Add nuspec
var nuspecPath = $"{packageInput.Identity.Id}.nuspec".ToLowerInvariant();
using (var nuspecStream = packageInput.Nuspec.Xml.AsMemoryStreamAsync())
{
var entryFile = _context.Source.Get(GetZipFileUri(packageInput.Identity, nuspecPath));
await entryFile.Write(nuspecStream, _context.Log, _context.Token);
}
}
/// <summary>
/// Create a package item entry.
/// </summary>
public async Task <JObject> CreateItem(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, "Package");
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
var packageDetailsFile = _context.Source.Get(packageInput.PackageDetailsUri);
var detailsJson = await packageDetailsFile.GetJson(_context.Log, _context.Token);
json.Add("packageContent", detailsJson["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"@id",
"@type",
"authors",
"dependencyGroups",
"description",
"iconUrl",
"id",
"language",
"licenseUrl",
"listed",
"minClientVersion",
"packageContent",
"projectUrl",
"published",
"requireLicenseAcceptance",
"summary",
"tags",
"title",
"version"
};
var catalogEntry = new JObject();
JsonUtility.CopyProperties(detailsJson, catalogEntry, copyProperties, skipEmpty: true);
json.Add("catalogEntry", catalogEntry);
return(JsonLDTokenComparer.Format(json));
}
private async Task AddPackageAsync(PackageInput packageInput, bool isSymbolsPackage)
{
var tasks = new List <Task>();
if (isSymbolsPackage)
{
// Add symbols packages to the feed regardless of assemblies
tasks.Add(AddSymbolsNupkgToFeed(packageInput));
}
// Read dll/pdb files from the package.
var assemblies = await GetAssembliesAsync(packageInput);
if (assemblies.Count > 0)
{
// Add the id/version to the package index.
if (isSymbolsPackage)
{
tasks.Add(PackageIndex.AddSymbolsPackageAsync(packageInput));
}
else
{
tasks.Add(PackageIndex.AddPackageAsync(packageInput));
}
// Add dll/pdb files to the feed.
tasks.AddRange(assemblies.Select(e => AddAssemblyAsync(e, packageInput)));
// Add assembly -> package reverse lookup
tasks.AddRange(assemblies.Select(e => AddAssemblyToPackageIndexAsync(packageInput, e.IndexFile, isSymbolsPackage: isSymbolsPackage)));
// Add index of all dll/pdb files added for the package.
tasks.Add(AddPackageToAssemblyIndexAsync(packageInput.Identity, assemblies, isSymbolsPackage: isSymbolsPackage));
}
else
{
await _context.Log.LogAsync(LogLevel.Verbose, $"No files found that could be added to the symbols feed. Skipping package {packageInput.Identity}");
}
// Wait for everything to finish
await Task.WhenAll(tasks);
// Dispose of memory streams
assemblies.ForEach(e => e.Dispose());
}
// Copy the symbols nupkg to the symbols folder and create a catalog details page for it.
private async Task AddSymbolsNupkgToFeed(PackageInput package)
{
// Write .nupkg to feed
var packagePath = SymbolsIndexUtility.GetSymbolsNupkgPath(package.Identity);
var packageFile = _context.Source.Get(packagePath);
await packageFile.Write(File.OpenRead(package.PackagePath), _context.Log, _context.Token);
// Write catalog entry to the symbols folder for the package
var detailsPath = SymbolsIndexUtility.GetSymbolsPackageDetailsPath(package.Identity);
var detailsFile = _context.Source.Get(detailsPath);
var commitId = Guid.NewGuid();
var detailsJson = await CatalogUtility.CreatePackageDetailsWithExactUriAsync(package, detailsFile.EntityUri, packageFile.EntityUri, commitId, writeFileList : false);
await detailsFile.Write(detailsJson, _context.Log, _context.Token);
}
/// <summary>
/// Create a package item entry.
/// </summary>
public JObject CreateItem(PackageInput packageInput)
{
var rootUri = GetPackageUri(packageInput.Identity);
var json = JsonUtility.Create(rootUri, "Package");
json.Add("commitId", _context.CommitId.ToString().ToLowerInvariant());
json.Add("commitTimeStamp", DateTimeOffset.UtcNow.GetDateString());
json.Add("packageContent", packageInput.PackageDetails["packageContent"].ToString());
json.Add("registration", GetIndexUri(packageInput.Identity));
var copyProperties = new List <string>()
{
"@id",
"@type",
"authors",
"dependencyGroups",
"description",
"iconUrl",
"id",
"language",
"licenseUrl",
"listed",
"minClientVersion",
"packageContent",
"projectUrl",
"published",
"requireLicenseAcceptance",
"summary",
"tags",
"title",
"version"
};
var catalogEntry = new JObject();
JsonUtility.CopyProperties(packageInput.PackageDetails, catalogEntry, copyProperties, skipEmpty: true);
json.Add("catalogEntry", catalogEntry);
// Format package details at creation time, and avoid doing it again later to improve perf.
return(JsonLDTokenComparer.Format(json));
}
public async Task AddPackage(PackageInput package)
{
// Retrieve index
var rootUri = GetIndexUri(package.Identity);
var rootFile = _context.Source.Get(rootUri);
var packages = new List <JObject>();
if (await rootFile.Exists(_context.Log, _context.Token))
{
var json = await rootFile.GetJson(_context.Log, _context.Token);
// Get all entries
packages = await GetPackageDetails(json);
}
// Add entry
var newEntry = await CreateItem(package);
var removed = packages.RemoveAll(p => GetPackageVersion(p) == package.Identity.Version);
if (removed > 0)
{
_context.Log.LogWarning($"Removed duplicate registration entry for: {package.Identity}");
}
packages.Add(newEntry);
// Create index
var newIndexJson = CreateIndex(rootUri, packages);
// Write
await rootFile.Write(newIndexJson, _context.Log, _context.Token);
// Create package page
var packageUri = GetPackageUri(package.Identity);
var packageFile = _context.Source.Get(packageUri);
var packageJson = await CreatePackageBlob(package);
// Write package page
await packageFile.Write(packageJson, _context.Log, _context.Token);
}
public async Task AddPackage(PackageInput packageInput)
{
// Add nupkg
var nupkgFile = _context.Source.Get(GetNupkgPath(packageInput.Identity));
await nupkgFile.Write(File.OpenRead(packageInput.PackagePath), _context.Log, _context.Token);
// Add nuspec
var nuspecPath = $"{packageInput.Identity.Id}.nuspec".ToLowerInvariant();
var nuspecEntry = packageInput.Zip.Entries
.Where(entry => nuspecPath.Equals(nuspecPath, StringComparison.OrdinalIgnoreCase))
.FirstOrDefault();
if (nuspecEntry == null)
{
throw new InvalidDataException($"Unable to find '{nuspecPath}'. Path: '{packageInput.PackagePath}'.");
}
var entryFile = _context.Source.Get(GetZipFileUri(packageInput.Identity, nuspecPath));
using (var stream = nuspecEntry.Open())
using (var ms = new MemoryStream())
{
stream.CopyTo(ms);
ms.Seek(0, SeekOrigin.Begin);
await entryFile.Write(ms, _context.Log, _context.Token);
}
// Update index
var indexFile = _context.Source.Get(GetIndexUri(packageInput.Identity.Id));
var versions = await GetVersions(packageInput.Identity.Id);
versions.Add(packageInput.Identity.Version);
var indexJson = CreateIndex(versions);
await indexFile.Write(indexJson, _context.Log, _context.Token);
// Set nupkg url
packageInput.NupkgUri = nupkgFile.EntityUri;
}
private static Task <bool> AddZipEntry(PackageInput packageInput, Uri detailsUri, JArray packageEntriesArray)
{
var packageEntryIndex = 0;
// This method is called from RunWithLockAsync
foreach (var entry in packageInput.Zip.Entries.OrderBy(e => e.FullName, StringComparer.OrdinalIgnoreCase))
{
var fileEntry = JsonUtility.Create(detailsUri, $"packageEntry/{packageEntryIndex}", "packageEntry");
fileEntry.Add("fullName", entry.FullName);
fileEntry.Add("length", entry.Length);
fileEntry.Add("lastWriteTime", entry.LastWriteTime.GetDateString());
packageEntriesArray.Add(fileEntry);
packageEntryIndex++;
}
// Result is not used
return(Task.FromResult(true));
}
private async Task AddIconAsync(PackageInput packageInput)
{
// Find icon path in package from nuspec
var iconPath = packageInput.Nuspec.GetIcon();
if (!string.IsNullOrWhiteSpace(iconPath))
{
iconPath = PathUtility.StripLeadingDirectorySeparators(iconPath).Trim();
using (var zip = packageInput.CreateZip())
{
var entry = zip.GetEntry(iconPath);
if (entry != null)
{
var entryFile = _context.Source.Get(GetIconPath(packageInput.Identity));
await entryFile.Write(entry.Open(), _context.Log, _context.Token);
}
}
}
}
public async Task AddPackageAsync(PackageInput packageInput)
{
// Add nupkg
var nupkgFile = _context.Source.Get(GetNupkgPath(packageInput.Identity));
await nupkgFile.Write(File.OpenRead(packageInput.PackagePath), _context.Log, _context.Token);
// Add nuspec
var nuspecPath = $"{packageInput.Identity.Id}.nuspec".ToLowerInvariant();
var nuspecEntry = await packageInput.RunWithLockAsync((p) => Task.FromResult(p.Zip.Entries
.Where(entry => entry.FullName.Equals(nuspecPath, StringComparison.OrdinalIgnoreCase))
.FirstOrDefault()));
if (nuspecEntry == null)
{
throw new InvalidDataException($"Unable to find '{nuspecPath}'. Path: '{packageInput.PackagePath}'.");
}
var nuspecStream = await packageInput.RunWithLockAsync(async p => await nuspecEntry.Open().AsMemoryStreamAsync());
using (nuspecStream)
{
var entryFile = _context.Source.Get(GetZipFileUri(packageInput.Identity, nuspecPath));
await entryFile.Write(nuspecStream, _context.Log, _context.Token);
}
// Update index
var indexFile = _context.Source.Get(GetIndexUri(packageInput.Identity.Id));
var versions = await GetVersions(packageInput.Identity.Id);
versions.Add(packageInput.Identity.Version);
var indexJson = CreateIndex(versions);
await indexFile.Write(indexJson, _context.Log, _context.Token);
// Set nupkg url
packageInput.NupkgUri = nupkgFile.EntityUri;
}