/// <summary>
/// Enable package retention and update settings.
/// </summary>
public static async Task <bool> UpdateRetentionSettings(SleetContext context, int stableVersionMax, int prereleaseVersionMax)
{
var exitCode = true;
var log = context.Log;
var feedSettings = context.SourceSettings;
feedSettings.RetentionMaxStableVersions = stableVersionMax;
feedSettings.RetentionMaxPrereleaseVersions = prereleaseVersionMax;
await FeedSettingsUtility.SaveSettings(context.Source, feedSettings, log, context.Token);
// Commit changes to source
exitCode = await context.Source.Commit(log, context.Token);
if (exitCode)
{
await log.LogAsync(LogLevel.Minimal, $"Successfully updated package retention settings. Stable: {stableVersionMax} Prerelease: {prereleaseVersionMax}.");
await log.LogAsync(LogLevel.Minimal, $"Run prune to apply the new settings and remove packages from the feed.");
}
else
{
await log.LogAsync(LogLevel.Error, "Failed to update package retention settings.");
}
return(exitCode);
}
/// <summary>
/// Build pipeline steps.
/// </summary>
private static List <SleetStep> GetSteps(SleetContext context)
{
var result = new List <SleetStep>();
var catalog = new SleetStep(GetCatalogService(context), SleetPerfGroup.Catalog);
result.Add(catalog);
if (context.SourceSettings.SymbolsEnabled)
{
result.Add(new SleetStep(new Symbols(context), SleetPerfGroup.Symbols));
}
result.Add(new SleetStep(new FlatContainer(context), SleetPerfGroup.FlatContainer));
result.Add(new SleetStep(new AutoComplete(context), SleetPerfGroup.AutoComplete));
result.Add(new SleetStep(new PackageIndex(context), SleetPerfGroup.PackageIndex));
// Registration depends on catalog pages
var registrations = new SleetStep(new Registrations(context), SleetPerfGroup.Registration, catalog);
result.Add(registrations);
// Search depends on registation files
var search = new SleetStep(new Search(context), SleetPerfGroup.Search, registrations);
result.Add(search);
return(result);
}
/// <summary>
/// Update all feed badges
/// </summary>
public static Task UpdateBadges(SleetContext context, ISet <PackageIdentity> before, ISet <PackageIdentity> after)
{
var stable = GetChanges(before, after, preRel: false);
var pre = GetChanges(before, after, preRel: true);
return(Task.WhenAll(UpdateBadges(context, stable, preRel: false), UpdateBadges(context, pre, preRel: true)));
}
public static Uri GetIconPath(SleetContext context, PackageIdentity package)
{
var id = package.Id;
var version = package.Version.ToIdentityString();
return(context.Source.GetPath($"/flatcontainer/{id}/{version}/icon".ToLowerInvariant()));
}
/// <summary>
/// Run prune
/// 1. Lock the feed
/// 2. Verify client compat
/// 3. Prune packges
/// 4. Commit
/// </summary>
public static async Task <bool> RunAsync(LocalSettings settings, ISleetFileSystem source, RetentionPruneCommandContext pruneContext, ILogger log)
{
var exitCode = true;
log.LogMinimal($"Pruning packages in {source.BaseURI}");
var token = CancellationToken.None;
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(settings, source, "Prune", log, token))
{
// Validate source
await UpgradeUtility.EnsureCompatibility(source, log, token);
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
exitCode = await PrunePackages(context, pruneContext);
}
return(exitCode);
}
public static IReadOnlyList <ISleetService> GetServices(SleetContext context)
{
// Order is important here
// Packages must be added to flat container, then the catalog, then registrations.
var services = new List <ISleetService>
{
new FlatContainer(context)
};
if (context.SourceSettings.CatalogEnabled)
{
// Catalog on disk
services.Add(new Catalog(context));
}
else
{
// In memory catalog
services.Add(new VirtualCatalog(context));
}
services.Add(new Registrations(context));
services.Add(new AutoComplete(context));
services.Add(new Search(context));
services.Add(new PackageIndex(context));
// Symbols
if (context.SourceSettings.SymbolsEnabled)
{
services.Add(new Symbols(context));
}
return(services);
}
/// <summary>
/// Update feed badges for stable or prerelease
/// </summary>
public static async Task UpdateBadges(SleetContext context, ISet <PackageIdentity> updates, bool preRel)
{
foreach (var package in updates)
{
await UpdateOrRemoveBadge(context, package, preRel);
}
}
/// <summary>
/// Prune feed packages without committing
/// </summary>
public static async Task <HashSet <PackageIdentity> > PrunePackagesNoCommit(SleetContext context, RetentionPruneCommandContext pruneContext)
{
var packageIndex = new PackageIndex(context);
var existingPackageSets = await packageIndex.GetPackageSetsAsync();
var allPackages = await RetentionUtility.ResolvePackageSets(existingPackageSets);
var stableMax = pruneContext.StableVersionMax == null ? context.SourceSettings.RetentionMaxStableVersions : pruneContext.StableVersionMax;
var prerelMax = pruneContext.PrereleaseVersionMax == null ? context.SourceSettings.RetentionMaxPrereleaseVersions : pruneContext.PrereleaseVersionMax;
if (stableMax == null || stableMax < 1)
{
throw new ArgumentException("Package retention must specify a maximum number of stable versions that is > 0");
}
if (prerelMax == null || prerelMax < 1)
{
throw new ArgumentException("Package retention must specify a maximum number of prerelease versions that is > 0");
}
if (pruneContext.PackageIds?.Count > 0)
{
allPackages.RemoveWhere(e => !pruneContext.PackageIds.Contains(e.Id));
}
var toPrune = RetentionUtility.GetPackagesToPrune(allPackages, pruneContext.PinnedPackages, (int)stableMax, (int)prerelMax);
await RemovePackages(context, existingPackageSets, toPrune, pruneContext.DryRun, context.Log);
return(toPrune);
}
/// <summary>
/// Remove package retention settings.
/// </summary>
public static async Task <bool> DisableRetention(SleetContext context)
{
var exitCode = true;
var log = context.Log;
var feedSettings = context.SourceSettings;
feedSettings.RetentionMaxStableVersions = null;
feedSettings.RetentionMaxPrereleaseVersions = null;
await FeedSettingsUtility.SaveSettings(context.Source, feedSettings, log, context.Token);
// Commit changes to source
exitCode = await context.Source.Commit(log, context.Token);
if (exitCode)
{
await log.LogAsync(LogLevel.Minimal, "Package retention disabled.");
}
else
{
await log.LogAsync(LogLevel.Error, "Failed to disable retention settings.");
}
return(exitCode);
}
public Catalog(SleetContext context, Uri catalogBaseURI)
{
_context = context;
var fileSystemBase = (FileSystemBase)context.Source;
CatalogBaseURI = fileSystemBase.FeedSubPath == null ? catalogBaseURI : UriUtility.GetPath(context.Source.BaseURI, fileSystemBase.FeedSubPath, "catalog/");
}
/// <summary>
/// Update all feed badges
/// </summary>
public static async Task UpdateBadges(SleetContext context, ISet <PackageIdentity> before, ISet <PackageIdentity> after)
{
var stable = GetChanges(before, after, preRel: false);
var pre = GetChanges(before, after, preRel: true);
await UpdateBadges(context, stable, preRel : false);
await UpdateBadges(context, pre, preRel : true);
}
/// <summary>
/// Returns the nupkg URI. This will be different for Symbols packages.
/// </summary>
public Uri GetNupkgUri(SleetContext context)
{
if (IsSymbolsPackage)
{
return(context.Source.Get(SymbolsIndexUtility.GetSymbolsNupkgPath(Identity)).EntityUri);
}
return(FlatContainer.GetNupkgPath(context, Identity));
}
public static Dictionary <string, IFeedSettingHandler> GetSettingHandlers(SleetContext context)
{
var search = new ExternalSearchHandler(context);
return(new Dictionary <string, IFeedSettingHandler>()
{
{ search.Name, search }
});
}
/// <summary>
/// Add a package to all services.
/// </summary>
public static async Task AddPackage(SleetContext context, PackageInput package)
{
var services = GetServices(context);
foreach (var service in services)
{
await service.AddPackage(package);
}
}
/// <summary>
/// Remove a package from all services.
/// </summary>
public static async Task RemovePackage(SleetContext context, PackageIdentity package)
{
var services = GetServices(context);
foreach (var service in services)
{
await service.RemovePackage(package);
}
}
private static async Task RemovePackages(bool force, ILogger log, SleetContext context, PackageSets existingPackageSets, HashSet <PackageIdentity> packages)
{
var toRemove = new HashSet <PackageIdentity>();
var toRemoveSymbols = new HashSet <PackageIdentity>();
foreach (var package in packages)
{
var exists = existingPackageSets.Packages.Exists(package);
var symbolsExists = existingPackageSets.Symbols.Exists(package);
if (!exists && !symbolsExists)
{
log.LogInformation($"{package.ToString()} does not exist.");
if (force)
{
// ignore failures
continue;
}
else
{
throw new InvalidOperationException($"Package does not exists: {package.ToString()}");
}
}
if (exists)
{
toRemove.Add(package);
}
if (symbolsExists)
{
toRemoveSymbols.Add(package);
}
var message = $"Removing {package.ToString()}";
if (exists && symbolsExists)
{
message = $"Removing {package.ToString()} and symbols package for {package.ToString()}";
}
else if (symbolsExists)
{
message = $"Removing symbols package {package.ToString()}";
}
await log.LogAsync(LogLevel.Information, message);
}
// Update feed
await log.LogAsync(LogLevel.Information, "Removing packages from feed locally");
// Add/Remove packages
var changeContext = SleetOperations.CreateDelete(existingPackageSets, toRemove, toRemoveSymbols);
await SleetUtility.ApplyPackageChangesAsync(context, changeContext);
}
/// <summary>
/// Lock the feed and delete all packages given.
/// </summary>
public static async Task <bool> DeletePackagesAsync(LocalSettings settings, ISleetFileSystem source, IEnumerable <PackageIdentity> packagesToDelete, string reason, bool force, ILogger log)
{
var success = true;
var token = CancellationToken.None;
log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(settings, source, "Delete", log, token))
{
// Validate source
await SourceUtility.ValidateFeedForClient(source, log, token);
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
var packageIndex = new PackageIndex(context);
var existingPackageSets = await packageIndex.GetPackageSetsAsync();
var packages = new HashSet <PackageIdentity>(packagesToDelete);
if (string.IsNullOrEmpty(reason))
{
reason = string.Empty;
}
await RemovePackages(force, log, context, existingPackageSets, packages);
// Save all
log.LogMinimal($"Committing changes to {source.BaseURI.AbsoluteUri}");
success &= await source.Commit(log, token);
}
if (success)
{
log.LogMinimal($"Successfully deleted packages.");
}
else
{
log.LogError($"Failed to delete packages.");
}
return(success);
}
public static async Task <int> RunCore(LocalSettings settings, ISleetFileSystem source, List <string> inputs, bool force, ILogger log)
{
var exitCode = 0;
var token = CancellationToken.None;
var now = DateTimeOffset.UtcNow;
// Get packages
var packages = GetPackageInputs(inputs, now, log);
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(source, log, token))
{
// Validate source
await UpgradeUtility.UpgradeIfNeeded(source, log, token);
// Get sleet.settings.json
var sourceSettings = new SourceSettings();
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
var packageIndex = new PackageIndex(context);
foreach (var package in packages)
{
if (await packageIndex.Exists(package.Identity))
{
if (force)
{
log.LogInformation($"Package already exists, removing {package.ToString()}");
await SleetUtility.RemovePackage(context, package.Identity);
}
else
{
throw new InvalidOperationException($"Package already exists: '{package.Identity}'.");
}
}
log.LogInformation($"Adding {package.Identity.ToString()}");
await SleetUtility.AddPackage(context, package);
}
// Save all
await source.Commit(log, token);
}
return(exitCode);
}
/// <summary>
/// Main entry point for updating the feed.
/// All add/remove operations can be added to a single changeContext and applied in
/// a single call using this method.
/// </summary>
public static async Task ApplyPackageChangesAsync(SleetContext context, SleetOperations changeContext)
{
var steps = GetSteps(context);
var tasks = steps.Select(e => new Func <Task>(() => e.RunAsync(changeContext))).ToList();
// Run each service on its own thread and in parallel
// Services with depenencies will pre-fetch files that will be used later
// and then wait until the other services have completed.
await TaskUtils.RunAsync(tasks, useTaskRun : true, maxThreads : steps.Count, token : CancellationToken.None);
}
/// <summary>
/// Remove a symbols package from all services.
/// </summary>
public static async Task RemoveSymbolsPackage(SleetContext context, PackageIdentity package)
{
var services = GetServices(context);
var symbolsEnabled = context.SourceSettings.SymbolsEnabled;
if (symbolsEnabled)
{
foreach (var symbolsService in services.Select(e => e as ISymbolsAddRemovePackages).Where(e => e != null))
{
await symbolsService.RemoveSymbolsPackageAsync(package);
}
}
}
/// <summary>
/// Main entry point for updating the feed.
/// All add/remove operations can be added to a single changeContext and applied in
/// a single call using this method.
/// </summary>
public static async Task ApplyPackageChangesAsync(SleetContext context, SleetOperations changeContext)
{
using (var timer = PerfEntryWrapper.CreateSummaryTimer("Updated all files locally. Total time: {0}", context.PerfTracker))
{
var steps = GetSteps(context);
var tasks = steps.Select(e => new Func <Task>(() => e.RunAsync(changeContext, context.PerfTracker))).ToList();
// Run each service on its own thread and in parallel
// Services with depenencies will pre-fetch files that will be used later
// and then wait until the other services have completed.
await TaskUtils.RunAsync(tasks, useTaskRun : true, maxThreads : steps.Count, token : CancellationToken.None);
}
}
/// <summary>
/// Push packages to a feed.
/// This assumes the feed is already locked.
/// </summary>
public static async Task <bool> PushPackages(LocalSettings settings, ISleetFileSystem source, List <PackageInput> packages, bool force, bool skipExisting, ILogger log, CancellationToken token)
{
var exitCode = true;
var now = DateTimeOffset.UtcNow;
// Verify no duplicate packages
CheckForDuplicates(packages);
// Get sleet.settings.json
await log.LogAsync(LogLevel.Minimal, "Reading feed");
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token,
PerfTracker = source.LocalCache.PerfTracker
};
await log.LogAsync(LogLevel.Verbose, "Reading existing package index");
// Push
var packageIndex = new PackageIndex(context);
var existingPackageSets = await packageIndex.GetPackageSetsAsync();
await PushPackages(packages, context, existingPackageSets, force, skipExisting, log);
// Prune packages
await PrunePackages(packages, context);
// Save all
await log.LogAsync(LogLevel.Minimal, $"Committing changes to {source.BaseURI.AbsoluteUri}");
await source.Commit(log, token);
if (exitCode)
{
await log.LogAsync(LogLevel.Minimal, "Successfully pushed packages.");
}
else
{
await log.LogAsync(LogLevel.Error, "Failed to push packages.");
}
return(exitCode);
}
public static IReadOnlyList <ISleetService> GetServices(SleetContext context)
{
// Order is important here
// Packages must be added to flat container, then the catalog, then registrations.
return(new List <ISleetService>()
{
new FlatContainer(context),
new Catalog(context),
new Registrations(context),
new AutoComplete(context),
new Search(context),
new PackageIndex(context),
});
}
private static async Task PrunePackages(List <PackageInput> packages, SleetContext context)
{
// Run only if the package retention settings are present for the feed
if (context.SourceSettings.RetentionMaxStableVersions > 0 && context.SourceSettings.RetentionMaxPrereleaseVersions > 0)
{
// Prune if prune is enabled for the feed
var pruneContext = new RetentionPruneCommandContext();
// Do not prune packages that were just pushed
pruneContext.PinnedPackages.UnionWith(packages.Select(e => e.Identity));
// Only prune package ids that were pushed
pruneContext.PackageIds.UnionWith(packages.Select(e => e.Identity.Id));
// Run prune against the local files
await RetentionPruneCommand.PrunePackagesNoCommit(context, pruneContext);
}
}
/// <summary>
/// Returns the nupkg icon URI..
/// </summary>
public Uri GetIconUri(SleetContext context)
{
Uri result = null;
// Ignore symbol packages, icons from these are not used
if (!IsSymbolsPackage)
{
// Ignore legacy iconURL nuspec properties
if (!string.IsNullOrEmpty(Nuspec.GetIcon()))
{
result = FlatContainer.GetIconPath(context, Identity);
}
}
return(result);
}
private static async Task <bool> CreatePackageIndexAsync(SleetContext context, JObject serviceIndexJson)
{
var packageIndex = context.Source.Get("sleet.packageindex.json");
AddServiceIndexEntry(context.Source.BaseURI, "sleet.packageindex.json", "http://schema.emgarten.com/sleet#PackageIndex/1.0.0", "Sleet package index.", serviceIndexJson);
if (!await packageIndex.Exists(context.Log, context.Token))
{
var index = new PackageIndex(context);
await index.InitAsync();
return(true);
}
return(false);
}
/// <summary>
/// Retrieve the catalog service based on the settings.
/// </summary>
private static ISleetService GetCatalogService(SleetContext context)
{
ISleetService catalog = null;
if (context.SourceSettings.CatalogEnabled)
{
// Full catalog that is written to the feed
catalog = new Catalog(context);
}
else
{
// In memory catalog that is not written to the feed
catalog = new VirtualCatalog(context);
}
return(catalog);
}
/// <summary>
/// Write or remove badge file from the feed.
/// </summary>
public static async Task UpdateOrRemoveBadge(SleetContext context, PackageIdentity package, bool preRel)
{
var svgPath = $"badges/{(preRel ? "vpre" : "v")}/{package.Id}.svg".ToLowerInvariant();
var jsonPath = $"badges/{(preRel ? "vpre" : "v")}/{package.Id}.json".ToLowerInvariant();
var svgFile = context.Source.Get(svgPath);
var jsonFile = context.Source.Get(jsonPath);
// If the identity doesn't have it version then it should be removed
if (package.HasVersion)
{
await jsonFile.Write(GetJsonBadge(package), context.Log, context.Token);
}
else
{
svgFile.Delete(context.Log, context.Token);
jsonFile.Delete(context.Log, context.Token);
}
}
public static async Task <int> RunCore(LocalSettings settings, ISleetFileSystem source, ILogger log)
{
var exitCode = 0;
log.LogMinimal($"Stats for {source.BaseURI}");
var token = CancellationToken.None;
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(source, log, token))
{
// Validate source
await UpgradeUtility.UpgradeIfNeeded(source, log, token);
// Get sleet.settings.json
var sourceSettings = new SourceSettings();
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
var catalog = new Catalog(context);
var existingEntries = await catalog.GetExistingPackagesIndex();
var packages = existingEntries.Select(e => e.PackageIdentity).ToList();
var uniqueIds = packages.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase);
var catalogEntries = await catalog.GetIndexEntries();
log.LogMinimal($"Catalog entries: {catalogEntries.Count}");
log.LogMinimal($"Packages: {existingEntries.Count}");
log.LogMinimal($"Unique package ids: {uniqueIds.Count()}");
}
return(exitCode);
}
public static async Task <bool> RunAsync(LocalSettings settings, ISleetFileSystem source, ILogger log)
{
var exitCode = true;
log.LogMinimal($"Stats for {source.BaseURI}");
var token = CancellationToken.None;
// Check if already initialized
using (var feedLock = await SourceUtility.VerifyInitAndLock(settings, source, "Stats", log, token))
{
// Validate source
await UpgradeUtility.EnsureFeedVersionMatchesTool(source, log, token);
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
var packageIndex = new PackageIndex(context);
var existingPackageSets = await packageIndex.GetPackageSetsAsync();
var uniqueIds = existingPackageSets.Packages.Index
.Concat(existingPackageSets.Symbols.Index)
.Select(e => e.Id).Distinct(StringComparer.OrdinalIgnoreCase);
log.LogMinimal($"Packages: {existingPackageSets.Packages.Index.Count}");
log.LogMinimal($"Symbols Packages: {existingPackageSets.Symbols.Index.Count}");
log.LogMinimal($"Unique package ids: {uniqueIds.Count()}");
}
return(exitCode);
}