public async Task <bool> Commit(ILogger log, CancellationToken token)
{
var perfTracker = LocalCache.PerfTracker;
// Find all files with changes
var withChanges = Files.Values.Where(e => e.HasChanges).ToList();
// Order files so that nupkgs are pushed first to help clients avoid
// missing files during the push.
withChanges.Sort(new SleetFileComparer());
if (withChanges.Count > 0)
{
var bytes = withChanges.Select(e => e as FileBase)
.Where(e => e != null)
.Sum(e => e.LocalFileSizeIfExists);
// Create tasks to run in parallel
var tasks = withChanges.Select(e => GetCommitFileFunc(e, log, token));
var message = $"Files committed: {withChanges.Count} Size: {PrintUtility.GetBytesString(bytes)} Total upload time: " + "{0}";
using (var timer = PerfEntryWrapper.CreateSummaryTimer(message, perfTracker))
{
// Push in parallel
await TaskUtils.RunAsync(
tasks : tasks,
useTaskRun : true,
maxThreads : 8,
token : token);
}
}
return(true);
}
public Task ApplyOperationsAsync(SleetOperations changeContext)
{
// Remove existing files, this will typically result in marking the files
// as deleted in the virtual file system since they have not been
// downloaded. It is a fast/safe operation that must be done first.
foreach (var toRemove in changeContext.ToRemove)
{
if (!toRemove.IsSymbolsPackage)
{
DeleteNupkg(toRemove.Identity);
}
}
var tasks = new List <Func <Task> >();
// Copy in nupkgs/nuspec files
// Ignore symbols packages
tasks.AddRange(changeContext.ToAdd.Where(e => !e.IsSymbolsPackage).Select(e => new Func <Task>(() => AddPackageAsync(e))));
// Rebuild index files as needed
var rebuildIds = changeContext.GetChangedIds();
tasks.AddRange(rebuildIds.Select(e => new Func <Task>(() => CreateIndexAsync(e, changeContext.UpdatedIndex.Packages))));
// Run all tasks
return(TaskUtils.RunAsync(tasks));
}
public Task AddPackagesAsync(IEnumerable <PackageInput> packageInputs)
{
// Create package details page
var tasks = packageInputs.Select(e => new Func <Task>(() => CreateDetailsForAdd(e)));
return(TaskUtils.RunAsync(tasks, useTaskRun: true, token: CancellationToken.None));
}
public async Task RemovePackagesAsync(IEnumerable <PackageIdentity> packages)
{
// Write catalog remove pages for each package
var tasks = packages.Select(e => new Func <Task <JObject> >(() => GetRemoveCommit(e)));
var pageCommits = await TaskUtils.RunAsync(tasks);
// Add pages to the index as commits
await AddCatalogCommits(pageCommits, "nuget:lastDeleted");
}
public async Task AddPackagesAsync(IEnumerable <PackageInput> packageInputs)
{
// Write catalog pages for each package
var tasks = packageInputs.Select(e => new Func <Task <JObject> >(() => AddPackageToCatalogAndGetCommit(e)));
var pageCommits = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None);
// Add pages to the index as commits
await AddCatalogCommits(pageCommits, "nuget:lastCreated");
}
/// <summary>
/// Parse input arguments for nupkg paths.
/// </summary>
private static async Task<List<PackageInput>> GetPackageInputs(List<string> packagePaths, DateTimeOffset now, IPerfTracker perfTracker, ILogger log)
{
using (var timer = PerfEntryWrapper.CreateSummaryTimer("Loaded package nuspecs in {0}", perfTracker))
{
var tasks = packagePaths.Select(e => new Func<Task<PackageInput>>(() => GetPackageInput(e, log)));
var packageInputs = await TaskUtils.RunAsync(tasks, useTaskRun: true, token: CancellationToken.None);
var packagesSorted = packageInputs.OrderBy(e => e).ToList();
return packagesSorted;
}
}
public async Task <bool> Commit(ILogger log, CancellationToken token)
{
// Push in parallel
await TaskUtils.RunAsync(
tasks : Files.Values.Select(e => GetCommitFileFunc(e, log, token)),
useTaskRun : true,
token : token);
return(true);
}
/// <summary>
/// Main entry point for updating the feed.
/// All add/remove operations can be added to a single changeContext and applied in
/// a single call using this method.
/// </summary>
public static async Task ApplyPackageChangesAsync(SleetContext context, SleetOperations changeContext)
{
var steps = GetSteps(context);
var tasks = steps.Select(e => new Func <Task>(() => e.RunAsync(changeContext))).ToList();
// Run each service on its own thread and in parallel
// Services with depenencies will pre-fetch files that will be used later
// and then wait until the other services have completed.
await TaskUtils.RunAsync(tasks, useTaskRun : true, maxThreads : steps.Count, token : CancellationToken.None);
}
public Task AddPackagesAsync(IEnumerable <PackageInput> packageInputs)
{
var byId = SleetUtility.GetPackageSetsById(packageInputs, e => e.Identity.Id);
var tasks = new List <Func <Task> >();
// Create page details pages and index pages in parallel.
tasks.AddRange(byId.Select(e => new Func <Task>(() => CreatePackageIndexAsync(e.Key, e.Value))));
tasks.AddRange(packageInputs.Select(e => new Func <Task>(() => CreatePackagePageAsync(e))));
return(TaskUtils.RunAsync(tasks));
}
/// <summary>
/// Main entry point for updating the feed.
/// All add/remove operations can be added to a single changeContext and applied in
/// a single call using this method.
/// </summary>
public static async Task ApplyPackageChangesAsync(SleetContext context, SleetOperations changeContext)
{
using (var timer = PerfEntryWrapper.CreateSummaryTimer("Updated all files locally. Total time: {0}", context.PerfTracker))
{
var steps = GetSteps(context);
var tasks = steps.Select(e => new Func <Task>(() => e.RunAsync(changeContext, context.PerfTracker))).ToList();
// Run each service on its own thread and in parallel
// Services with depenencies will pre-fetch files that will be used later
// and then wait until the other services have completed.
await TaskUtils.RunAsync(tasks, useTaskRun : true, maxThreads : steps.Count, token : CancellationToken.None);
}
}
public Task RemovePackagesAsync(IEnumerable <PackageIdentity> packagesToDelete)
{
var byId = SleetUtility.GetPackageSetsById(packagesToDelete, e => e.Id);
var tasks = new List <Func <Task> >();
foreach (var pair in byId)
{
var packageId = pair.Key;
var versions = new HashSet <NuGetVersion>(pair.Value.Select(e => e.Version));
tasks.Add(new Func <Task>(() => RemovePackagesFromIndexAsync(packageId, versions)));
}
return(TaskUtils.RunAsync(tasks));
}
/// <summary>
/// Parse input arguments for nupkg paths.
/// </summary>
private static async Task <List <PackageInput> > GetPackageInputs(List <string> inputs, DateTimeOffset now, ILogger log)
{
// Check inputs
if (inputs.Count < 1)
{
throw new ArgumentException("No packages found.");
}
// Get package inputs
var packagePaths = inputs.SelectMany(GetFiles)
.Distinct(PathUtility.GetStringComparerBasedOnOS())
.ToList();
var tasks = packagePaths.Select(e => new Func <Task <PackageInput> >(() => GetPackageInput(e, log)));
var packageInputs = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None);
var packagesSorted = packageInputs.OrderBy(e => e).ToList();
return(packagesSorted);
}
/// <summary>
/// Download packages. This method does not lock the feed or verify the client version.
/// </summary>
public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, bool skipExisting, ILogger log, CancellationToken token)
{
if (string.IsNullOrEmpty(outputPath))
{
throw new ArgumentException("Missing output path parameter!");
}
var success = true;
// Get sleet.settings.json
var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token);
// Settings context used for all operations
var context = new SleetContext()
{
LocalSettings = settings,
SourceSettings = sourceSettings,
Log = log,
Source = source,
Token = token
};
log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}");
// Find all packages
var packageIndex = new PackageIndex(context);
var flatContainer = new FlatContainer(context);
var symbols = new Symbols(context);
// Discover all packages
var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >();
packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e)))));
packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e =>
new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e))));
log.LogMinimal($"Downloading nupkgs to {outputPath}");
// Run downloads
var tasks = packages.Select(e => new Func <Task <bool> >(() => DownloadPackageAsync(outputPath, skipExisting, log, e, token)));
var results = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None);
var downloadSuccess = results.All(e => e);
success &= downloadSuccess;
if (packages.Count < 1)
{
log.LogWarning("The feed does not contain any packages.");
}
if (downloadSuccess)
{
if (packages.Count > 0)
{
log.LogMinimal("Successfully downloaded packages.");
}
}
else
{
var message = $"Failed to download all packages!";
if (ignoreErrors)
{
log.LogWarning(message);
}
else
{
log.LogError(message);
}
}
return(success);
}
public Task PreLoadAsync(SleetOperations operations)
{
// Retrieve the index pages for all ids we will later modify.
return(TaskUtils.RunAsync(operations.GetChangedIds().Select(e => new Func <Task>(() => FetchPageAsync(e)))));
}
/// <summary>
/// Update feed badges for stable or prerelease
/// </summary>
public static Task UpdateBadges(SleetContext context, ISet <PackageIdentity> updates, bool preRel)
{
var tasks = new List <Func <Task> >(updates.Select(e => new Func <Task>(() => UpdateOrRemoveBadge(context, e, preRel))));
return(TaskUtils.RunAsync(tasks));
}
