public async Task <bool> Commit(ILogger log, CancellationToken token) { var perfTracker = LocalCache.PerfTracker; // Find all files with changes var withChanges = Files.Values.Where(e => e.HasChanges).ToList(); // Order files so that nupkgs are pushed first to help clients avoid // missing files during the push. withChanges.Sort(new SleetFileComparer()); if (withChanges.Count > 0) { var bytes = withChanges.Select(e => e as FileBase) .Where(e => e != null) .Sum(e => e.LocalFileSizeIfExists); // Create tasks to run in parallel var tasks = withChanges.Select(e => GetCommitFileFunc(e, log, token)); var message = $"Files committed: {withChanges.Count} Size: {PrintUtility.GetBytesString(bytes)} Total upload time: " + "{0}"; using (var timer = PerfEntryWrapper.CreateSummaryTimer(message, perfTracker)) { // Push in parallel await TaskUtils.RunAsync( tasks : tasks, useTaskRun : true, maxThreads : 8, token : token); } } return(true); }
public Task ApplyOperationsAsync(SleetOperations changeContext) { // Remove existing files, this will typically result in marking the files // as deleted in the virtual file system since they have not been // downloaded. It is a fast/safe operation that must be done first. foreach (var toRemove in changeContext.ToRemove) { if (!toRemove.IsSymbolsPackage) { DeleteNupkg(toRemove.Identity); } } var tasks = new List <Func <Task> >(); // Copy in nupkgs/nuspec files // Ignore symbols packages tasks.AddRange(changeContext.ToAdd.Where(e => !e.IsSymbolsPackage).Select(e => new Func <Task>(() => AddPackageAsync(e)))); // Rebuild index files as needed var rebuildIds = changeContext.GetChangedIds(); tasks.AddRange(rebuildIds.Select(e => new Func <Task>(() => CreateIndexAsync(e, changeContext.UpdatedIndex.Packages)))); // Run all tasks return(TaskUtils.RunAsync(tasks)); }
public Task AddPackagesAsync(IEnumerable <PackageInput> packageInputs) { // Create package details page var tasks = packageInputs.Select(e => new Func <Task>(() => CreateDetailsForAdd(e))); return(TaskUtils.RunAsync(tasks, useTaskRun: true, token: CancellationToken.None)); }
public async Task RemovePackagesAsync(IEnumerable <PackageIdentity> packages) { // Write catalog remove pages for each package var tasks = packages.Select(e => new Func <Task <JObject> >(() => GetRemoveCommit(e))); var pageCommits = await TaskUtils.RunAsync(tasks); // Add pages to the index as commits await AddCatalogCommits(pageCommits, "nuget:lastDeleted"); }
public async Task AddPackagesAsync(IEnumerable <PackageInput> packageInputs) { // Write catalog pages for each package var tasks = packageInputs.Select(e => new Func <Task <JObject> >(() => AddPackageToCatalogAndGetCommit(e))); var pageCommits = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None); // Add pages to the index as commits await AddCatalogCommits(pageCommits, "nuget:lastCreated"); }
/// <summary> /// Parse input arguments for nupkg paths. /// </summary> private static async Task<List<PackageInput>> GetPackageInputs(List<string> packagePaths, DateTimeOffset now, IPerfTracker perfTracker, ILogger log) { using (var timer = PerfEntryWrapper.CreateSummaryTimer("Loaded package nuspecs in {0}", perfTracker)) { var tasks = packagePaths.Select(e => new Func<Task<PackageInput>>(() => GetPackageInput(e, log))); var packageInputs = await TaskUtils.RunAsync(tasks, useTaskRun: true, token: CancellationToken.None); var packagesSorted = packageInputs.OrderBy(e => e).ToList(); return packagesSorted; } }
public async Task <bool> Commit(ILogger log, CancellationToken token) { // Push in parallel await TaskUtils.RunAsync( tasks : Files.Values.Select(e => GetCommitFileFunc(e, log, token)), useTaskRun : true, token : token); return(true); }
/// <summary> /// Main entry point for updating the feed. /// All add/remove operations can be added to a single changeContext and applied in /// a single call using this method. /// </summary> public static async Task ApplyPackageChangesAsync(SleetContext context, SleetOperations changeContext) { var steps = GetSteps(context); var tasks = steps.Select(e => new Func <Task>(() => e.RunAsync(changeContext))).ToList(); // Run each service on its own thread and in parallel // Services with depenencies will pre-fetch files that will be used later // and then wait until the other services have completed. await TaskUtils.RunAsync(tasks, useTaskRun : true, maxThreads : steps.Count, token : CancellationToken.None); }
public Task AddPackagesAsync(IEnumerable <PackageInput> packageInputs) { var byId = SleetUtility.GetPackageSetsById(packageInputs, e => e.Identity.Id); var tasks = new List <Func <Task> >(); // Create page details pages and index pages in parallel. tasks.AddRange(byId.Select(e => new Func <Task>(() => CreatePackageIndexAsync(e.Key, e.Value)))); tasks.AddRange(packageInputs.Select(e => new Func <Task>(() => CreatePackagePageAsync(e)))); return(TaskUtils.RunAsync(tasks)); }
/// <summary> /// Main entry point for updating the feed. /// All add/remove operations can be added to a single changeContext and applied in /// a single call using this method. /// </summary> public static async Task ApplyPackageChangesAsync(SleetContext context, SleetOperations changeContext) { using (var timer = PerfEntryWrapper.CreateSummaryTimer("Updated all files locally. Total time: {0}", context.PerfTracker)) { var steps = GetSteps(context); var tasks = steps.Select(e => new Func <Task>(() => e.RunAsync(changeContext, context.PerfTracker))).ToList(); // Run each service on its own thread and in parallel // Services with depenencies will pre-fetch files that will be used later // and then wait until the other services have completed. await TaskUtils.RunAsync(tasks, useTaskRun : true, maxThreads : steps.Count, token : CancellationToken.None); } }
public Task RemovePackagesAsync(IEnumerable <PackageIdentity> packagesToDelete) { var byId = SleetUtility.GetPackageSetsById(packagesToDelete, e => e.Id); var tasks = new List <Func <Task> >(); foreach (var pair in byId) { var packageId = pair.Key; var versions = new HashSet <NuGetVersion>(pair.Value.Select(e => e.Version)); tasks.Add(new Func <Task>(() => RemovePackagesFromIndexAsync(packageId, versions))); } return(TaskUtils.RunAsync(tasks)); }
/// <summary> /// Parse input arguments for nupkg paths. /// </summary> private static async Task <List <PackageInput> > GetPackageInputs(List <string> inputs, DateTimeOffset now, ILogger log) { // Check inputs if (inputs.Count < 1) { throw new ArgumentException("No packages found."); } // Get package inputs var packagePaths = inputs.SelectMany(GetFiles) .Distinct(PathUtility.GetStringComparerBasedOnOS()) .ToList(); var tasks = packagePaths.Select(e => new Func <Task <PackageInput> >(() => GetPackageInput(e, log))); var packageInputs = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None); var packagesSorted = packageInputs.OrderBy(e => e).ToList(); return(packagesSorted); }
/// <summary> /// Download packages. This method does not lock the feed or verify the client version. /// </summary> public static async Task <bool> DownloadPackages(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, bool skipExisting, ILogger log, CancellationToken token) { if (string.IsNullOrEmpty(outputPath)) { throw new ArgumentException("Missing output path parameter!"); } var success = true; // Get sleet.settings.json var sourceSettings = await FeedSettingsUtility.GetSettingsOrDefault(source, log, token); // Settings context used for all operations var context = new SleetContext() { LocalSettings = settings, SourceSettings = sourceSettings, Log = log, Source = source, Token = token }; log.LogMinimal($"Reading feed {source.BaseURI.AbsoluteUri}"); // Find all packages var packageIndex = new PackageIndex(context); var flatContainer = new FlatContainer(context); var symbols = new Symbols(context); // Discover all packages var packages = new List <KeyValuePair <PackageIdentity, ISleetFile> >(); packages.AddRange((await packageIndex.GetPackagesAsync()).Select(e => new KeyValuePair <PackageIdentity, ISleetFile>(e, context.Source.Get(flatContainer.GetNupkgPath(e))))); packages.AddRange((await packageIndex.GetSymbolsPackagesAsync()).Select(e => new KeyValuePair <PackageIdentity, ISleetFile>(e, symbols.GetSymbolsNupkgFile(e)))); log.LogMinimal($"Downloading nupkgs to {outputPath}"); // Run downloads var tasks = packages.Select(e => new Func <Task <bool> >(() => DownloadPackageAsync(outputPath, skipExisting, log, e, token))); var results = await TaskUtils.RunAsync(tasks, useTaskRun : true, token : CancellationToken.None); var downloadSuccess = results.All(e => e); success &= downloadSuccess; if (packages.Count < 1) { log.LogWarning("The feed does not contain any packages."); } if (downloadSuccess) { if (packages.Count > 0) { log.LogMinimal("Successfully downloaded packages."); } } else { var message = $"Failed to download all packages!"; if (ignoreErrors) { log.LogWarning(message); } else { log.LogError(message); } } return(success); }
public Task PreLoadAsync(SleetOperations operations) { // Retrieve the index pages for all ids we will later modify. return(TaskUtils.RunAsync(operations.GetChangedIds().Select(e => new Func <Task>(() => FetchPageAsync(e))))); }
/// <summary> /// Update feed badges for stable or prerelease /// </summary> public static Task UpdateBadges(SleetContext context, ISet <PackageIdentity> updates, bool preRel) { var tasks = new List <Func <Task> >(updates.Select(e => new Func <Task>(() => UpdateOrRemoveBadge(context, e, preRel)))); return(TaskUtils.RunAsync(tasks)); }