public static async Task <bool> RunAsync(LocalSettings settings, ISleetFileSystem source, string outputPath, bool ignoreErrors, bool noLock, bool skipExisting, ILogger log) { var token = CancellationToken.None; ISleetFileSystemLock feedLock = null; var success = true; var perfTracker = source.LocalCache.PerfTracker; using (var timer = PerfEntryWrapper.CreateSummaryTimer("Total execution time: {0}", perfTracker)) { // Check if already initialized try { if (!noLock) { // Lock feedLock = await SourceUtility.VerifyInitAndLock(settings, source, "Download", log, token); // Validate source await UpgradeUtility.EnsureCompatibility(source, log, token); } success = await DownloadPackages(settings, source, outputPath, ignoreErrors, log, token); } finally { feedLock?.Dispose(); } } // Write out perf summary await perfTracker.LogSummary(log); return(success); }
public static async Task <bool> RunAsync(LocalSettings settings, ISleetFileSystem source, List <string> inputs, bool force, bool skipExisting, ILogger log) { var token = CancellationToken.None; var now = DateTimeOffset.UtcNow; var success = false; var perfTracker = source.LocalCache.PerfTracker; await log.LogAsync(LogLevel.Minimal, $"Reading feed {source.BaseURI.AbsoluteUri}"); using (var timer = PerfEntryWrapper.CreateSummaryTimer("Total execution time: {0}", perfTracker)) { // Partition package inputs to avoid reading 100K nuspecs at the same time. var packagePaths = GetPackagePaths(inputs); var inputBatches = packagePaths.Partition(DefaultBatchSize); ISleetFileSystemLock feedLock = null; try { for (var i = 0; i < inputBatches.Count; i++) { var inputBatch = inputBatches[i]; if (inputBatches.Count > 1) { await log.LogAsync(LogLevel.Minimal, $"Pushing {inputBatch.Count} packages. Batch: {i+1} / {inputBatches.Count}"); } // Read packages before locking the feed the first time. var packages = new List <PackageInput>(await GetPackageInputs(inputBatch, now, perfTracker, log)); if (feedLock == null) { string lockMessage = null; if (packages.Count > 0) { lockMessage = $"Push of {packages[0].Identity.ToString()}"; } // Check if already initialized feedLock = await SourceUtility.VerifyInitAndLock(settings, source, lockMessage, log, token); // Validate source await SourceUtility.ValidateFeedForClient(source, log, token); } // Push success = await PushPackages(settings, source, packages, force, skipExisting, log, token); } } finally { // Unlock the feed feedLock?.Dispose(); } } // Write out perf summary await perfTracker.LogSummary(log); return(success); }
public static async Task <ISleetFileSystemLock> VerifyInitAndLock(LocalSettings settings, ISleetFileSystem fileSystem, string lockMessage, ILogger log, CancellationToken token) { ISleetFileSystemLock feedLock = null; ValidateFileSystem(fileSystem); try { // Validate source var exists = await fileSystem.Validate(log, token); if (!exists) { throw new InvalidOperationException($"Unable to use feed."); } var timer = Stopwatch.StartNew(); feedLock = fileSystem.CreateLock(log); // Use the message from settings as an override if it exists. var lockInfoMessage = string.IsNullOrEmpty(settings.FeedLockMessage) ? lockMessage : settings.FeedLockMessage; // Get lock var isLocked = await feedLock.GetLock(settings.FeedLockTimeout, lockInfoMessage, token); if (!isLocked) { throw new InvalidOperationException("Unable to obtain a lock on the feed."); } // Log perf timer.Stop(); fileSystem.LocalCache.PerfTracker.Add(new PerfSummaryEntry(timer.Elapsed, "Obtained feed lock in {0}", TimeSpan.FromSeconds(30))); var indexPath = fileSystem.Get("index.json"); if (!await indexPath.ExistsWithFetch(log, token)) { throw new InvalidOperationException($"{fileSystem.BaseURI} is missing sleet files. Use 'sleet.exe init' to create a new feed."); } } catch { if (feedLock != null) { feedLock.Release(); } throw; } return(feedLock); }
/// <summary> /// Ensure a feed is initialized. If the feed is not initialized it can be automatically created and initialized. /// Feeds that are not in a vaild state will fail, these must be manually fixed to avoid losing data. /// </summary> /// <param name="lockMessage">Optional message to display when the feed is locked.</param> /// <param name="autoCreateBucket">Automatically create the folder/container/bucket without files.</param> /// <param name="autoInit">Automatically initialize the files in the feed.</param> public static async Task <ISleetFileSystemLock> InitAndLock(LocalSettings settings, ISleetFileSystem fileSystem, string lockMessage, bool autoCreateBucket, bool autoInit, ILogger log, CancellationToken token) { ISleetFileSystemLock feedLock = null; // Validate URI path ValidateFileSystem(fileSystem); // Create the bucket if allowed or throw. await EnsureBucketOrThrow(fileSystem, autoCreateBucket, log, token); try { var timer = Stopwatch.StartNew(); feedLock = fileSystem.CreateLock(log); // Use the message from settings as an override if it exists. var lockInfoMessage = string.IsNullOrEmpty(settings.FeedLockMessage) ? lockMessage : settings.FeedLockMessage; // Get lock var isLocked = await feedLock.GetLock(settings.FeedLockTimeout, lockInfoMessage, token); if (!isLocked) { throw new InvalidOperationException("Unable to obtain a lock on the feed."); } // Log perf timer.Stop(); fileSystem.LocalCache.PerfTracker.Add(new PerfSummaryEntry(timer.Elapsed, "Obtained feed lock in {0}", TimeSpan.FromSeconds(30))); // Reset the file system to avoid using files retrieved before the lock, this would be unsafe fileSystem.Reset(); await EnsureFeedIndexOrThrow(settings, fileSystem, autoInit, log, token); } catch { if (feedLock != null) { feedLock.Release(); } throw; } return(feedLock); }
public static async Task <ISleetFileSystemLock> VerifyInitAndLock(LocalSettings settings, ISleetFileSystem fileSystem, ILogger log, CancellationToken token) { ISleetFileSystemLock feedLock = null; ValidateFileSystem(fileSystem); try { // Validate source var exists = await fileSystem.Validate(log, token); if (!exists) { throw new InvalidOperationException($"Unable to use feed."); } feedLock = fileSystem.CreateLock(log); var isLocked = await feedLock.GetLock(settings.FeedLockTimeout, token); if (!isLocked) { throw new InvalidOperationException("Unable to obtain a lock on the feed."); } var indexPath = fileSystem.Get("index.json"); if (!await indexPath.ExistsWithFetch(log, token)) { throw new InvalidOperationException($"{fileSystem.BaseURI} is missing sleet files. Use 'sleet.exe init' to create a new feed."); } } catch { if (feedLock != null) { feedLock.Release(); } throw; } return(feedLock); }