public void Dispose() { if (reader != null) { reader.Dispose(); } reader.ClearCache(); }
private static void Run(CommandLineApplication cmd, HttpSource httpSource, ILogger consoleLog) { cmd.Description = "Mirror nupkgs to a folder."; var output = cmd.Option("-o|--output", "Output directory for nupkgs.", CommandOptionType.SingleValue); var folderFormat = cmd.Option("--folder-format", "Output folder format. Defaults to v3. Options: (v2|v3)", CommandOptionType.SingleValue); var ignoreErrors = cmd.Option("--ignore-errors", "Continue on errors.", CommandOptionType.NoValue); var delay = cmd.Option("--delay", "Avoid downloading the very latest packages on the feed to avoid errors. This value is in minutes. Default: 10", CommandOptionType.SingleValue); var maxThreadsOption = cmd.Option("--max-threads", "Maximum number of concurrent downloads. Default: 8", CommandOptionType.SingleValue); var verbose = cmd.Option("--verbose", "Output additional network information.", CommandOptionType.NoValue); var includeIdOption = cmd.Option("-i|--include-id", "Include only these package ids or wildcards. May be provided multiple times.", CommandOptionType.MultipleValue); var excludeIdOption = cmd.Option("-e|--exclude-id", "Exclude these package ids or wildcards. May be provided multiple times.", CommandOptionType.MultipleValue); var additionalOutput = cmd.Option("--additional-output", "Additional output directory for nupkgs. The output path with the most free space will be used.", CommandOptionType.MultipleValue); var onlyLatestVersion = cmd.Option("--latest-only", "Include only the latest version of that package in the result", CommandOptionType.NoValue); var argRoot = cmd.Argument( "[root]", "V3 feed index.json URI", multipleValues: false); cmd.HelpOption(Constants.HelpOption); cmd.OnExecute(async() => { var timer = new Stopwatch(); timer.Start(); if (string.IsNullOrEmpty(argRoot.Value)) { throw new ArgumentException("Provide the full http url to a v3 nuget feed."); } var index = new Uri(argRoot.Value); if (!index.AbsolutePath.EndsWith("/index.json", StringComparison.OrdinalIgnoreCase)) { throw new ArgumentException($"Invalid feed url: '{argRoot.Value}'. Provide the full http url to a v3 nuget feed. For nuget.org use: https://api.nuget.org/v3/index.json"); } // Create root var outputPath = Directory.GetCurrentDirectory(); if (output.HasValue()) { outputPath = output.Value(); } var tmpCachePath = Path.Combine(outputPath, ".tmp"); var storagePaths = new HashSet <DirectoryInfo>() { new DirectoryInfo(outputPath) }; if (additionalOutput.Values?.Any() == true) { storagePaths.UnionWith(additionalOutput.Values.Select(e => new DirectoryInfo(e))); } // Create all output folders foreach (var path in storagePaths) { path.Create(); } var delayTime = TimeSpan.FromMinutes(10); if (delay.HasValue()) { if (int.TryParse(delay.Value(), out int x)) { var delayMinutes = Math.Max(0, x); delayTime = TimeSpan.FromMinutes(delayMinutes); } else { throw new ArgumentException("Invalid --delay value. This must be an integer."); } } var maxThreads = 8; if (maxThreadsOption.HasValue()) { if (int.TryParse(maxThreadsOption.Value(), out int x)) { maxThreads = Math.Max(1, x); } else { throw new ArgumentException("Invalid --max-threads value. This must be an integer."); } } var batchSize = 64; var outputRoot = new DirectoryInfo(outputPath); var outputFilesInfo = new FileInfo(Path.Combine(outputRoot.FullName, "updatedFiles.txt")); FileUtility.Delete(outputFilesInfo.FullName); var useV3Format = true; if (folderFormat.HasValue()) { switch (folderFormat.Value().ToLowerInvariant()) { case "v2": useV3Format = false; break; case "v3": useV3Format = true; break; default: throw new ArgumentException($"Invalid {folderFormat.LongName} value: '{folderFormat.Value()}'."); } } var start = MirrorUtility.LoadCursor(outputRoot); var end = DateTimeOffset.UtcNow.Subtract(delayTime); var token = CancellationToken.None; var mode = DownloadMode.OverwriteIfNewer; var errorLogPath = Path.Combine(outputPath, "lastRunErrors.txt"); FileUtility.Delete(errorLogPath); // Loggers // source -> deep -> file -> Console var log = new FileLogger(consoleLog, LogLevel.Error, errorLogPath); var deepLogger = new FilterLogger(log, LogLevel.Error); // Init log.LogInformation($"Mirroring {index.AbsoluteUri} -> {outputPath}"); var formatName = useV3Format ? "{id}/{version}/{id}.{version}.nupkg" : "{id}/{id}.{version}.nupkg"; log.LogInformation($"Folder format:\t{formatName}"); log.LogInformation($"Cursor:\t\t{Path.Combine(outputPath, "cursor.json")}"); log.LogInformation($"Change log:\t{outputFilesInfo.FullName}"); log.LogInformation($"Error log:\t{errorLogPath}"); log.LogInformation("Range start:\t" + start.ToString("o")); log.LogInformation("Range end:\t" + end.ToString("o")); log.LogInformation($"Batch size:\t{batchSize}"); log.LogInformation($"Threads:\t{maxThreads}"); // CatalogReader using (var cacheContext = new SourceCacheContext()) { cacheContext.SetTempRoot(tmpCachePath); using (var catalogReader = new CatalogReader(index, httpSource, cacheContext, TimeSpan.Zero, deepLogger)) { // Clear old cache files catalogReader.ClearCache(); // Find the most recent entry for each package in the range // Order by oldest first IEnumerable <CatalogEntry> entryQuery = (await catalogReader .GetFlattenedEntriesAsync(start, end, token)); // Remove all but includes if given if (includeIdOption.HasValue()) { var regex = includeIdOption.Values.Select(s => MirrorUtility.WildcardToRegex(s)).ToArray(); entryQuery = entryQuery.Where(e => regex.Any(r => r.IsMatch(e.Id))); } // Remove all excludes if given if (excludeIdOption.HasValue()) { var regex = excludeIdOption.Values.Select(s => MirrorUtility.WildcardToRegex(s)).ToArray(); entryQuery = entryQuery.Where(e => regex.All(r => !r.IsMatch(e.Id))); } if (onlyLatestVersion.HasValue()) { entryQuery = entryQuery.GroupBy(x => x.Id).Select(y => y.OrderByDescending(z => z.Version).First()); } var toProcess = new Queue <CatalogEntry>(entryQuery.OrderBy(e => e.CommitTimeStamp)); log.LogInformation($"Catalog entries found: {toProcess.Count}"); var done = new List <CatalogEntry>(batchSize); var complete = 0; var total = toProcess.Count; var totalDownloads = 0; // Download files var tasks = new List <Task <NupkgResult> >(maxThreads); var batchTimersMax = 20; var batchTimers = new Queue <Tuple <Stopwatch, int> >(batchTimersMax); // Download with throttling while (toProcess.Count > 0) { // Create batches var batch = new Queue <CatalogEntry>(batchSize); var files = new List <string>(); var batchTimer = new Stopwatch(); batchTimer.Start(); while (toProcess.Count > 0 && batch.Count < batchSize) { batch.Enqueue(toProcess.Dequeue()); } while (batch.Count > 0) { if (tasks.Count == maxThreads) { await CompleteTaskAsync(files, tasks, done); } var entry = batch.Dequeue(); Func <CatalogEntry, Task <FileInfo> > getNupkg = null; if (useV3Format) { getNupkg = (e) => DownloadNupkgV3Async(e, storagePaths, mode, log, deepLogger, token); } else { getNupkg = (e) => DownloadNupkgV2Async(e, storagePaths, mode, log, token); } // Queue download task tasks.Add(Task.Run(async() => await RunWithRetryAsync(entry, ignoreErrors.HasValue(), getNupkg, log, token))); } // Wait for all batch downloads while (tasks.Count > 0) { await CompleteTaskAsync(files, tasks, done); } files = files.Where(e => e != null).ToList(); // Write out new files using (var newFileWriter = new StreamWriter(new FileStream(outputFilesInfo.FullName, FileMode.Append, FileAccess.Write))) { foreach (var file in files) { newFileWriter.WriteLine(file); } } complete += done.Count; totalDownloads += files.Count; batchTimer.Stop(); batchTimers.Enqueue(new Tuple <Stopwatch, int>(batchTimer, done.Count)); while (batchTimers.Count > batchTimersMax) { batchTimers.Dequeue(); } // Update cursor var newestCommit = GetNewestCommit(done, toProcess); if (newestCommit != null) { log.LogMinimal($"================[batch complete]================"); log.LogMinimal($"Processed:\t\t{complete} / {total}"); log.LogMinimal($"Batch downloads:\t{files.Count}"); log.LogMinimal($"Batch time:\t\t{batchTimer.Elapsed}"); log.LogMinimal($"Updating cursor.json:\t{newestCommit.Value.ToString("o")}"); var rate = batchTimers.Sum(e => e.Item1.Elapsed.TotalSeconds) / Math.Max(1, batchTimers.Sum(e => e.Item2)); var timeLeft = TimeSpan.FromSeconds(rate * (total - complete)); var timeLeftString = string.Empty; if (timeLeft.TotalHours >= 1) { timeLeftString = $"{(int)timeLeft.TotalHours} hours"; } else if (timeLeft.TotalMinutes >= 1) { timeLeftString = $"{(int)timeLeft.TotalMinutes} minutes"; } else { timeLeftString = $"{(int)timeLeft.TotalSeconds} seconds"; } log.LogMinimal($"Estimated time left:\t{timeLeftString}"); log.LogMinimal($"================================================"); MirrorUtility.SaveCursor(outputRoot, newestCommit.Value); } done.Clear(); // Free up space catalogReader.ClearCache(); } // Set cursor to end time MirrorUtility.SaveCursor(outputRoot, end); timer.Stop(); var plural = totalDownloads == 1 ? "" : "s"; log.LogMinimal($"Downloaded {totalDownloads} nupkg{plural} in {timer.Elapsed.ToString()}."); } } return(0); }); }