private void ProcessNonTemplatePackage( IDownloadedPackInfo sourceInfo, string filterReason, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, string?oldTemplatePackageVersion, string?oldNonTemplatePackageVersion) { Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is skipped, {filterReason}"); filteredPackages[sourceInfo.Name] = new FilteredPackageInfo(sourceInfo, filterReason); if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion)) { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.NewNonTemplatePacksCount++; } else { scanningStats.BecameNonTemplatePacksCount++; } } else { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.UpdatedNonTemplatePacksCount++; } } }
private void ProcessTemplatePackage( IDownloadedPackInfo sourceInfo, Dictionary <string, TemplatePackageSearchData> newCache, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, string?oldTemplatePackageVersion, string?oldNonTemplatePackageVersion ) { IEnumerable <TemplateSearchData> foundTemplates = TryGetTemplatesInPack(sourceInfo, _additionalDataProducers); Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is processed"); if (foundTemplates.Any()) { Verbose.WriteLine("Found templates:"); foreach (TemplateSearchData template in foundTemplates) { string shortNames = string.Join(",", template.ShortNameList); Verbose.WriteLine($" - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}"); } newCache[sourceInfo.Name] = new TemplatePackageSearchData(sourceInfo, foundTemplates, sourceInfo.ProduceAdditionalData(_additionalDataProducers)); if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion)) { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.NewTemplatePacks.Add(newCache[sourceInfo.Name]); } else { scanningStats.UpdatedTemplatePacks.Add((newCache[sourceInfo.Name], oldTemplatePackageVersion)); } } else { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.BecameTemplatePacks.Add((newCache[sourceInfo.Name])); } } } else { ProcessNonTemplatePackage( sourceInfo, "Failed to scan the package for the templates, the package may contain invalid templates.", filteredPackages, scanningStats, oldTemplatePackageVersion, oldNonTemplatePackageVersion); } }
private bool CheckIfPackageIsFiltered( IDownloadedPackInfo sourceInfo, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, string?oldTemplatePackageVersion, string?oldNonTemplatePackageVersion) { PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(sourceInfo); if (preFilterResult.ShouldBeFiltered) { ProcessNonTemplatePackage(sourceInfo, preFilterResult.Reason, filteredPackages, scanningStats, oldTemplatePackageVersion, oldNonTemplatePackageVersion); } return(preFilterResult.ShouldBeFiltered); }
private async Task ProcessTemplatePackagesUsingSearchAsync( Dictionary <string, TemplatePackageSearchData> newCache, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, CancellationToken cancellationToken) { foreach (IPackProvider packProvider in _packProviders) { int count = -1; Console.WriteLine($"Processing pack provider {packProvider.Name}:"); Console.WriteLine($"{await packProvider.GetPackageCountAsync(cancellationToken).ConfigureAwait(false)} packs discovered, starting processing"); await foreach (ITemplatePackageInfo sourceInfo in packProvider.GetCandidatePacksAsync(cancellationToken).ConfigureAwait(false)) { count = ProcessCount(count); if (newCache.ContainsKey(sourceInfo.Name) || filteredPackages.ContainsKey(sourceInfo.Name)) { Verbose.WriteLine($"Package {sourceInfo.Name}::{sourceInfo.Version} is already processed."); continue; } string?oldTemplateVersion = null; string?oldNonTemplateVersion = null; if (_diffEnabled) { if (CheckIfVersionIsKnownTemplatePackage(sourceInfo, newCache, scanningStats, out oldTemplateVersion)) { continue; } if (CheckIfVersionIsKnownNonTemplatePackage(sourceInfo, filteredPackages, scanningStats, out oldNonTemplateVersion)) { continue; } } IDownloadedPackInfo?packInfo = await packProvider.DownloadPackageAsync(sourceInfo, cancellationToken).ConfigureAwait(false); if (CheckIfPackageIsFiltered(packInfo, filteredPackages, scanningStats, oldTemplateVersion, oldNonTemplateVersion)) { continue; } ProcessTemplatePackage(packInfo, newCache, filteredPackages, scanningStats, oldTemplateVersion, oldNonTemplateVersion); } ProcessCount(count); Console.WriteLine($"All packs from pack provider {packProvider.Name} are processed"); } Console.WriteLine("All packs are processed"); }
private void DisplayScanningStats(ScanningStats scanningStats) { Console.WriteLine("Run summary:"); Console.WriteLine("Template packages:"); Console.WriteLine($" new: {scanningStats.NewTemplatePacks.Count}"); foreach (var package in scanningStats.NewTemplatePacks) { Console.WriteLine($" {package.Name}::{package.Version}"); } Console.WriteLine($" updated: {scanningStats.UpdatedTemplatePacks.Count}"); foreach (var package in scanningStats.UpdatedTemplatePacks) { Console.WriteLine($" {package.Package.Name}, {package.OldVersion} --> {package.Package.Version}"); } Console.WriteLine($" removed: {scanningStats.RemovedTemplatePacks.Count}"); foreach (var package in scanningStats.RemovedTemplatePacks) { Console.WriteLine($" {package.Name}::{package.Version}"); } if (scanningStats.BecameTemplatePacks.Any()) { Console.WriteLine($" became template packages: {scanningStats.BecameTemplatePacks.Count}"); foreach (var package in scanningStats.BecameTemplatePacks) { Console.WriteLine($" {package.Name}::{package.Version}"); } } Console.WriteLine($" not changed: {scanningStats.SameTemplatePacksCount}"); Console.WriteLine($"Non template packages:"); Console.WriteLine($" new: {scanningStats.NewNonTemplatePacksCount}"); Console.WriteLine($" updated: {scanningStats.UpdatedNonTemplatePacksCount}"); Console.WriteLine($" removed: {scanningStats.RemovedNonTemplatePacksCount}"); Console.WriteLine($" not changed: {scanningStats.SameNonTemplatePacksCount}"); Console.WriteLine($" unavailable: {scanningStats.UnavailableNonTemplatePacksCount}"); if (scanningStats.BecameNonTemplatePacksCount > 0) { Console.WriteLine($" became non template package: {scanningStats.BecameNonTemplatePacksCount}"); } }
internal async Task <PackSourceCheckResult> CheckPackagesAsync(CancellationToken token) { Dictionary <string, TemplatePackageSearchData> newCache = new Dictionary <string, TemplatePackageSearchData>(StringComparer.OrdinalIgnoreCase); Dictionary <string, FilteredPackageInfo> filteredPackages = new Dictionary <string, FilteredPackageInfo>(StringComparer.OrdinalIgnoreCase); ScanningStats scanningStats = new ScanningStats(); try { await ProcessTemplatePackagesUsingSearchAsync(newCache, filteredPackages, scanningStats, token).ConfigureAwait(false); if (_diffEnabled) { await CheckRemovedPackagesAsync(newCache, filteredPackages, scanningStats, token).ConfigureAwait(false); } DisplayScanningStats(scanningStats); return(new PackSourceCheckResult(new TemplateSearchCache(newCache.Values.ToList()), filteredPackages.Values.ToList(), _additionalDataProducers)); } finally { if (!_saveCandidatePacks) { try { Console.WriteLine("Removing downloaded packs"); foreach (IPackProvider provider in _packProviders) { await provider.DeleteDownloadedPacksAsync().ConfigureAwait(false); } Console.WriteLine("Downloaded packs were removed"); } catch (Exception ex) { Console.WriteLine($"Failed to remove downloaded packs, details: {ex}"); } } } }
private bool CheckIfVersionIsKnownNonTemplatePackage( ITemplatePackageInfo sourceInfo, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, out string?currentVersion) { currentVersion = null; if (_knownFilteredPackages.TryGetValue(sourceInfo.Name, out FilteredPackageInfo? info)) { if (sourceInfo.Version == info.Version) { Verbose.WriteLine($"Package {sourceInfo.Name}::{sourceInfo.Version} has not changed since last scan, skipping as it was filtered last time."); filteredPackages[sourceInfo.Name] = info; scanningStats.SameNonTemplatePacksCount++; return(true); } else { currentVersion = info.Version; } } return(false); }
private bool CheckIfVersionIsKnownTemplatePackage( ITemplatePackageInfo sourceInfo, Dictionary <string, TemplatePackageSearchData> newCache, ScanningStats scanningStats, out string?currentVersion) { currentVersion = null; if (_existingCache.TryGetValue(sourceInfo.Name, out TemplatePackageSearchData? existingInfo)) { if (sourceInfo.Version == existingInfo.Version) { Verbose.WriteLine($"Package {sourceInfo.Name}::{sourceInfo.Version} has not changed since last scan, updating metadata only."); newCache[sourceInfo.Name] = new TemplatePackageSearchData(sourceInfo, existingInfo.Templates, sourceInfo.ProduceAdditionalData(_additionalDataProducers)); scanningStats.SameTemplatePacksCount++; return(true); } else { currentVersion = existingInfo.Version; } } return(false); }
private async Task CheckRemovedPackagesAsync( Dictionary <string, TemplatePackageSearchData> newCache, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, CancellationToken cancellationToken) { IReadOnlyList <TemplatePackageSearchData> removedTemplatePacks; IReadOnlyList <FilteredPackageInfo> removedNonTemplatePacks; (removedTemplatePacks, removedNonTemplatePacks) = EvaluateRemovedPackages(newCache, filteredPackages); if (!removedTemplatePacks.Any() && removedNonTemplatePacks.Any()) { return; } var provider = _packProviders.FirstOrDefault(); if (provider == null) { Console.WriteLine($"Providers do not support checking information about package, removed packages won't be verified and added to cache."); return; } Console.WriteLine("Checking template packages via API: "); foreach (var package in removedTemplatePacks) { var packageInfo = await provider.GetPackageInfoAsync(package.Name, cancellationToken).ConfigureAwait(false); if (packageInfo == default) { Console.WriteLine($"Package {package.Name} cannot be verified."); throw new Exception($"Package {package.Name} is missing and cannot be verified."); } if (packageInfo.Removed) { Console.WriteLine($"Package {package.Name} was unlisted."); scanningStats.RemovedTemplatePacks.Add(package); } else { Console.WriteLine($"Package {package.Name} was verified, adding to cache."); if (CheckIfVersionIsKnownTemplatePackage(package, newCache, scanningStats, out string?oldVersion)) { continue; } IDownloadedPackInfo?packInfo = await provider.DownloadPackageAsync(package, cancellationToken).ConfigureAwait(false); if (CheckIfPackageIsFiltered(packInfo, filteredPackages, scanningStats, oldVersion, null)) { continue; } ProcessTemplatePackage(packInfo, newCache, filteredPackages, scanningStats, oldVersion, null); } } Verbose.WriteLine("Checking non template packages via API: "); foreach (var package in removedNonTemplatePacks) { var packageInfo = await provider.GetPackageInfoAsync(package.Name, cancellationToken).ConfigureAwait(false); if (packageInfo == default) { Verbose.WriteLine($"Package {package.Name} cannot be verified."); //it's ok to continue if the package was known non-package - ignore error scanningStats.UnavailableNonTemplatePacksCount++; continue; } if (packageInfo.Removed) { Verbose.WriteLine($"Package {package.Name} was unlisted."); scanningStats.RemovedNonTemplatePacksCount++; } else { Verbose.WriteLine($"Package {package.Name} was verified."); if (CheckIfVersionIsKnownNonTemplatePackage(package, filteredPackages, scanningStats, out string?oldNonTemplateVersion)) { continue; } IDownloadedPackInfo?packInfo = await provider.DownloadPackageAsync(package, cancellationToken).ConfigureAwait(false); if (packInfo == null) { Console.WriteLine($"[Error] Package {package.Name}::{package.Version} is not processed."); continue; } if (CheckIfPackageIsFiltered(packInfo, filteredPackages, scanningStats, null, oldNonTemplateVersion)) { continue; } ProcessTemplatePackage(packInfo, newCache, filteredPackages, scanningStats, null, oldNonTemplateVersion); } } }