public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities) { ITemplateEngineHost host = CreateHost(packInfo); using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true); PackCheckResult checkResult; try { if (TryInstallPackage(packInfo.Path, environmentSettings, out IReadOnlyList <ITemplateInfo> installedTemplates)) { IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList(); checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates); ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environmentSettings); } else { IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>(); checkResult = new PackCheckResult(packInfo, foundTemplates); } } catch { IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>(); checkResult = new PackCheckResult(packInfo, foundTemplates); } return(checkResult); }
public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities, bool persistHive = false) { ITemplateEngineHost host = CreateHost(packInfo); EngineEnvironmentSettings environment = new EngineEnvironmentSettings(host, x => new SettingsLoader(x)); PackCheckResult checkResult; try { if (TryInstallPackage(packInfo.Path, environment, out IReadOnlyList <ITemplateInfo> installedTemplates)) { IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList(); checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates); ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environment); } else { IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>(); checkResult = new PackCheckResult(packInfo, foundTemplates); } } catch { IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>(); checkResult = new PackCheckResult(packInfo, foundTemplates); } if (!persistHive) { TryCleanup(environment); } return(checkResult); }
private IEnumerable <TemplateSearchData> TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers) { ITemplateEngineHost host = TemplateEngineHostHelper.CreateHost(HostIdentifierBase + packInfo.Name); using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true); Scanner scanner = new Scanner(environmentSettings); try { using var scanResult = scanner.Scan(packInfo.Path, scanForComponents: false); if (scanResult.Templates.Any()) { foreach (IAdditionalDataProducer dataProducer in additionalDataProducers) { #pragma warning disable CS0612 // Type or member is obsolete dataProducer.CreateDataForTemplatePack(packInfo, scanResult.Templates, environmentSettings); #pragma warning restore CS0612 // Type or member is obsolete } return(scanResult.Templates.Select(t => new TemplateSearchData(t, t.ProduceAdditionalData(additionalDataProducers, environmentSettings)))); } return(Array.Empty <TemplateSearchData>()); } catch (TaskCanceledException) { throw; } catch (Exception ex) { Console.WriteLine("Failed to read package {0}::{1}, details: {2}. The package will be skipped.", packInfo.Name, packInfo.Version, ex); return(Array.Empty <TemplateSearchData>()); } }
private void ProcessNonTemplatePackage( IDownloadedPackInfo sourceInfo, string filterReason, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, string?oldTemplatePackageVersion, string?oldNonTemplatePackageVersion) { Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is skipped, {filterReason}"); filteredPackages[sourceInfo.Name] = new FilteredPackageInfo(sourceInfo, filterReason); if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion)) { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.NewNonTemplatePacksCount++; } else { scanningStats.BecameNonTemplatePacksCount++; } } else { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.UpdatedNonTemplatePacksCount++; } } }
private static ITemplateEngineHost CreateHost(IDownloadedPackInfo packInfo) { string hostIdentifier = HostIdentifierBase + packInfo.Id; ITemplateEngineHost host = TemplateEngineHostHelper.CreateHost(hostIdentifier); return(host); }
internal PreFilterResultList FilterPack(IDownloadedPackInfo packInfo) { List <PreFilterResult> resultList = new List <PreFilterResult>(); foreach (Func <IDownloadedPackInfo, PreFilterResult> filter in _preFilters) { PreFilterResult result = filter(packInfo); resultList.Add(result); } return(new PreFilterResultList(resultList)); }
private void ProcessTemplatePackage( IDownloadedPackInfo sourceInfo, Dictionary <string, TemplatePackageSearchData> newCache, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, string?oldTemplatePackageVersion, string?oldNonTemplatePackageVersion ) { IEnumerable <TemplateSearchData> foundTemplates = TryGetTemplatesInPack(sourceInfo, _additionalDataProducers); Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is processed"); if (foundTemplates.Any()) { Verbose.WriteLine("Found templates:"); foreach (TemplateSearchData template in foundTemplates) { string shortNames = string.Join(",", template.ShortNameList); Verbose.WriteLine($" - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}"); } newCache[sourceInfo.Name] = new TemplatePackageSearchData(sourceInfo, foundTemplates, sourceInfo.ProduceAdditionalData(_additionalDataProducers)); if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion)) { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.NewTemplatePacks.Add(newCache[sourceInfo.Name]); } else { scanningStats.UpdatedTemplatePacks.Add((newCache[sourceInfo.Name], oldTemplatePackageVersion)); } } else { if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion)) { scanningStats.BecameTemplatePacks.Add((newCache[sourceInfo.Name])); } } } else { ProcessNonTemplatePackage( sourceInfo, "Failed to scan the package for the templates, the package may contain invalid templates.", filteredPackages, scanningStats, oldTemplatePackageVersion, oldNonTemplatePackageVersion); } }
public void CreateDataForTemplatePack(IDownloadedPackInfo packInfo, IReadOnlyList <ITemplateInfo> templateList, IEngineEnvironmentSettings environment) { IHostSpecificDataLoader hostDataLoader = new HostSpecificDataLoader(environment.SettingsLoader); foreach (ITemplateInfo template in templateList) { HostSpecificTemplateData hostData = hostDataLoader.ReadHostSpecificTemplateData(template); // store the host data if it has any info that could affect searching for this template. if (hostData.IsHidden || hostData.SymbolInfo.Count > 0) { _hostDataForPackByTemplate[template.Identity] = hostData; } } }
private bool CheckIfPackageIsFiltered( IDownloadedPackInfo sourceInfo, Dictionary <string, FilteredPackageInfo> filteredPackages, ScanningStats scanningStats, string?oldTemplatePackageVersion, string?oldNonTemplatePackageVersion) { PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(sourceInfo); if (preFilterResult.ShouldBeFiltered) { ProcessNonTemplatePackage(sourceInfo, preFilterResult.Reason, filteredPackages, scanningStats, oldTemplatePackageVersion, oldNonTemplatePackageVersion); } return(preFilterResult.ShouldBeFiltered); }
public async Task <PackSourceCheckResult> CheckPackagesAsync() { List <PackCheckResult> checkResultList = new List <PackCheckResult>(); HashSet <IPackInfo> alreadySeenPacks = new HashSet <IPackInfo>(); HashSet <string> alreadySeenTemplateIdentities = new HashSet <string>(); foreach (IPackProvider packProvider in _packProviders) { Console.WriteLine($"Processing pack provider {packProvider.Name}:"); int count = 0; Console.WriteLine($"{await packProvider.GetPackageCountAsync().ConfigureAwait(false)} packs discovered, starting processing"); await foreach (IPackInfo sourceInfo in packProvider.GetCandidatePacksAsync().ConfigureAwait(false)) { if (alreadySeenPacks.Contains(sourceInfo)) { Verbose.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is already processed."); continue; } alreadySeenPacks.Add(sourceInfo); IDownloadedPackInfo packInfo = await packProvider.DownloadPackageAsync(sourceInfo).ConfigureAwait(false); if (packInfo == null) { Console.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is not processed."); continue; } PackCheckResult preFilterResult = PrefilterPackInfo(packInfo); if (preFilterResult.PreFilterResults.ShouldBeFiltered) { Verbose.WriteLine($"{packInfo.Id}::{packInfo.Version} is skipped, {preFilterResult.PreFilterResults.Reason}"); checkResultList.Add(preFilterResult); } else { PackCheckResult packCheckResult = _packChecker.TryGetTemplatesInPack(packInfo, _additionalDataProducers, alreadySeenTemplateIdentities); checkResultList.Add(packCheckResult); Verbose.WriteLine($"{packCheckResult.PackInfo.Id}::{packCheckResult.PackInfo.Version} is processed"); if (packCheckResult.FoundTemplates.Any()) { Verbose.WriteLine("Found templates:"); foreach (ITemplateInfo template in packCheckResult.FoundTemplates) { string shortNames = string.Join(",", template.ShortNameList); Verbose.WriteLine($" - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}"); } } else { Verbose.WriteLine("No templates were found"); } // Record the found identities - to skip these templates when checking additional packs. // Some template identities are the same in multiple packs on nuget. // For this scraper, first in wins. alreadySeenTemplateIdentities.UnionWith(packCheckResult.FoundTemplates.Select(t => t.Identity)); } ++count; if ((count % 10) == 0) { Console.WriteLine($"{count} packs are processed"); } } Console.WriteLine($"All packs from pack provider {packProvider.Name} are processed"); } Console.WriteLine("All packs are processed"); if (!_saveCandidatePacks) { Console.WriteLine("Removing downloaded packs"); foreach (IPackProvider provider in _packProviders) { provider.DeleteDownloadedPacks(); } Console.WriteLine("Downloaded packs were removed"); } return(new PackSourceCheckResult(checkResultList, _additionalDataProducers)); }
private PackCheckResult PrefilterPackInfo(IDownloadedPackInfo packInfo) { PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(packInfo); return(new PackCheckResult(packInfo, preFilterResult)); }
internal PackCheckResult(IDownloadedPackInfo packInfo, PreFilterResultList preFilterResults) { PackInfo = packInfo; PreFilterResults = preFilterResults; FoundTemplates = new List <ITemplateInfo>(); }
internal PackCheckResult(IDownloadedPackInfo packInfo, IReadOnlyList <ITemplateInfo> foundTemplates) { PackInfo = packInfo; PreFilterResults = new PreFilterResultList(); FoundTemplates = foundTemplates; }