public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities)
{
ITemplateEngineHost host = CreateHost(packInfo);
using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true);
PackCheckResult checkResult;
try
{
if (TryInstallPackage(packInfo.Path, environmentSettings, out IReadOnlyList <ITemplateInfo> installedTemplates))
{
IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList();
checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates);
ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environmentSettings);
}
else
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
}
catch
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
return(checkResult);
}
public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities, bool persistHive = false)
{
ITemplateEngineHost host = CreateHost(packInfo);
EngineEnvironmentSettings environment = new EngineEnvironmentSettings(host, x => new SettingsLoader(x));
PackCheckResult checkResult;
try
{
if (TryInstallPackage(packInfo.Path, environment, out IReadOnlyList <ITemplateInfo> installedTemplates))
{
IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList();
checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates);
ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environment);
}
else
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
}
catch
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
if (!persistHive)
{
TryCleanup(environment);
}
return(checkResult);
}
private IEnumerable <TemplateSearchData> TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers)
{
ITemplateEngineHost host = TemplateEngineHostHelper.CreateHost(HostIdentifierBase + packInfo.Name);
using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true);
Scanner scanner = new Scanner(environmentSettings);
try
{
using var scanResult = scanner.Scan(packInfo.Path, scanForComponents: false);
if (scanResult.Templates.Any())
{
foreach (IAdditionalDataProducer dataProducer in additionalDataProducers)
{
#pragma warning disable CS0612 // Type or member is obsolete
dataProducer.CreateDataForTemplatePack(packInfo, scanResult.Templates, environmentSettings);
#pragma warning restore CS0612 // Type or member is obsolete
}
return(scanResult.Templates.Select(t => new TemplateSearchData(t, t.ProduceAdditionalData(additionalDataProducers, environmentSettings))));
}
return(Array.Empty <TemplateSearchData>());
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
Console.WriteLine("Failed to read package {0}::{1}, details: {2}. The package will be skipped.", packInfo.Name, packInfo.Version, ex);
return(Array.Empty <TemplateSearchData>());
}
}
private void ProcessNonTemplatePackage(
IDownloadedPackInfo sourceInfo,
string filterReason,
Dictionary <string, FilteredPackageInfo> filteredPackages,
ScanningStats scanningStats,
string?oldTemplatePackageVersion,
string?oldNonTemplatePackageVersion)
{
Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is skipped, {filterReason}");
filteredPackages[sourceInfo.Name] = new FilteredPackageInfo(sourceInfo, filterReason);
if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion))
{
if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
{
scanningStats.NewNonTemplatePacksCount++;
}
else
{
scanningStats.BecameNonTemplatePacksCount++;
}
}
else
{
if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
{
scanningStats.UpdatedNonTemplatePacksCount++;
}
}
}
private static ITemplateEngineHost CreateHost(IDownloadedPackInfo packInfo)
{
string hostIdentifier = HostIdentifierBase + packInfo.Id;
ITemplateEngineHost host = TemplateEngineHostHelper.CreateHost(hostIdentifier);
return(host);
}
internal PreFilterResultList FilterPack(IDownloadedPackInfo packInfo)
{
List <PreFilterResult> resultList = new List <PreFilterResult>();
foreach (Func <IDownloadedPackInfo, PreFilterResult> filter in _preFilters)
{
PreFilterResult result = filter(packInfo);
resultList.Add(result);
}
return(new PreFilterResultList(resultList));
}
private void ProcessTemplatePackage(
IDownloadedPackInfo sourceInfo,
Dictionary <string, TemplatePackageSearchData> newCache,
Dictionary <string, FilteredPackageInfo> filteredPackages,
ScanningStats scanningStats,
string?oldTemplatePackageVersion,
string?oldNonTemplatePackageVersion
)
{
IEnumerable <TemplateSearchData> foundTemplates = TryGetTemplatesInPack(sourceInfo, _additionalDataProducers);
Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is processed");
if (foundTemplates.Any())
{
Verbose.WriteLine("Found templates:");
foreach (TemplateSearchData template in foundTemplates)
{
string shortNames = string.Join(",", template.ShortNameList);
Verbose.WriteLine($" - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}");
}
newCache[sourceInfo.Name] = new TemplatePackageSearchData(sourceInfo, foundTemplates, sourceInfo.ProduceAdditionalData(_additionalDataProducers));
if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion))
{
if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
{
scanningStats.NewTemplatePacks.Add(newCache[sourceInfo.Name]);
}
else
{
scanningStats.UpdatedTemplatePacks.Add((newCache[sourceInfo.Name], oldTemplatePackageVersion));
}
}
else
{
if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
{
scanningStats.BecameTemplatePacks.Add((newCache[sourceInfo.Name]));
}
}
}
else
{
ProcessNonTemplatePackage(
sourceInfo,
"Failed to scan the package for the templates, the package may contain invalid templates.",
filteredPackages,
scanningStats,
oldTemplatePackageVersion,
oldNonTemplatePackageVersion);
}
}
public void CreateDataForTemplatePack(IDownloadedPackInfo packInfo, IReadOnlyList <ITemplateInfo> templateList, IEngineEnvironmentSettings environment)
{
IHostSpecificDataLoader hostDataLoader = new HostSpecificDataLoader(environment.SettingsLoader);
foreach (ITemplateInfo template in templateList)
{
HostSpecificTemplateData hostData = hostDataLoader.ReadHostSpecificTemplateData(template);
// store the host data if it has any info that could affect searching for this template.
if (hostData.IsHidden || hostData.SymbolInfo.Count > 0)
{
_hostDataForPackByTemplate[template.Identity] = hostData;
}
}
}
private bool CheckIfPackageIsFiltered(
IDownloadedPackInfo sourceInfo,
Dictionary <string, FilteredPackageInfo> filteredPackages,
ScanningStats scanningStats,
string?oldTemplatePackageVersion,
string?oldNonTemplatePackageVersion)
{
PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(sourceInfo);
if (preFilterResult.ShouldBeFiltered)
{
ProcessNonTemplatePackage(sourceInfo, preFilterResult.Reason, filteredPackages, scanningStats, oldTemplatePackageVersion, oldNonTemplatePackageVersion);
}
return(preFilterResult.ShouldBeFiltered);
}
public async Task <PackSourceCheckResult> CheckPackagesAsync()
{
List <PackCheckResult> checkResultList = new List <PackCheckResult>();
HashSet <IPackInfo> alreadySeenPacks = new HashSet <IPackInfo>();
HashSet <string> alreadySeenTemplateIdentities = new HashSet <string>();
foreach (IPackProvider packProvider in _packProviders)
{
Console.WriteLine($"Processing pack provider {packProvider.Name}:");
int count = 0;
Console.WriteLine($"{await packProvider.GetPackageCountAsync().ConfigureAwait(false)} packs discovered, starting processing");
await foreach (IPackInfo sourceInfo in packProvider.GetCandidatePacksAsync().ConfigureAwait(false))
{
if (alreadySeenPacks.Contains(sourceInfo))
{
Verbose.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is already processed.");
continue;
}
alreadySeenPacks.Add(sourceInfo);
IDownloadedPackInfo packInfo = await packProvider.DownloadPackageAsync(sourceInfo).ConfigureAwait(false);
if (packInfo == null)
{
Console.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is not processed.");
continue;
}
PackCheckResult preFilterResult = PrefilterPackInfo(packInfo);
if (preFilterResult.PreFilterResults.ShouldBeFiltered)
{
Verbose.WriteLine($"{packInfo.Id}::{packInfo.Version} is skipped, {preFilterResult.PreFilterResults.Reason}");
checkResultList.Add(preFilterResult);
}
else
{
PackCheckResult packCheckResult = _packChecker.TryGetTemplatesInPack(packInfo, _additionalDataProducers, alreadySeenTemplateIdentities);
checkResultList.Add(packCheckResult);
Verbose.WriteLine($"{packCheckResult.PackInfo.Id}::{packCheckResult.PackInfo.Version} is processed");
if (packCheckResult.FoundTemplates.Any())
{
Verbose.WriteLine("Found templates:");
foreach (ITemplateInfo template in packCheckResult.FoundTemplates)
{
string shortNames = string.Join(",", template.ShortNameList);
Verbose.WriteLine($" - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}");
}
}
else
{
Verbose.WriteLine("No templates were found");
}
// Record the found identities - to skip these templates when checking additional packs.
// Some template identities are the same in multiple packs on nuget.
// For this scraper, first in wins.
alreadySeenTemplateIdentities.UnionWith(packCheckResult.FoundTemplates.Select(t => t.Identity));
}
++count;
if ((count % 10) == 0)
{
Console.WriteLine($"{count} packs are processed");
}
}
Console.WriteLine($"All packs from pack provider {packProvider.Name} are processed");
}
Console.WriteLine("All packs are processed");
if (!_saveCandidatePacks)
{
Console.WriteLine("Removing downloaded packs");
foreach (IPackProvider provider in _packProviders)
{
provider.DeleteDownloadedPacks();
}
Console.WriteLine("Downloaded packs were removed");
}
return(new PackSourceCheckResult(checkResultList, _additionalDataProducers));
}
private PackCheckResult PrefilterPackInfo(IDownloadedPackInfo packInfo)
{
PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(packInfo);
return(new PackCheckResult(packInfo, preFilterResult));
}
internal PackCheckResult(IDownloadedPackInfo packInfo, PreFilterResultList preFilterResults)
{
PackInfo = packInfo;
PreFilterResults = preFilterResults;
FoundTemplates = new List <ITemplateInfo>();
}
internal PackCheckResult(IDownloadedPackInfo packInfo, IReadOnlyList <ITemplateInfo> foundTemplates)
{
PackInfo = packInfo;
PreFilterResults = new PreFilterResultList();
FoundTemplates = foundTemplates;
}
