public PackCheckResult TryGetTemplatesInPack(IInstalledPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities, bool persistHive = false)
{
ITemplateEngineHost host = CreateHost(packInfo);
EngineEnvironmentSettings environment = new EngineEnvironmentSettings(host, x => new SettingsLoader(x));
PackCheckResult checkResult;
try
{
if (TryInstallPackage(packInfo.Path, environment, out IReadOnlyList <ITemplateInfo> installedTemplates))
{
IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList();
checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates);
ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environment);
}
else
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
}
catch
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
if (!persistHive)
{
TryCleanup(environment);
}
return(checkResult);
}
public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities)
{
ITemplateEngineHost host = CreateHost(packInfo);
using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true);
PackCheckResult checkResult;
try
{
if (TryInstallPackage(packInfo.Path, environmentSettings, out IReadOnlyList <ITemplateInfo> installedTemplates))
{
IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList();
checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates);
ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environmentSettings);
}
else
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
}
catch
{
IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
checkResult = new PackCheckResult(packInfo, foundTemplates);
}
return(checkResult);
}
public PackSourceCheckResult CheckPackages()
{
List <PackCheckResult> checkResultList = new List <PackCheckResult>();
HashSet <string> alreadySeenTemplateIdentities = new HashSet <string>();
int count = 0;
foreach (IInstalledPackInfo packInfo in _packProvider.CandidatePacks)
{
PackCheckResult preFilterResult = PrefilterPackInfo(packInfo);
if (preFilterResult.PreFilterResults.ShouldBeFiltered)
{
checkResultList.Add(preFilterResult);
}
else
{
PackCheckResult packCheckResult = _packChecker.TryGetTemplatesInPack(packInfo, _additionalDataProducers, alreadySeenTemplateIdentities);
checkResultList.Add(packCheckResult);
// Record the found identities - to skip these templates when checking additional packs.
// Some template identities are the same in multiple packs on nuget.
// For this scraper, first in wins.
alreadySeenTemplateIdentities.UnionWith(packCheckResult.FoundTemplates.Select(t => t.Identity));
}
++count;
if ((count % 10) == 0)
{
Console.WriteLine($"{count} packs processed");
}
}
if (!_saveCandidatePacks)
{
_packProvider.DeleteDownloadedPacks();
}
return(new PackSourceCheckResult(checkResultList, _additionalDataProducers));
}
public async Task <PackSourceCheckResult> CheckPackagesAsync()
{
List <PackCheckResult> checkResultList = new List <PackCheckResult>();
HashSet <IPackInfo> alreadySeenPacks = new HashSet <IPackInfo>();
HashSet <string> alreadySeenTemplateIdentities = new HashSet <string>();
foreach (IPackProvider packProvider in _packProviders)
{
Console.WriteLine($"Processing pack provider {packProvider.Name}:");
int count = 0;
Console.WriteLine($"{await packProvider.GetPackageCountAsync().ConfigureAwait(false)} packs discovered, starting processing");
await foreach (IPackInfo sourceInfo in packProvider.GetCandidatePacksAsync().ConfigureAwait(false))
{
if (alreadySeenPacks.Contains(sourceInfo))
{
Verbose.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is already processed.");
continue;
}
alreadySeenPacks.Add(sourceInfo);
IDownloadedPackInfo packInfo = await packProvider.DownloadPackageAsync(sourceInfo).ConfigureAwait(false);
if (packInfo == null)
{
Console.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is not processed.");
continue;
}
PackCheckResult preFilterResult = PrefilterPackInfo(packInfo);
if (preFilterResult.PreFilterResults.ShouldBeFiltered)
{
Verbose.WriteLine($"{packInfo.Id}::{packInfo.Version} is skipped, {preFilterResult.PreFilterResults.Reason}");
checkResultList.Add(preFilterResult);
}
else
{
PackCheckResult packCheckResult = _packChecker.TryGetTemplatesInPack(packInfo, _additionalDataProducers, alreadySeenTemplateIdentities);
checkResultList.Add(packCheckResult);
Verbose.WriteLine($"{packCheckResult.PackInfo.Id}::{packCheckResult.PackInfo.Version} is processed");
if (packCheckResult.FoundTemplates.Any())
{
Verbose.WriteLine("Found templates:");
foreach (ITemplateInfo template in packCheckResult.FoundTemplates)
{
string shortNames = string.Join(",", template.ShortNameList);
Verbose.WriteLine($" - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}");
}
}
else
{
Verbose.WriteLine("No templates were found");
}
// Record the found identities - to skip these templates when checking additional packs.
// Some template identities are the same in multiple packs on nuget.
// For this scraper, first in wins.
alreadySeenTemplateIdentities.UnionWith(packCheckResult.FoundTemplates.Select(t => t.Identity));
}
++count;
if ((count % 10) == 0)
{
Console.WriteLine($"{count} packs are processed");
}
}
Console.WriteLine($"All packs from pack provider {packProvider.Name} are processed");
}
Console.WriteLine("All packs are processed");
if (!_saveCandidatePacks)
{
Console.WriteLine("Removing downloaded packs");
foreach (IPackProvider provider in _packProviders)
{
provider.DeleteDownloadedPacks();
}
Console.WriteLine("Downloaded packs were removed");
}
return(new PackSourceCheckResult(checkResultList, _additionalDataProducers));
}
private void ProduceAdditionalDataForPack(IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, PackCheckResult packCheckResult, EngineEnvironmentSettings environment)
{
if (!packCheckResult.AnyTemplates)
{
return;
}
foreach (IAdditionalDataProducer dataProducer in additionalDataProducers)
{
dataProducer.CreateDataForTemplatePack(packCheckResult.PackInfo, packCheckResult.FoundTemplates, environment);
}
}
public PackSourceCheckResult CheckPackages()
{
List <PackCheckResult> checkResultList = new List <PackCheckResult>();
HashSet <string> alreadySeenTemplateIdentities = new HashSet <string>();
int count = 0;
Console.WriteLine($"{_packProvider.CandidatePacksCount} packs discovered, starting processing");
foreach (IInstalledPackInfo packInfo in _packProvider.CandidatePacks)
{
PackCheckResult preFilterResult = PrefilterPackInfo(packInfo);
if (preFilterResult.PreFilterResults.ShouldBeFiltered)
{
Verbose.WriteLine($"{packInfo.Id}::{packInfo.Version} is skipped, {preFilterResult.PreFilterResults.Reason}");
checkResultList.Add(preFilterResult);
}
else
{
PackCheckResult packCheckResult = _packChecker.TryGetTemplatesInPack(packInfo, _additionalDataProducers, alreadySeenTemplateIdentities);
checkResultList.Add(packCheckResult);
Verbose.WriteLine($"{packCheckResult.PackInfo.Id}::{packCheckResult.PackInfo.Version} was processed");
if (packCheckResult.FoundTemplates.Any())
{
Verbose.WriteLine("Found templates:");
foreach (ITemplateInfo template in packCheckResult.FoundTemplates)
{
Verbose.WriteLine($" - {template.Identity} ({template.ShortName}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}");
}
}
else
{
Verbose.WriteLine("No templates were found");
}
// Record the found identities - to skip these templates when checking additional packs.
// Some template identities are the same in multiple packs on nuget.
// For this scraper, first in wins.
alreadySeenTemplateIdentities.UnionWith(packCheckResult.FoundTemplates.Select(t => t.Identity));
}
++count;
if ((count % 10) == 0)
{
Console.WriteLine($"{count} packs processed");
}
}
Console.WriteLine("All packs processed");
if (!_saveCandidatePacks)
{
Console.WriteLine("Removing downloaded packs");
_packProvider.DeleteDownloadedPacks();
Console.WriteLine("Downloaded packs were removed");
}
return(new PackSourceCheckResult(checkResultList, _additionalDataProducers));
}