public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities)
        {
            ITemplateEngineHost host = CreateHost(packInfo);

            using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true);

            PackCheckResult checkResult;

            try
            {
                if (TryInstallPackage(packInfo.Path, environmentSettings, out IReadOnlyList <ITemplateInfo> installedTemplates))
                {
                    IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList();
                    checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates);
                    ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environmentSettings);
                }
                else
                {
                    IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
                    checkResult = new PackCheckResult(packInfo, foundTemplates);
                }
            }
            catch
            {
                IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
                checkResult = new PackCheckResult(packInfo, foundTemplates);
            }
            return(checkResult);
        }
Esempio n. 2
0
        public PackCheckResult TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers, HashSet <string> alreadySeenTemplateIdentities, bool persistHive = false)
        {
            ITemplateEngineHost       host        = CreateHost(packInfo);
            EngineEnvironmentSettings environment = new EngineEnvironmentSettings(host, x => new SettingsLoader(x));
            PackCheckResult           checkResult;

            try
            {
                if (TryInstallPackage(packInfo.Path, environment, out IReadOnlyList <ITemplateInfo> installedTemplates))
                {
                    IReadOnlyList <ITemplateInfo> filteredInstalledTemplates = installedTemplates.Where(t => !alreadySeenTemplateIdentities.Contains(t.Identity)).ToList();
                    checkResult = new PackCheckResult(packInfo, filteredInstalledTemplates);
                    ProduceAdditionalDataForPack(additionalDataProducers, checkResult, environment);
                }
                else
                {
                    IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
                    checkResult = new PackCheckResult(packInfo, foundTemplates);
                }
            }
            catch
            {
                IReadOnlyList <ITemplateInfo> foundTemplates = new List <ITemplateInfo>();
                checkResult = new PackCheckResult(packInfo, foundTemplates);
            }

            if (!persistHive)
            {
                TryCleanup(environment);
            }

            return(checkResult);
        }
Esempio n. 3
0
        private IEnumerable <TemplateSearchData> TryGetTemplatesInPack(IDownloadedPackInfo packInfo, IReadOnlyList <IAdditionalDataProducer> additionalDataProducers)
        {
            ITemplateEngineHost host = TemplateEngineHostHelper.CreateHost(HostIdentifierBase + packInfo.Name);

            using EngineEnvironmentSettings environmentSettings = new EngineEnvironmentSettings(host, virtualizeSettings: true);
            Scanner scanner = new Scanner(environmentSettings);

            try
            {
                using var scanResult = scanner.Scan(packInfo.Path, scanForComponents: false);
                if (scanResult.Templates.Any())
                {
                    foreach (IAdditionalDataProducer dataProducer in additionalDataProducers)
                    {
#pragma warning disable CS0612 // Type or member is obsolete
                        dataProducer.CreateDataForTemplatePack(packInfo, scanResult.Templates, environmentSettings);
#pragma warning restore CS0612 // Type or member is obsolete
                    }

                    return(scanResult.Templates.Select(t => new TemplateSearchData(t, t.ProduceAdditionalData(additionalDataProducers, environmentSettings))));
                }
                return(Array.Empty <TemplateSearchData>());
            }
            catch (TaskCanceledException)
            {
                throw;
            }
            catch (Exception ex)
            {
                Console.WriteLine("Failed to read package {0}::{1}, details: {2}. The package will be skipped.", packInfo.Name, packInfo.Version, ex);
                return(Array.Empty <TemplateSearchData>());
            }
        }
Esempio n. 4
0
 private void ProcessNonTemplatePackage(
     IDownloadedPackInfo sourceInfo,
     string filterReason,
     Dictionary <string, FilteredPackageInfo> filteredPackages,
     ScanningStats scanningStats,
     string?oldTemplatePackageVersion,
     string?oldNonTemplatePackageVersion)
 {
     Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is skipped, {filterReason}");
     filteredPackages[sourceInfo.Name] = new FilteredPackageInfo(sourceInfo, filterReason);
     if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion))
     {
         if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
         {
             scanningStats.NewNonTemplatePacksCount++;
         }
         else
         {
             scanningStats.BecameNonTemplatePacksCount++;
         }
     }
     else
     {
         if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
         {
             scanningStats.UpdatedNonTemplatePacksCount++;
         }
     }
 }
        private static ITemplateEngineHost CreateHost(IDownloadedPackInfo packInfo)
        {
            string hostIdentifier = HostIdentifierBase + packInfo.Id;

            ITemplateEngineHost host = TemplateEngineHostHelper.CreateHost(hostIdentifier);

            return(host);
        }
Esempio n. 6
0
        internal PreFilterResultList FilterPack(IDownloadedPackInfo packInfo)
        {
            List <PreFilterResult> resultList = new List <PreFilterResult>();

            foreach (Func <IDownloadedPackInfo, PreFilterResult> filter in _preFilters)
            {
                PreFilterResult result = filter(packInfo);
                resultList.Add(result);
            }

            return(new PreFilterResultList(resultList));
        }
Esempio n. 7
0
        private void ProcessTemplatePackage(
            IDownloadedPackInfo sourceInfo,
            Dictionary <string, TemplatePackageSearchData> newCache,
            Dictionary <string, FilteredPackageInfo> filteredPackages,
            ScanningStats scanningStats,
            string?oldTemplatePackageVersion,
            string?oldNonTemplatePackageVersion
            )
        {
            IEnumerable <TemplateSearchData> foundTemplates = TryGetTemplatesInPack(sourceInfo, _additionalDataProducers);

            Verbose.WriteLine($"{sourceInfo.Name}::{sourceInfo.Version} is processed");
            if (foundTemplates.Any())
            {
                Verbose.WriteLine("Found templates:");
                foreach (TemplateSearchData template in foundTemplates)
                {
                    string shortNames = string.Join(",", template.ShortNameList);
                    Verbose.WriteLine($"  - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}");
                }
                newCache[sourceInfo.Name] = new TemplatePackageSearchData(sourceInfo, foundTemplates, sourceInfo.ProduceAdditionalData(_additionalDataProducers));
                if (string.IsNullOrWhiteSpace(oldNonTemplatePackageVersion))
                {
                    if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
                    {
                        scanningStats.NewTemplatePacks.Add(newCache[sourceInfo.Name]);
                    }
                    else
                    {
                        scanningStats.UpdatedTemplatePacks.Add((newCache[sourceInfo.Name], oldTemplatePackageVersion));
                    }
                }
                else
                {
                    if (string.IsNullOrWhiteSpace(oldTemplatePackageVersion))
                    {
                        scanningStats.BecameTemplatePacks.Add((newCache[sourceInfo.Name]));
                    }
                }
            }
            else
            {
                ProcessNonTemplatePackage(
                    sourceInfo,
                    "Failed to scan the package for the templates, the package may contain invalid templates.",
                    filteredPackages,
                    scanningStats,
                    oldTemplatePackageVersion,
                    oldNonTemplatePackageVersion);
            }
        }
Esempio n. 8
0
        public void CreateDataForTemplatePack(IDownloadedPackInfo packInfo, IReadOnlyList <ITemplateInfo> templateList, IEngineEnvironmentSettings environment)
        {
            IHostSpecificDataLoader hostDataLoader = new HostSpecificDataLoader(environment.SettingsLoader);

            foreach (ITemplateInfo template in templateList)
            {
                HostSpecificTemplateData hostData = hostDataLoader.ReadHostSpecificTemplateData(template);

                // store the host data if it has any info that could affect searching for this template.
                if (hostData.IsHidden || hostData.SymbolInfo.Count > 0)
                {
                    _hostDataForPackByTemplate[template.Identity] = hostData;
                }
            }
        }
Esempio n. 9
0
        private bool CheckIfPackageIsFiltered(
            IDownloadedPackInfo sourceInfo,
            Dictionary <string, FilteredPackageInfo> filteredPackages,
            ScanningStats scanningStats,
            string?oldTemplatePackageVersion,
            string?oldNonTemplatePackageVersion)
        {
            PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(sourceInfo);

            if (preFilterResult.ShouldBeFiltered)
            {
                ProcessNonTemplatePackage(sourceInfo, preFilterResult.Reason, filteredPackages, scanningStats, oldTemplatePackageVersion, oldNonTemplatePackageVersion);
            }
            return(preFilterResult.ShouldBeFiltered);
        }
Esempio n. 10
0
        public async Task <PackSourceCheckResult> CheckPackagesAsync()
        {
            List <PackCheckResult> checkResultList  = new List <PackCheckResult>();
            HashSet <IPackInfo>    alreadySeenPacks = new HashSet <IPackInfo>();
            HashSet <string>       alreadySeenTemplateIdentities = new HashSet <string>();

            foreach (IPackProvider packProvider in _packProviders)
            {
                Console.WriteLine($"Processing pack provider {packProvider.Name}:");
                int count = 0;
                Console.WriteLine($"{await packProvider.GetPackageCountAsync().ConfigureAwait(false)} packs discovered, starting processing");

                await foreach (IPackInfo sourceInfo in packProvider.GetCandidatePacksAsync().ConfigureAwait(false))
                {
                    if (alreadySeenPacks.Contains(sourceInfo))
                    {
                        Verbose.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is already processed.");
                        continue;
                    }
                    alreadySeenPacks.Add(sourceInfo);

                    IDownloadedPackInfo packInfo = await packProvider.DownloadPackageAsync(sourceInfo).ConfigureAwait(false);

                    if (packInfo == null)
                    {
                        Console.WriteLine($"Package {sourceInfo.Id}::{sourceInfo.Version} is not processed.");
                        continue;
                    }

                    PackCheckResult preFilterResult = PrefilterPackInfo(packInfo);
                    if (preFilterResult.PreFilterResults.ShouldBeFiltered)
                    {
                        Verbose.WriteLine($"{packInfo.Id}::{packInfo.Version} is skipped, {preFilterResult.PreFilterResults.Reason}");
                        checkResultList.Add(preFilterResult);
                    }
                    else
                    {
                        PackCheckResult packCheckResult = _packChecker.TryGetTemplatesInPack(packInfo, _additionalDataProducers, alreadySeenTemplateIdentities);
                        checkResultList.Add(packCheckResult);

                        Verbose.WriteLine($"{packCheckResult.PackInfo.Id}::{packCheckResult.PackInfo.Version} is processed");
                        if (packCheckResult.FoundTemplates.Any())
                        {
                            Verbose.WriteLine("Found templates:");
                            foreach (ITemplateInfo template in packCheckResult.FoundTemplates)
                            {
                                string shortNames = string.Join(",", template.ShortNameList);
                                Verbose.WriteLine($"  - {template.Identity} ({shortNames}) by {template.Author}, group: {(string.IsNullOrWhiteSpace(template.GroupIdentity) ? "<not set>" : template.GroupIdentity)}, precedence: {template.Precedence}");
                            }
                        }
                        else
                        {
                            Verbose.WriteLine("No templates were found");
                        }

                        // Record the found identities - to skip these templates when checking additional packs.
                        // Some template identities are the same in multiple packs on nuget.
                        // For this scraper, first in wins.
                        alreadySeenTemplateIdentities.UnionWith(packCheckResult.FoundTemplates.Select(t => t.Identity));
                    }

                    ++count;
                    if ((count % 10) == 0)
                    {
                        Console.WriteLine($"{count} packs are processed");
                    }
                }
                Console.WriteLine($"All packs from pack provider {packProvider.Name} are processed");
            }
            Console.WriteLine("All packs are processed");

            if (!_saveCandidatePacks)
            {
                Console.WriteLine("Removing downloaded packs");
                foreach (IPackProvider provider in _packProviders)
                {
                    provider.DeleteDownloadedPacks();
                }
                Console.WriteLine("Downloaded packs were removed");
            }

            return(new PackSourceCheckResult(checkResultList, _additionalDataProducers));
        }
Esempio n. 11
0
        private PackCheckResult PrefilterPackInfo(IDownloadedPackInfo packInfo)
        {
            PreFilterResultList preFilterResult = _packPreFilterer.FilterPack(packInfo);

            return(new PackCheckResult(packInfo, preFilterResult));
        }
Esempio n. 12
0
 internal PackCheckResult(IDownloadedPackInfo packInfo, PreFilterResultList preFilterResults)
 {
     PackInfo         = packInfo;
     PreFilterResults = preFilterResults;
     FoundTemplates   = new List <ITemplateInfo>();
 }
Esempio n. 13
0
 internal PackCheckResult(IDownloadedPackInfo packInfo, IReadOnlyList <ITemplateInfo> foundTemplates)
 {
     PackInfo         = packInfo;
     PreFilterResults = new PreFilterResultList();
     FoundTemplates   = foundTemplates;
 }