public void Init(HttpApplication context)
        {
            this._context    = context;
            _prerenderConfig = ConfigurationManager.GetSection(PRERENDER_SECTION_KEY) as PrerenderConfigSection;

            context.BeginRequest += context_BeginRequest;
        }
        public void Init(HttpApplication context)
        {
            this._context = context;
            _prerenderConfig = ConfigurationManager.GetSection(PRERENDER_SECTION_KEY) as PrerenderConfigSection;

            context.BeginRequest += context_BeginRequest;
        }
        public void Init(HttpApplication context)
        {
            _context         = context;
            _prerenderConfig = ConfigurationManager.GetSection(PRERENDER_SECTION_KEY) as PrerenderConfigSection;

            _crawlerUserAgents  = GetCrawlerUserAgents(_prerenderConfig);
            _extensionsToIgnore = GetExtensionsToIgnore(_prerenderConfig);

            context.BeginRequest += context_BeginRequest;
        }
示例#4
0
        public void Init(HttpApplication context)
        {
            this._context    = context;
            _prerenderConfig = ConfigurationManager.GetSection(PRERENDER_SECTION_KEY) as PrerenderConfigSection;
            BuildCrawlerUsageAgentsList();
            BuildExtensionsToIgnoreList();
            BuildBlackList();
            BuildWhiteList();
            var wrapper = new EventHandlerTaskAsyncHelper(ContextBeginRequest);

            context.AddOnBeginRequestAsync(wrapper.BeginEventHandler, wrapper.EndEventHandler);
        }
        private List <String> GetExtensionsToIgnore(PrerenderConfigSection config)
        {
            List <string> extensionsToIgnore = new List <string>()
            {
                ".js", ".css", ".less", ".png", ".jpg", ".jpeg",
                ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg",
                ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg",
                ".iso", ".flv", ".m4v", ".torrent"
            };

            if (config.ExtensionsToIgnore.IsNotEmpty())
            {
                extensionsToIgnore.AddRange(config.ExtensionsToIgnore);
            }
            return(extensionsToIgnore);
        }
        private List <String> GetCrawlerUserAgents(PrerenderConfigSection config)
        {
            List <string> crawlerUserAgents = new List <string>()
            {
                /*"googlebot",*/ "yahoo", "bingbot", "yandex", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot",
                "embedly", "quora link preview", "showyoubot", "outbrain", "pinterest/0.", "pinterestbot/",
                "developers.google.com/+/web/snippet", "slackbot", "vkShare", "W3C_Validator",
                "redditbot", "Applebot", "WhatsApp", "flipboard", "tumblr", "bitlybot",
                "SkypeUriPreview", "nuzzel", "Discordbot", "Google Page Speed", "x-bufferbot"
            };

            if (config.CrawlerUserAgents.IsNotEmpty())
            {
                crawlerUserAgents.AddRange(config.CrawlerUserAgents);
            }

            return(crawlerUserAgents);
        }