Example #1
0
        private static void ExecuteMockTest(string template)
        {
            var compiler = new WebcrawlerCompiler();
            var commandset = compiler.ParseCommandset(Regex.Split(template, Environment.NewLine));

            var crawler = new WebcrawlingUtility(new ProviderMock());
            crawler.ContextCommandset = compiler.ContextCommandset;
            crawler.Crawling(commandset);
        }
Example #2
0
        private static void ExecuteMockTest(string file)
        {
            var compiler = new WebcrawlerCompiler();
            var commandset = compiler.ParseCommandset(File.ReadAllLines(file));

            var crawler = new WebcrawlingUtility(new ProviderMock());
            crawler.ContextCommandset = compiler.ContextCommandset;
            crawler.Crawling(commandset);
        }
Example #3
0
        public static void Process(string template, Dictionary<string, string> parameters)
        {
            if (!string.IsNullOrEmpty(template)) {
                var compiler = new WebcrawlerCompiler();
                var commandset = compiler.ParseCommandset(Regex.Split(template, Environment.NewLine));                

                var crawler = new WebcrawlingUtility(new InfostoreQueueProvider());
                crawler.ContextCommandset = compiler.ContextCommandset;
                crawler.Crawling(commandset);
            }
        }
Example #4
0
        private static void CrawlInformation(string templatefile)
        {
            var encoding = Encoding.GetEncoding(1252);

            var index = default(int);
            var lines = LoadMappingTemplate(templatefile);

            var baseUri = string.Empty;
            var sourceUri = string.Empty;
            foreach (var line in lines)
            {
                var tokens = line.Split('=');
                var key = tokens != null ? tokens.FirstOrDefault() ?? string.Empty : string.Empty;
                var value = tokens != null ? tokens.Skip(1).FirstOrDefault() ?? string.Empty : string.Empty;
                if (key.StartsWith("BaseUri"))
                {
                    baseUri = value;
                }
                else if (key.StartsWith("Source"))
                {
                    sourceUri = value;
                    break;
                }
                index++;
            }
            for (int i = index + 1; i <= lines.Length; i++)
            {
                if (lines[i] != string.Empty)
                {
                    index = i;
                    break;
                }
            }

            var compiler = new WebcrawlerCompiler();
            var commandset = compiler.ParseCommandset(lines, index);
            ContextDictionary[WebcrawlingUtilityConstants.BaseUri] = new[] { baseUri };
            ContextDictionary[WebcrawlingUtilityConstants.CurrentUri] = new[] { sourceUri };

            var miningutility = new WebcrawlingUtility
            {
                ContextCommandset = compiler.ContextCommandset,
                ContextDictionary = ContextDictionary
            };
            miningutility.Crawling(commandset);
        }