private static void ExecuteMockTest(string file)
{
var compiler = new WebcrawlerCompiler();
var commandset = compiler.ParseCommandset(File.ReadAllLines(file));
var crawler = new WebcrawlingUtility(new ProviderMock());
crawler.ContextCommandset = compiler.ContextCommandset;
crawler.Crawling(commandset);
}
private static void ExecuteMockTest(string template)
{
var compiler = new WebcrawlerCompiler();
var commandset = compiler.ParseCommandset(Regex.Split(template, Environment.NewLine));
var crawler = new WebcrawlingUtility(new ProviderMock());
crawler.ContextCommandset = compiler.ContextCommandset;
crawler.Crawling(commandset);
}
public static void Process(string template, Dictionary<string, string> parameters)
{
if (!string.IsNullOrEmpty(template)) {
var compiler = new WebcrawlerCompiler();
var commandset = compiler.ParseCommandset(Regex.Split(template, Environment.NewLine));
var crawler = new WebcrawlingUtility(new InfostoreQueueProvider());
crawler.ContextCommandset = compiler.ContextCommandset;
crawler.Crawling(commandset);
}
}
static void Main(string[] args)
{
var templatesfiles = new[]
{
@"G:\Entwicklung\GitHUB\ButlerFramework\InformationSatellite\SatelliteTest\SatelliteTest\App_Data\shopping\mytoys.crawl",
//@"..\..\App_Data\chefkoch.crawl",
//@"..\..\App_Data\stackoverflow.crawl",
//@"..\..\App_Data\fun\9gag.crawl"
};
templatesfiles.ToList().ForEach(templatefile =>
{
var template = File.ReadAllText(templatefile);
var compiler = new WebcrawlerCompiler();
var commandset = compiler.ParseCommandset(Regex.Split(template, Environment.NewLine));
});
}
private static void CrawlInformation(string templatefile)
{
var encoding = Encoding.GetEncoding(1252);
var index = default(int);
var lines = LoadMappingTemplate(templatefile);
var baseUri = string.Empty;
var sourceUri = string.Empty;
foreach (var line in lines)
{
var tokens = line.Split('=');
var key = tokens != null ? tokens.FirstOrDefault() ?? string.Empty : string.Empty;
var value = tokens != null ? tokens.Skip(1).FirstOrDefault() ?? string.Empty : string.Empty;
if (key.StartsWith("BaseUri"))
{
baseUri = value;
}
else if (key.StartsWith("Source"))
{
sourceUri = value;
break;
}
index++;
}
for (int i = index + 1; i <= lines.Length; i++)
{
if (lines[i] != string.Empty)
{
index = i;
break;
}
}
var compiler = new WebcrawlerCompiler();
var commandset = compiler.ParseCommandset(lines, index);
ContextDictionary[WebcrawlingUtilityConstants.BaseUri] = new[] { baseUri };
ContextDictionary[WebcrawlingUtilityConstants.CurrentUri] = new[] { sourceUri };
var miningutility = new WebcrawlingUtility
{
ContextCommandset = compiler.ContextCommandset,
ContextDictionary = ContextDictionary
};
miningutility.Crawling(commandset);
}
