public string StartCrawling()
{
sm.AddtoStartStopQueue("start");
sm.AddToXMLQueue("http://www.bleacherreport.com/robots.txt");
sm.AddToXMLQueue("http://www.cnn.com/robots.txt");
return("started");
}
private void AddToXMLQueue(List <string> xmls)
{
foreach (string s in xmls)
{
sm.AddToXMLQueue(s);
}
}
