public Crawler(string url)
{
errorList = new List<string>();
myurl = url;
robot = new getRobot(url);
CloudStorageAccount storageAccount;storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
tableClient = storageAccount.CreateCloudTableClient();
visitedUrl = new HashSet<string>();
siteMap = new getSiteMap(robot.GetSiteMap());
disAllow = robot.GetDisAllowed();
myStorage = new Storage();
}
public Crawler(string url)
{
errorList = new List <string>();
myurl = url;
robot = new getRobot(url);
CloudStorageAccount storageAccount; storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
tableClient = storageAccount.CreateCloudTableClient();
visitedUrl = new HashSet <string>();
siteMap = new getSiteMap(robot.GetSiteMap());
disAllow = robot.GetDisAllowed();
myStorage = new Storage();
}