public void Crawl()
{
Init();
_srcs = new List <string>();
string connectionString = Db.For <Image>().ConnectionString;
OutLineFormat("Connection String: {0}", ConsoleColor.Cyan, connectionString);
try
{
connectionString.SafeWriteToFile(".\\Analytics.sqlite.connection.txt");
}catch {}
ImageCrawler crawler = new ImageCrawler();
crawler.OnImageFound = (u, src) =>
{
if (!_srcs.Contains(src))
{
_srcs.Add(src);
string url = u.ToString();
OutFormat("Url: {0}\r\n", ConsoleColor.Cyan, url);
OutFormat("Src: {0}\r\n", ConsoleColor.Green, src);
}
};
crawler.Crawl("http://www.funnycatpix.com/");
}
private static void FillTeamDetails(Team team)
{
var locationSpider = new EuroleagueTeamLocationSpider(team.Url);
var location = locationSpider.Run();
var openCageDataApiCrawler = new OpenCageDataApiCrawler();
var geolocation = openCageDataApiCrawler.GetAsync(location.Address).Result;
var teamLocation = new Location(location.Address, location.Arena, geolocation.Longitude,
geolocation.Latitude);
team.SetLocation(teamLocation);
var imageCrawler = new ImageCrawler();
var teamLogoBase64 = imageCrawler.GetBase64(team.LogoUrl);
team.SetLogo(teamLogoBase64);
}
public void CrawlForImages()
{
ConnectionStringSettings s = new ConnectionStringSettings();
s.ProviderName = SQLiteRegistrar.SQLiteFactoryAssemblyQualifiedName();
string path = Prompt("Please enter the path to the Analytics database");
s.ConnectionString = string.Format("Data Source={0};Version=3;", path);
DefaultConnectionStringResolver.Instance.Resolver = (cn) =>
{
if (cn.Equals(Image.ConnectionName(typeof(Image))))
{
OutFormat("Resolved connectionString {0}", s.ConnectionString);
return(s);
}
OutFormat("Couldn't resolve connectionString for connectionName {0}", cn);
return(null);
};
SQLiteRegistrar.Register <Image>();
ImageCrawler crawler = new ImageCrawler();
crawler.OnImageFound = (u, src) =>
{
string url = u.ToString();
OutFormat("Url: {0}\r\n", ConsoleColor.Cyan, url);
OutFormat("Src: {0}\r\n", ConsoleColor.Green, src);
};
crawler.ActionChanged += (cr, args) =>
{
OutFormat("{0}::{1}", args.OldAction.ToString(), args.NewAction.ToString());
};
string startUrl = Prompt("Enter the starting url");
crawler.Name = startUrl;
Crawler data = new Crawler();
data.Name = startUrl;
data.RootUrl = startUrl;
data.Save();
crawler.Crawl(startUrl);
}
public void ImageCrawlerCreateMineShouldSetRootUrlAndName()
{
Dictionary <string, string> testSettings = new Dictionary <string, string>();
string setAppNameTo = "TheMonkey";
testSettings.Add("ApplicationName", setAppNameTo);
DefaultConfiguration.SetAppSettings(testSettings);
SQLiteRegistrar.Register(Dao.ConnectionName(typeof(Url)));
Db.TryEnsureSchema <Url>();
string appName = DefaultConfiguration.GetAppSetting("ApplicationName", "BAD");
string rootUrl = "http://www.flickr.com/galleries/";
Expect.IsFalse(appName.Equals("BAD"), "ApplicationName was not set in the config file");
ImageCrawler c = (ImageCrawler)ImageCrawler.CreateMine(rootUrl);
Expect.AreEqual(appName, setAppNameTo);
Expect.AreEqual(appName, c.Name);
Expect.AreEqual(rootUrl, c.Root);
}
