public void Crawl() { Init(); _srcs = new List <string>(); string connectionString = Db.For <Image>().ConnectionString; OutLineFormat("Connection String: {0}", ConsoleColor.Cyan, connectionString); try { connectionString.SafeWriteToFile(".\\Analytics.sqlite.connection.txt"); }catch {} ImageCrawler crawler = new ImageCrawler(); crawler.OnImageFound = (u, src) => { if (!_srcs.Contains(src)) { _srcs.Add(src); string url = u.ToString(); OutFormat("Url: {0}\r\n", ConsoleColor.Cyan, url); OutFormat("Src: {0}\r\n", ConsoleColor.Green, src); } }; crawler.Crawl("http://www.funnycatpix.com/"); }
private static void FillTeamDetails(Team team) { var locationSpider = new EuroleagueTeamLocationSpider(team.Url); var location = locationSpider.Run(); var openCageDataApiCrawler = new OpenCageDataApiCrawler(); var geolocation = openCageDataApiCrawler.GetAsync(location.Address).Result; var teamLocation = new Location(location.Address, location.Arena, geolocation.Longitude, geolocation.Latitude); team.SetLocation(teamLocation); var imageCrawler = new ImageCrawler(); var teamLogoBase64 = imageCrawler.GetBase64(team.LogoUrl); team.SetLogo(teamLogoBase64); }
public void CrawlForImages() { ConnectionStringSettings s = new ConnectionStringSettings(); s.ProviderName = SQLiteRegistrar.SQLiteFactoryAssemblyQualifiedName(); string path = Prompt("Please enter the path to the Analytics database"); s.ConnectionString = string.Format("Data Source={0};Version=3;", path); DefaultConnectionStringResolver.Instance.Resolver = (cn) => { if (cn.Equals(Image.ConnectionName(typeof(Image)))) { OutFormat("Resolved connectionString {0}", s.ConnectionString); return(s); } OutFormat("Couldn't resolve connectionString for connectionName {0}", cn); return(null); }; SQLiteRegistrar.Register <Image>(); ImageCrawler crawler = new ImageCrawler(); crawler.OnImageFound = (u, src) => { string url = u.ToString(); OutFormat("Url: {0}\r\n", ConsoleColor.Cyan, url); OutFormat("Src: {0}\r\n", ConsoleColor.Green, src); }; crawler.ActionChanged += (cr, args) => { OutFormat("{0}::{1}", args.OldAction.ToString(), args.NewAction.ToString()); }; string startUrl = Prompt("Enter the starting url"); crawler.Name = startUrl; Crawler data = new Crawler(); data.Name = startUrl; data.RootUrl = startUrl; data.Save(); crawler.Crawl(startUrl); }
public void ImageCrawlerCreateMineShouldSetRootUrlAndName() { Dictionary <string, string> testSettings = new Dictionary <string, string>(); string setAppNameTo = "TheMonkey"; testSettings.Add("ApplicationName", setAppNameTo); DefaultConfiguration.SetAppSettings(testSettings); SQLiteRegistrar.Register(Dao.ConnectionName(typeof(Url))); Db.TryEnsureSchema <Url>(); string appName = DefaultConfiguration.GetAppSetting("ApplicationName", "BAD"); string rootUrl = "http://www.flickr.com/galleries/"; Expect.IsFalse(appName.Equals("BAD"), "ApplicationName was not set in the config file"); ImageCrawler c = (ImageCrawler)ImageCrawler.CreateMine(rootUrl); Expect.AreEqual(appName, setAppNameTo); Expect.AreEqual(appName, c.Name); Expect.AreEqual(rootUrl, c.Root); }