public static void EnsureOne(Bam.Net.Analytics.ImageCrawler crawler) { Crawler value = Crawler.OneWhere(c => c.Name == crawler.Name); if (value == null) { value = new Crawler(); value.Name = crawler.Name; value.RootUrl = crawler.Root; value.Save(); } else { crawler.Root = value.RootUrl; } }
/// <summary> /// Reads the target and saves the url of any img tag it finds /// </summary> /// <param name="target"></param> public override void ProcessTarget(string target) { Url url = Url.FromUri(target);//new Url(target); if (url.Id == null) { url.Save(); } CQ q = CQ.Create(url.Html); q["img"].Each((i, d) => { try { string imgUrl = d.Attributes["src"]; if (Uri.IsWellFormedUriString(imgUrl, UriKind.Relative)) { imgUrl = $"{url.ProtocolOfProtocolId.Value}://{url.DomainOfDomainId.Value}{url.PathOfPathId.Value}{imgUrl}"; } Url image = Url.FromUri(new Uri(imgUrl), true);// new Url(imgUrl); Image img = Image.OneWhere(c => c.UrlId == image.Id); if (img == null) { Crawler cr = Crawler.OneWhere(c => c.Name == this.Name); cr.RootUrl = target; img = new Image(); img.UrlId = image.Id; img.Date = DateTime.UtcNow; img.CrawlerId = cr.Id; img.Save(); } if (OnImageFound != null) { OnImageFound(url, imgUrl); } } catch (Exception ex) { Logging.Log.AddEntry("Error occurred in image crawler: {0}", ex, ex.Message); } }); }