public void Process(NCrawler.Crawler crawler, NCrawler.PropertyBag propertyBag) { String uri = propertyBag.Step.Uri.ToString(); try { ImportedItem item = new ImportedItem(); item.ImportHash = this.siteHash; item.Guid = System.Guid.NewGuid().ToString(); item.SubscriptionId = this.siteGuid; item.ContentType = propertyBag.ContentType; item.ContentEncoding = propertyBag.ContentEncoding; item.Expires = UtcDateTime.Now.AddHours(6); item.Inserted = UtcDateTime.Now; item.Title = propertyBag.Title; item.Uri = propertyBag.Step.Uri.ToString(); ImportedItemDao dao = new ImportedItemDao(); using (Transaction tx = new Transaction()) { dao.Save<ImportedItem>(item); tx.Commit(); } } catch (Exception e) { Logging.Database.Write("import-error", "Unexpected exception importing url:" + uri + ", stack:" + e.StackTrace); } }
void c_AfterDownload(object sender, NCrawler.Events.AfterDownloadEventArgs e) { CrawlProgressEventArgs e2 = new CrawlProgressEventArgs(); e2.Uri = e.CrawlStep.Uri; e2.EventType = CrawlProgressEventType.AfterCrawl; OnProgress(e2); }
public void Process(NCrawler.Crawler crawler, NCrawler.PropertyBag propertyBag) { CrawlerEntry entry = new CrawlerEntry(); entry.Uri = propertyBag.Step.Uri; entry.ContentType = propertyBag.ContentType; lock (key) { links.Add(entry); } }
private void crawler_CrawlFinished(object sender, NCrawler.Events.CrawlFinishedEventArgs e) { log.Info("Crawling complete"); if (this.config.Optimize) { log.Info("Optimizing index"); this.repository.Optimize(); } else { log.Info("Index optimization disabled"); } repository.Dispose(); }
public void Process(NCrawler.Crawler crawler, NCrawler.PropertyBag propertyBag) { if (propertyBag.StatusCode != HttpStatusCode.OK) { return; } if (!("text/css".Equals(propertyBag.ContentType,StringComparison.OrdinalIgnoreCase))) { return; } String fullpath = propertyBag.ResponseUri.GetLeftPart(UriPartial.Path); String relative = fullpath.Substring(0, fullpath.LastIndexOf("/")); //Look for images within the css file using (Stream reader = propertyBag.GetResponse()) using (StreamReader sr = new StreamReader(reader)) { String css = sr.ReadToEnd(); MatchCollection matches = pattern.Matches(css); foreach (Match match in matches) { String link = match.Groups[1].Value; if (!link.Contains("http")) { String normalized = link.NormalizeUri(fullpath); if (normalized.IsNull()) continue; crawler.AddStep(new Uri(normalized), propertyBag.Step.Depth + 1, propertyBag.Step, new Dictionary<string, object> { }); } } } }
void c_DownloadProgress(object sender, NCrawler.Events.DownloadProgressEventArgs e) { }