public void Process(NCrawler.Crawler crawler, NCrawler.PropertyBag propertyBag)
{
String uri = propertyBag.Step.Uri.ToString();
try
{
ImportedItem item = new ImportedItem();
item.ImportHash = this.siteHash;
item.Guid = System.Guid.NewGuid().ToString();
item.SubscriptionId = this.siteGuid;
item.ContentType = propertyBag.ContentType;
item.ContentEncoding = propertyBag.ContentEncoding;
item.Expires = UtcDateTime.Now.AddHours(6);
item.Inserted = UtcDateTime.Now;
item.Title = propertyBag.Title;
item.Uri = propertyBag.Step.Uri.ToString();
ImportedItemDao dao = new ImportedItemDao();
using (Transaction tx = new Transaction())
{
dao.Save<ImportedItem>(item);
tx.Commit();
}
}
catch (Exception e)
{
Logging.Database.Write("import-error", "Unexpected exception importing url:" + uri + ", stack:" + e.StackTrace);
}
}
void c_AfterDownload(object sender, NCrawler.Events.AfterDownloadEventArgs e)
{
CrawlProgressEventArgs e2 = new CrawlProgressEventArgs();
e2.Uri = e.CrawlStep.Uri;
e2.EventType = CrawlProgressEventType.AfterCrawl;
OnProgress(e2);
}
public void Process(NCrawler.Crawler crawler, NCrawler.PropertyBag propertyBag)
{
CrawlerEntry entry = new CrawlerEntry();
entry.Uri = propertyBag.Step.Uri;
entry.ContentType = propertyBag.ContentType;
lock (key)
{
links.Add(entry);
}
}
private void crawler_CrawlFinished(object sender, NCrawler.Events.CrawlFinishedEventArgs e)
{
log.Info("Crawling complete");
if (this.config.Optimize)
{
log.Info("Optimizing index");
this.repository.Optimize();
} else
{
log.Info("Index optimization disabled");
}
repository.Dispose();
}
public void Process(NCrawler.Crawler crawler, NCrawler.PropertyBag propertyBag)
{
if (propertyBag.StatusCode != HttpStatusCode.OK)
{
return;
}
if (!("text/css".Equals(propertyBag.ContentType,StringComparison.OrdinalIgnoreCase)))
{
return;
}
String fullpath = propertyBag.ResponseUri.GetLeftPart(UriPartial.Path);
String relative = fullpath.Substring(0, fullpath.LastIndexOf("/"));
//Look for images within the css file
using (Stream reader = propertyBag.GetResponse())
using (StreamReader sr = new StreamReader(reader))
{
String css = sr.ReadToEnd();
MatchCollection matches = pattern.Matches(css);
foreach (Match match in matches)
{
String link = match.Groups[1].Value;
if (!link.Contains("http"))
{
String normalized = link.NormalizeUri(fullpath);
if (normalized.IsNull())
continue;
crawler.AddStep(new Uri(normalized), propertyBag.Step.Depth + 1,
propertyBag.Step, new Dictionary<string, object>
{
});
}
}
}
}
void c_DownloadProgress(object sender, NCrawler.Events.DownloadProgressEventArgs e)
{
}