private Link addPage(Uri pageToVisit, Uri sourcePage) { if (pageToVisit.Host != BaseUri.Host && !Configuration.SpiderAllowHostViolation) { if (!hViolated.Contains(pageToVisit.Host)) // ignore the entire domain { lock (hViolated) { hViolated.Add(pageToVisit.Host); } log.Warning($"[WRN] Host Violation {pageToVisit}"); } return(null); } var lnk = new Link(pageToVisit, sourcePage); if (FetchRewrite != null) { var ev = new FetchRewriteEventArgs(pageToVisit); FetchRewrite(this, ev); // Default Uri Equality ignores Fragment if (ev.NewUri != null && ev.NewUri.ToString() != pageToVisit.ToString()) { if (ev.ShowOnLog) { log.Information($"[REW] {pageToVisit} -> {ev.NewUri}"); } lnk.ResourceRewritten(ev.NewUri); } } if (SpiderWorkData.Moved301.ContainsKey(lnk.Uri.ToString())) { string newUri = SpiderWorkData.Moved301[lnk.Uri.ToString()]; lnk.ResourceMoved(new Uri(newUri)); if (alreadyExecuted(lnk.Uri)) { return(null); } } if (alreadyExecuted(lnk.Uri)) { return(null); } var args = new ShouldFetchEventArgs(lnk); ShouldFetch?.Invoke(this, args); if (args.Cancel) { return(null); } qAdded.Enqueue(lnk); return(lnk); }
private void fetchRewrite_AutoRewrite(object Sender, FetchRewriteEventArgs args) { try { if (!Configuration.Auto_RewriteRemoveFragment) { return; } if (args.CurrentUri.HasFragment()) { args.NewUri = args.CurrentUri.RemoveFragment(); args.ShowOnLog = false; } } catch (Exception ex) { Configuration.Auto_RewriteRemoveFragment = false; log.Error(ex, "Failed while auto-removing fragments. Auto-removing disabled"); } }
private void fetchRewrite_AutoRewrite(object Sender, FetchRewriteEventArgs args) { try { if (!Configuration.Auto_RewriteRemoveFragment) { return; } if (args.CurrentUri.HasFragment()) { args.NewUri = args.CurrentUri.RemoveFragment(); args.ShowOnLog = false; } } catch (Exception ex) { Configuration.Auto_RewriteRemoveFragment = false; log.Error(ex, "Failed while auto-removing fragments. Auto-removing disabled"); OnError?.Invoke(this, new ErrorEventArgs() { Source = FetchEventArgs.EventSource.Scheduler, Exception = ex }); } }
private Link addPage(Uri pageToVisit, Uri sourcePage) { if (!Configuration.SpiderAllowHostViolation && isHostViolation(pageToVisit, BaseUri)) { lock (hViolated) { if (!hViolated.Contains(pageToVisit.Host)) // ignore the entire domain { hViolated.Add(pageToVisit.Host); log.Warning($"[WRN] Host Violation {pageToVisit}"); } } return(null); } var lnk = new Link(pageToVisit, sourcePage); try { if (AddPageFilter != null && !AddPageFilter(lnk)) { return(null); } } catch (Exception ex) { log.Error(ex, "AddPageFilter error"); OnError?.Invoke(this, new ErrorEventArgs() { Source = FetchEventArgs.EventSource.Scheduler, Exception = ex }); } if (FetchRewrite != null) { var ev = new FetchRewriteEventArgs(pageToVisit); FetchRewrite(this, ev); // Default Uri Equality ignores Fragment if (ev.NewUri != null && ev.NewUri.ToString() != pageToVisit.ToString()) { if (ev.ShowOnLog) { log.Information($"[REW] {pageToVisit} -> {ev.NewUri}"); } lnk.ResourceRewritten(ev.NewUri); } } if (SpiderWorkData.Moved301.ContainsKey(lnk.Uri.ToString())) { string newUri = SpiderWorkData.Moved301[lnk.Uri.ToString()]; lnk.ResourceMoved(new Uri(newUri)); if (alreadyExecuted(lnk.Uri)) { return(null); } } if (alreadyExecuted(lnk.Uri)) { return(null); } try { var args = new ShouldFetchEventArgs(lnk); ShouldFetch?.Invoke(this, args); if (args.Cancel) { return(null); } } catch (Exception ex) { log.Error(ex, "ShouldFetch error"); OnError?.Invoke(this, new ErrorEventArgs() { Source = FetchEventArgs.EventSource.Scheduler, Exception = ex }); } qAdded.Enqueue(lnk); return(lnk); }