private IDocument HandleBinaryPage(BinaryPage page, WebConnectorJobConfiguration config, PageState pageState, StringBuilder information) { if (!pageState.PageIsChanged(page)) { information.AppendLine($"{page.Id} (No change detected, skipping)"); return(new IgnoreDocument(page.Id, config.JobName, "No change detected, skipping")); } Log.Information($"Adding BinaryPage {page.Id}"); _documentStateService.UpdatePageState(page, config.JobName); information.AppendLine(pageState == null ? $"{page.Id} (Adding new binary page" : $"{page.Id} (Adding updated binary page"); return(_pageService.ConvertToDocument(page, config.JobName)); }
private IDocument HandleWebpage(WebPage page, WebConnectorJobConfiguration config, PageState pageState, StringBuilder information) { if (!pageState.PageIsChanged(page)) { information.AppendLine($"{page.Id} (No change detected, skipping)"); return(new IgnoreDocument(page.Id, config.JobName, "No change detected, skipping")); } var newLinkCount = HandleLinks(page, config); Log.Information($"Adding WebPage {page.Id}"); _documentStateService.UpdatePageState(page, config.JobName); var linkInfo = page.Depth >= config.Depth ? $"and ignoring links since the depth is {page.Depth}" : $"with {newLinkCount} new links (with depth {page.Depth + 1})"; information.AppendLine(pageState == null ? $"{page.Id} (Adding new html page {linkInfo}" : $"{page.Id} (Adding updated html page {linkInfo}"); return(_pageService.ConvertToDocument(page, config.JobName)); }