private IDocument HandleBinaryPage(BinaryPage page, WebConnectorJobConfiguration config, PageState pageState,
StringBuilder information)
{
if (!pageState.PageIsChanged(page))
{
information.AppendLine($"{page.Id} (No change detected, skipping)");
return(new IgnoreDocument(page.Id, config.JobName, "No change detected, skipping"));
}
Log.Information($"Adding BinaryPage {page.Id}");
_documentStateService.UpdatePageState(page, config.JobName);
information.AppendLine(pageState == null
? $"{page.Id} (Adding new binary page"
: $"{page.Id} (Adding updated binary page");
return(_pageService.ConvertToDocument(page, config.JobName));
}
private IDocument HandleWebpage(WebPage page, WebConnectorJobConfiguration config, PageState pageState, StringBuilder information)
{
if (!pageState.PageIsChanged(page))
{
information.AppendLine($"{page.Id} (No change detected, skipping)");
return(new IgnoreDocument(page.Id, config.JobName, "No change detected, skipping"));
}
var newLinkCount = HandleLinks(page, config);
Log.Information($"Adding WebPage {page.Id}");
_documentStateService.UpdatePageState(page, config.JobName);
var linkInfo = page.Depth >= config.Depth
? $"and ignoring links since the depth is {page.Depth}"
: $"with {newLinkCount} new links (with depth {page.Depth + 1})";
information.AppendLine(pageState == null
? $"{page.Id} (Adding new html page {linkInfo}"
: $"{page.Id} (Adding updated html page {linkInfo}");
return(_pageService.ConvertToDocument(page, config.JobName));
}
