private IEnumerable <SiteNode> GetAllSiteNodes(string rootUri, int depthLevel, TransitionToOtherDomainsLimits transactionLimits)
{
while (depthLevel >= 0)
{
depthLevel--;
var rootNode = SiteNodeHelper.GetFilledSiteNode(rootUri, _httpResponseProvider);
OnSiteNodeFounded(this, rootNode.Uri);
yield return(rootNode);
var links = SiteNodeHelper.GetSiteNodeLinks(_rootUri, rootNode, transactionLimits, _htmlCrawler);
foreach (var link in links)
{
var absoluteLink = UriHelper.GetAbsoluteLink(link, rootNode.Uri);
var linkNodes = GetAllSiteNodes(absoluteLink, depthLevel, transactionLimits);
foreach (var node in linkNodes)
{
NLogger.Logger.Info($"Node found: uri {node.Uri}");
yield return(node);
}
}
}
}
private void WriteSiteNodesToFileSystem(IEnumerable <SiteNode> siteNodes, string outPath)
{
FileSystemHelper.CreateDirectory(outPath);
var totalNodesCount = siteNodes.Count();
var nodeNumber = 1;
foreach (var node in siteNodes)
{
try
{
var uriParts = SiteNodeHelper.GetSiteNodeUriParts(node);
var writePath = SiteNodeHelper.CreateAndGetWritePathForAUri(uriParts, outPath);
SiteNodeHelper.WriteSiteNodeToFileSystem(node, writePath);
OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(node.Uri, nodeNumber, totalNodesCount));
}
catch (InvalidOperationException exc)
{
OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(exc.Message, nodeNumber, totalNodesCount));
}
finally
{
nodeNumber++;
}
}
}
private void WriteSiteNodesToFileSystem(IEnumerable <SiteNode> siteNodes, string outPath)
{
NLogger.Logger.Info("Successfully writing site nodes to file system...");
FileSystemHelper.CreateDirectory(outPath);
var totalNodesCount = siteNodes.Count();
var nodeNumber = 1;
foreach (var node in siteNodes)
{
try
{
var uriParts = SiteNodeHelper.GetSiteNodeUriParts(node);
var writePath = SiteNodeHelper.CreateAndGetWritePathForAUri(uriParts, outPath);
SiteNodeHelper.WriteSiteNodeToFileSystem(node, writePath);
OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(node.Uri, nodeNumber, totalNodesCount));
NLogger.Logger.Info($"Copied a site node with uri {node.Uri}");
}
catch (InvalidOperationException exc)
{
NLogger.Logger.Error($"Copying a site node with uri {node.Uri} finished with exception: {exc.Message}");
OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(exc.Message, nodeNumber, totalNodesCount));
}
finally
{
nodeNumber++;
}
}
NLogger.Logger.Info("Successfully finished writing site nodes to file system.");
}
