private IEnumerable <SiteNode> GetAllSiteNodes(string rootUri, int depthLevel, TransitionToOtherDomainsLimits transactionLimits) { while (depthLevel >= 0) { depthLevel--; var rootNode = SiteNodeHelper.GetFilledSiteNode(rootUri, _httpResponseProvider); OnSiteNodeFounded(this, rootNode.Uri); yield return(rootNode); var links = SiteNodeHelper.GetSiteNodeLinks(_rootUri, rootNode, transactionLimits, _htmlCrawler); foreach (var link in links) { var absoluteLink = UriHelper.GetAbsoluteLink(link, rootNode.Uri); var linkNodes = GetAllSiteNodes(absoluteLink, depthLevel, transactionLimits); foreach (var node in linkNodes) { NLogger.Logger.Info($"Node found: uri {node.Uri}"); yield return(node); } } } }
private void WriteSiteNodesToFileSystem(IEnumerable <SiteNode> siteNodes, string outPath) { FileSystemHelper.CreateDirectory(outPath); var totalNodesCount = siteNodes.Count(); var nodeNumber = 1; foreach (var node in siteNodes) { try { var uriParts = SiteNodeHelper.GetSiteNodeUriParts(node); var writePath = SiteNodeHelper.CreateAndGetWritePathForAUri(uriParts, outPath); SiteNodeHelper.WriteSiteNodeToFileSystem(node, writePath); OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(node.Uri, nodeNumber, totalNodesCount)); } catch (InvalidOperationException exc) { OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(exc.Message, nodeNumber, totalNodesCount)); } finally { nodeNumber++; } } }
private void WriteSiteNodesToFileSystem(IEnumerable <SiteNode> siteNodes, string outPath) { NLogger.Logger.Info("Successfully writing site nodes to file system..."); FileSystemHelper.CreateDirectory(outPath); var totalNodesCount = siteNodes.Count(); var nodeNumber = 1; foreach (var node in siteNodes) { try { var uriParts = SiteNodeHelper.GetSiteNodeUriParts(node); var writePath = SiteNodeHelper.CreateAndGetWritePathForAUri(uriParts, outPath); SiteNodeHelper.WriteSiteNodeToFileSystem(node, writePath); OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(node.Uri, nodeNumber, totalNodesCount)); NLogger.Logger.Info($"Copied a site node with uri {node.Uri}"); } catch (InvalidOperationException exc) { NLogger.Logger.Error($"Copying a site node with uri {node.Uri} finished with exception: {exc.Message}"); OnSiteNodeCopiedToFileSystem(this, new SiteNodeCopiedToFileSystemEventArgs(exc.Message, nodeNumber, totalNodesCount)); } finally { nodeNumber++; } } NLogger.Logger.Info("Successfully finished writing site nodes to file system."); }