예제 #1
0
        private async Task CraulRootAsync(CraulResult result, int level)
        {
            if (level > depth)
            {
                return;
            }
            foreach (var rootUrl in result.Childs)
            {
                string htmlPage = "";
                IEnumerable <string> newRootUrls = new List <string>();
                WebClient            webClient   = new WebClient();
                try
                {
                    htmlPage = await webClient.DownloadStringTaskAsync(rootUrl.Url);

                    newRootUrls = await GetRootUrlsAsync(htmlPage);

                    AddChildsToResult(rootUrl, newRootUrls);
                }
                catch (Exception e)
                {
                    errorList.Add(e);
                }
                await CraulRootAsync(rootUrl, level + 1);
            }
        }
예제 #2
0
 private void AddChildsToResult(CraulResult result, IEnumerable <string> rootUrls)
 {
     foreach (string url in rootUrls)
     {
         if (url.StartsWith("http"))
         {
             result.Childs.Add(new CraulResult(url));
         }
     }
 }
예제 #3
0
 public Task <CraulResult> PerformCraulingAsync(IEnumerable <string> rootUrls)
 {
     return(Task <CraulResult> .Run(async() =>
     {
         CraulResult result = new CraulResult("root");
         AddChildsToResult(result, rootUrls);
         await CraulRootAsync(result, 0);
         if (errorList.Count != 0)
         {
             EncounteredErrors = new AggregateException(errorList);
         }
         return result;
     }
                                    ));
 }