public void getUrls(string httpResponseBody)
{
try
{
List <string> urlsFound = CrawlerRegex.regexUrls(httpResponseBody);
//Add to listURL
lock (this.locker)
{
foreach (string url in urlsFound)
{
listURL.Add(url);
}
}
}
catch (NullReferenceException e)
{
Debug.WriteLine("getUrls() geeft NullReferenceException: " + e.Message);
}
}
public void getUrls(string httpResponseBody)
{
try
{
List <string> gevondenUrls = CrawlerRegex.regexUrls(httpResponseBody);
//List<string> gevondenUrls = regexUrls(httpResponseBody);
//toevoegen aan lijstUrls
lock (this.locker)
{
foreach (string url in gevondenUrls)
{
lijstUrls.Add(url);
//Debug.WriteLine("getUrls: " + url);
}
}
}
catch (NullReferenceException e)
{
Debug.WriteLine("getUrls() geeft NullReferenceException: " + e.Message);
}
}
//zoek op userinput
static public async Task <string> crawlZoekterm(string zoekterm)
{
MaakHttpClientAan httpClientRequest = new MaakHttpClientAan();
string httpResponseBody = await httpClientRequest.doeHttpRequestYoutubeMetZoektermEnGeefResults(zoekterm);
//haal de results uit de response
httpResponseBody = CrawlerRegex.regexResults(httpResponseBody);
await Task.Factory.StartNew(async() =>
{
//haal uit results urls
List <string> urls = CrawlerRegex.regexUrls(httpResponseBody);
//ga over de gevonden urls
foreach (String url in urls)
{
//haal uit urls bodys
string body = "";
string antwoord = "";
//getResponseBody url
httpClientRequest = new MaakHttpClientAan();
await Task.Delay(1000);
//welke url crawlen
//Debug.WriteLine("url in getResponseBody() = " + url);
antwoord = await httpClientRequest.doeHttpRequestYoutubeVoorScrawlerEnGeefResults(url);
//haal content uit string
body = CrawlerRegex.regexContent(antwoord);
//haal keywords uit body
CrawlerRegex.regexKeywords(body);
} //gevonden urls gedaan
});
//return string van httpResponseBody
return(httpResponseBody);
}
//Search with user input
static public async Task <string> crawlerSearchterm(string zoekterm)
{
HTTPFactory httpClientRequest = new HTTPFactory();
string httpResponseBody = await httpClientRequest.YoutubeSearchResults(zoekterm);
//Get results from response
httpResponseBody = CrawlerRegex.regexResults(httpResponseBody);
await Task.Factory.StartNew(async() =>
{
//Extract urls from response
List <string> urls = CrawlerRegex.regexUrls(httpResponseBody);
//Iterate on the extracted results
foreach (String url in urls)
{
//Extract body from URL
string body = "";
string response = "";
//getResponseBody url
httpClientRequest = new HTTPFactory();
await Task.Delay(1000);
//Crawl on URL
response = await httpClientRequest.YoutubeCrawlRequest(url);
//Extract content from string
body = CrawlerRegex.regexContent(response);
//Get keywords from body
CrawlerRegex.regexKeywords(body);
} //End code with URLS found
});
//return string from httpResponseBody
return(httpResponseBody);
}