protected (string html, List <string> linkList) GetMultiResultBySelector(string link, string selector,
string selectorAttributeForValue, SiteResultType resultType, bool resultLinksAreRelative, bool useProxy)
{
IRequester requester = new WebRenderer();
var retryCount = 0;
string html = null;
while (retryCount < MaxRetry)
{
WebProxy proxy = null;
if (useProxy)
{
proxy = Proxier.GetProxy();
}
Logger.Log(
proxy == null
? $"Fetch: {resultType} | Retry: {retryCount} | Link: {link}"
: $"Fetch: {resultType} | Proxy: {proxy.Address.Host}:{proxy.Address.Port} | Retry: {retryCount} | Link: {link}",
LogType.Log);
bool isSuccessful;
(html, isSuccessful) = requester.DownloadHtml(link, SitePageType, SiteSignature, proxy);
if (!isSuccessful)
{
retryCount++;
continue;
}
break;
}
if (string.IsNullOrWhiteSpace(html))
{
return(null, null);
}
var doc = new HtmlDocument();
doc.LoadHtml(html);
var linkNodes = doc.DocumentNode.SelectNodes(selector);
if (linkNodes == null)
{
Logger.Log($"Empty {resultType} Nodes: {link}", LogType.Error);
Console.WriteLine(
$@"{SavePath}/Debug/{SiteUri.Host}/{
CurrentTimeStamp
:d-M-yy_HH-mm-ss}/Error/ErrorHtml{resultType}.txt");
DataDumper.Write(doc.DocumentNode.InnerHtml,
$"{SavePath}/Debug/{SiteUri.Host}/{CurrentTimeStamp:d-M-yy_HH-mm-ss}/Error/ErrorHtml{resultType}.txt");
return(html, null);
}
var linkList = new List <string>();
foreach (var linkNode in linkNodes)
{
var relativeLink = linkNode.Attributes[selectorAttributeForValue].Value;
if (string.IsNullOrWhiteSpace(relativeLink))
{
continue;
}
if (resultLinksAreRelative)
{
Logger.Log($"Result {resultType} Link: {BaseUrl + relativeLink}", LogType.Log);
linkList.Add(BaseUrl + relativeLink);
}
else
{
Logger.Log($"Result {resultType} Link: {relativeLink}", LogType.Log);
linkList.Add(relativeLink);
}
}
return(html, linkList);
}
