private async void Worker()
{
loader = new HtmlLoader(parserSettings);
for (int i = parserSettings.StartPoint; i <= parserSettings.EndPoint; i++)
{
if (!Program.ParserActive)
{
return;
}
var source = await loader.GetSourceByPageId(i);
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
Parser.Parse(document);
lock (locker)
OnParseOneLink?.Invoke();
}
if (Program.ParserActive)
{
ParseComplete?.Invoke();
}
}
private static Task RunParsePage(int taskId, int lineIndex)
{
return(Task.Run(async() =>
{
if (!Program.ParserActive)
{
return;
}
var items = lines[lineIndex].Split(' ');
var client = new HttpClient();
var response = await client.GetAsync("https://www.g2g.com/" + items[3]);
string source = null;
if (response != null && response.StatusCode == HttpStatusCode.OK)
{
source = await response.Content.ReadAsStringAsync();
}
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var count = GetAnnouncementCount(document);
if (count != 0)
{
if (count > 20)
{
int requestsCount = count / 20;
while (true)
{
int index = -1;
while (!CefSourceLoader.FreeBrowsers.TryDequeue(out index))
{
await Task.Delay(100);
}
if (index != -1)
{
document = await GetCefDocument(items[3], requestsCount, count, index);
break;
}
}
}
new ParserGamesG2G().Parse(document, SetSpaces(items[0]), SetSpaces(items[1]), SetSpaces(items[2]), count);
}
lock (locker)
CreateNewTask(taskId, countOfParsedLines++ + beginTasksCount);
OnParseOneLink?.Invoke();
//linesList.Remove(lines[lineIndex]);
}));
}
