private async void Worker()
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
try
{
var source = await loader.GetSourceByPageId();
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
IsNetworkError = false;
}
catch
{
IsNetworkError = true;
}
OnCompleted?.Invoke(this);
isActive = false;
}
private async void Worker()
{
string sourse = null;
try
{
sourse = await loader.GetSourceByPageId();
if (String.IsNullOrEmpty(sourse))
{
throw new Exception();
}
}
catch
{
OnError?.Invoke(this);
return;
}
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(sourse);
var result = parser.Parser(document);
OnNewData?.Invoke(this, result);
}
private async void Worker()
{
for (int i = parserSettings.StartPart; i <= parserSettings.EndPart; i++)
{
for (int j = parserSettings.StartArticle; j <= parserSettings.EndArticle; j++)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
var source = await loader.GetSourceByPage(i, j);
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
}
OnCompleted?.Invoke(this);
isActive = false;
}
//Do all of the dirty work
private async void Worker()
{
for (int i = parserSettings.StartPoint; i <= parserSettings.EndPoint; i++)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
//loads the source code of the current web-page
var source = await loader.GetSourceByPageId(i);
//AngleSharp parser to parse source code to IHtmlDocument object
var domParser = new HtmlParser();
//gets IHtmlDocument object
var document = await domParser.ParseAsync(source);
//Parse needed data with help of out Parser
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
OnCompleted?.Invoke(this);
isActive = false;
}
private async void Worker()
{
for (int i = parserSettings.StartPoint; i <= parserSettings.EndPoint; i++)
{
if (!isActive)
{
OnComleted?.Invoke(this);
return;
}
HabrParserForm.idPage = i.ToString();
var source = await loader.GetSoutseByPageId(i);
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
OnComleted?.Invoke(this);
isActive = false;
}
//закрытый асинхр метод, который контролирует процесс парсинга
private async void Worker()
{
//прогоняем цикл от стартовой до конечной настроек
for (int i = parserSetings.StartPoint; i <= parserSetings.EndPoint; i++)
{
if (!isActive)
{
//вызов события информирует о конце работы парсера
OnCompleted?.Invoke(this);
return;
}
//получаем исходный код страници с индексом из цикла
var source = await loader.GetSourceByPageId(i);
var domParser = new HtmlParser();
//спарсим асинхронно код и получим док с которым можно работать
var document = await domParser.ParseAsync(source);
//передаём док в парсер и записываем в переменную
var result = parser.Parse(document);
//вызов события которрое передаёт ссылку и результат
OnNewData?.Invoke(this, result);
}
//вызов события информирует о конце работы парсера
OnCompleted?.Invoke(this);
isActive = false;
}
void m_Protocol_OnPacketReceived(byte Packet, byte[] arr_bArguments)
{
if (OnNewData != null)
{
OnNewData.Invoke(this, Packet, arr_bArguments);
}
}
private async void Worker()
{
int pageNum = 1;
while (true)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
var source = await loader.GetSourceByPage(pageNum.ToString());
pageNum = pageNum + 20;
var domParser = new HtmlParser();
var document = domParser.ParseDocument(source);
var result = parser.Parse(document);
if (result == null)
{
break;
}
else
{
OnNewData?.Invoke(this, result);
}
}
OnCompleted?.Invoke(this);
isActive = false;
}
public async void Start()
{
HtmlLoader htmlLoader = new HtmlLoader(Settings);
var html = await htmlLoader.GetSoursePage();
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
var countPage = htmlDocument.DocumentNode.Descendants("span").Where(count => count.GetAttributeValue("id", "").Equals("total_pages")).FirstOrDefault().InnerText;
for (int i = parserSettings.Page; i < Convert.ToInt32(countPage); i++)
{
if (!IsWorked)
{
OnComplated?.Invoke(this);
return;
}
html = await htmlLoader.GetSoursePage();
htmlDocument.LoadHtml(html);
var items = parser.ParserData(htmlDocument);
OnNewData?.Invoke(this, items);
}
OnComplated?.Invoke(this);
IsWorked = false;
}
private async void Worker()
{
while (isActive)
{
var source = await loader.GetSourceByPageId(id); // скачиваем html код страницы
// в виде строки
var domParser = new HtmlParser();
var result = source as T; // приводим строку к T
// (в данном случае это необязательно)
OnNewData?.Invoke(this, result); // вызываем событие OnNewData
for (int i = 0; i < 60; i++) // задержка
{
if (!isActive)
{
break;
}
await Task.Delay(500);
}
}
OnCompleted?.Invoke(this); // вызываем событие OnCompleted
}
protected override void WndProc(ref Message m)
{
switch (m.Msg)
{
case WM_DRAWCLIPBOARD:
if (HandleNewItem)
{
OnNewData?.Invoke(Clipboard.GetDataObject());
}
SendMessage(nextViewer, m.Msg, m.WParam, m.LParam);
break;
case WM_CHANGECBCHAIN:
if (m.WParam == nextViewer)
{
nextViewer = m.LParam;
}
else
{
SendMessage(nextViewer, m.Msg, m.WParam, m.LParam);
}
break;
}
base.WndProc(ref m);
}
private async void Worker()
{
for (int i = collectorSettings.StartPoint; i <= collectorSettings.EndPoint; i++)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
var source = await loader.GetSourceByPageIdAsync(i);
var domParser = new HtmlParser();
var document = await domParser.ParseAsync(source);
var result = collector.Collect(document);
//todo: crutch. async bug must be fixed
OnNewData?.Invoke(this, result);
Thread.Sleep(10000);
}
OnCompleted?.Invoke(this);
isActive = false;
}
/// <summary>
/// Загружает и анализирует Html
/// </summary>
/// <param name="link"></param>
/// <param name="pattern"></param>
/// <returns></returns>
public bool HandleHtml(string link, string pattern)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(link);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
bool isSucceseded = false;
string data = "";
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
data = readStream.ReadToEnd();
response.Close();
readStream.Close();
isSucceseded = true;
OnNewData.Invoke(link + " downloaded");
HtmlAnalysis(link, data, pattern);
}
else
{
OnNewData.Invoke(link + " fail to download");
}
return(isSucceseded);
}
//Работа парсера
private async void Worker()
{
int pageValue = 1, i = 0;
while (i++ < pageValue)
{
if (!isActive) //Если решили остановить парсер Abort()
{
OnCompleted?.Invoke(this);
return;
}
var source = await loader.GetSourceByPageId(i); // {CurrentId}=i для реализации перехода по страницам
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document); //Парсинг необходимых данных
if (i == 1) //Парсим количество страниц
{
var page = document.QuerySelector(".paginator:nth-last-of-type(2)");
if (page != null)
{
pageValue = int.Parse(page.GetAttribute("title"));
}
}
OnNewData?.Invoke(this, result, i, pageValue);
}
OnCompleted?.Invoke(this);
isActive = false;
}
private async void Worker()
{
for (int i = parserSettings.StartPoint; i <= parserSettings.EndPoint; i++)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
var source = await HTMLoader.GetSourceByPageId(i);
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(ExpandFullPage());
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
OnCompleted?.Invoke(this);
isActive = false;
}
public async Task <string> Worker(CodeContext context)
{
isActive = true;
for (int i = parserSettings.StartPoint; i <= parserSettings.EndPoint; i++)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return(null);
}
var source = await loader.GetSourceByPageId(i);
var domParser = new HtmlParser();
var document = await domParser.ParseAsync(source);
var result = parser.Parse(document);
//Database.SetInitializer(new Init());
foreach (Product attendee in parser.ProductsList)
{
context.Products.Add(attendee);
}
foreach (Price attendee in parser.PricesList)
{
context.Prices.Add(attendee);
}
try
{
context.SaveChanges();
}
catch (System.Data.Entity.Validation.DbEntityValidationException dbEx)
{
Exception raise = dbEx;
foreach (var validationErrors in dbEx.EntityValidationErrors)
{
foreach (var validationError in validationErrors.ValidationErrors)
{
string message = string.Format("{0}:{1}",
validationErrors.Entry.Entity.ToString(),
validationError.ErrorMessage);
// raise a new exception nesting
// the current instance as InnerException
raise = new InvalidOperationException(message, raise);
}
}
throw raise;
}
OnNewData?.Invoke(this, result);
}
OnCompleted?.Invoke(this);
isActive = false;
return(await loader.GetSourceByPageId(1));
}
private async void Worker()
{
for (int i = 0; i < CurUrl.iter; i++)
{
if (i == 0)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
var loader = new HtmlLoader();
var source = await loader.GetHtmlByUrl(CurUrl.url);
var domParser = new HtmlParser();
var document = await domParser.ParseAsync(source);
var result = parser.Parse(document);
foreach (var item in CurUrl.changes)
{
CurUrl.urls.Add(item);
}
OnNewData?.Invoke(this, result);
}
else
{
foreach (var url in CurUrl.urls)
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
var loader = new HtmlLoader();
var source = await loader.GetHtmlByUrl(url.Replace("https", "http"));
var domParser = new HtmlParser();
var document = await domParser.ParseAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
foreach (var item in CurUrl.changes)
{
CurUrl.urls.Add(item);
}
}
}
OnCompleted?.Invoke(this);
isActive = false;
}
public async void Worker(string name, string ChatId)
{
var source = await loader.GetSourceByPageName(name);
var domParser = new HtmlParser();
var document = await domParser.ParseAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result, ChatId);
}
public async Task UpdatePartAsync(string league)
{
var source = await loader.GetSourceByUrl(league);
var domParser = new HtmlParser();
var document = await domParser.ParseAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(result, league);
}
public async Task <bool> MainCompWorker()
{
dbChecker = new DbChecker();
int max = await dbChecker.GetMaxCountDB("category");
LinkToCompanyParser parser = new LinkToCompanyParser();
curentValueProgress = 0;
ActivateVisibilityMonitor();
for (int i = 1; i <= max; i++)
{
if (!isActive)
{
return(true);
}
(string link, int categoryId) = await dbChecker.GetLinkAndIdByKey(i, "category");
if (link == "0" || link == "")
{
continue;
}
for (int j = 1; j <= 100; j++)
{
curentValueProgress++;
prepareProcessUpdate((max - 7) * 100);
if (!isActive)
{
return(true);
}
var result = await parser.Parse(j, link, i);
if (result == null)
{
curentValueProgress += (100 - j);
prepareProcessUpdate((max - 7) * 100);
break;
}
ShowTotalParsed();
OnNewData?.Invoke(this, result);
Console.WriteLine($"Цикл страницы {j} завершен");
}
}
dbChecker.PutIntProcessToDb("step_2", 0, true);
dbChecker.PutIntProcessToDb("internal-string-time-elapsed", 0, false);
isActive = false;
FinishProcessMonitoring();
return(true);
}
private async void Worker()
{
var source = await loader.GetSource();
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
public async void StartOnce()
{
HtmlLoader htmlLoader = new HtmlLoader(Settings);
var html = await htmlLoader.GetSoursePage();
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
var result = parser.ParserData(htmlDocument);
OnNewData?.Invoke(this, result);
}
private async void Worker()
{
if (!IsActive)
{
OnCompleted?.Invoke(this);
return;
}
OnNewData?.Invoke(this, Parser.Parse(await new HtmlParser().ParseAsync(await loader.GetSourseBySearch(parserSettings.Search, parserSettings.CurrentId))));
OnCompleted?.Invoke(this);
IsActive = false;
}
private async void Worker()
{
for (int i = parserSettings.StartPoint; i <= parserSettings.EndPoint; i++)
{
var source = await loader.GetSourceByPageId(i);
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document);
OnNewData?.Invoke(this, result);
}
}
public async Task <bool> Worker(string link)
{
CategoryParser categoryParser = new CategoryParser();
SubCategoryParser subCatParser = new SubCategoryParser();
var result = await categoryParser.Parse(1, $"{link}");
var subresult = await subCatParser.Parse(1, $"{link}");
OnNewData?.Invoke(this, result);
OnNewData?.Invoke(this, subresult);
dbChecker.PutIntProcessToDb("step_1", 0, true);
isActive = false;
return(true);
}
private async void ExportCpu()
{
if (!isActive)
{
return;
}
var source = await loader.Load();
var result = parser.Parse(source);
OnNewData?.Invoke(this, result);
OnComplete?.Invoke(this);
isActive = false;
}
private async Task ParseHtml(string htmlDock)
{
try
{
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(htmlDock);
var result = _parser.Parse(document);
OnNewData?.Invoke(this, result);
}
catch (Exception e)
{
_logger.Error(e, "Error deal with html parsing");
}
}
//Контроль процесса парсинга
private void Worker()
{
if (!isActive)
{
OnCompleted?.Invoke(this);
return;
}
//Присвоение списку объектов ItemModel новых значений, полученных из rss-ленты
items = Loader();
OnNewData?.Invoke(this, items);
OnCompleted?.Invoke(this);
}
// По видосу посмотри что да как
private async void Worker()
{
var source = await loader.GetSourcePage();
var domParser = new HtmlParser();
var document = await domParser.ParseDocumentAsync(source);
var result = parser.Parse(document);
List <T> temp = new List <T>();
temp.Add(result);
OnNewData?.Invoke(this, temp);
OnCompleted?.Invoke(this);
}
private void ParserResultMessageHandler(ParserResultMessage <string[]> message)
{
if (message.Result != null)
{
if (message.Result.Length > 1 && message.Result[0].Equals("captcha"))
{
OnCaptcha?.Invoke(message.Result);
return;
}
foreach (var item in message.Result)
{
OnNewData?.Invoke(item);
}
}
}
