/// <summary>
///
/// </summary>
/// <param name="htmlDocument"></param>
/// <param name="actionAdExtracted"></param>
/// <returns></returns>
public Models.Advertisement.Advertisements ExtractAds(string htmlDocument, Action <Handlers.EventHandlers.AdExtractedEventArgs> actionAdExtracted)
{
Models.Advertisement.Advertisements _ads = null;
try
{
HtmlAgilityPack.HtmlDocument _doc = new HtmlAgilityPack.HtmlDocument();
_doc.LoadHtml(htmlDocument);
var _N = HtmlUtil.GetNode(_doc.DocumentNode, "ul", "id", "searchResultListings");
HtmlAgilityPack.HtmlNode[] _nodes = _N.Descendants("li")
.Where(li => li.Attributes.Contains("class") &&
li.Attributes[@"class"].Value.Contains("listingContainer"))
.ToArray();
if (_nodes != null && _nodes.Count() > 0)
{
_ads = new Models.Advertisement.Advertisements();
int _pgItemIdx = 0;
foreach (HtmlAgilityPack.HtmlNode _n in _nodes)
{
Models.Advertisement _ad = new Models.Advertisement();
_pgItemIdx++;
Stopwatch _stopwatch = new Stopwatch();
_stopwatch.Reset();
_stopwatch.Start();
_ad = this.EscrapeAdInfo(_n);
if (_ad != null)
{
_ad = this.EscrapeAdInfoExtend(_ad);
}
_stopwatch.Stop();
_ads.Add(_ad);
if (actionAdExtracted != null)
{
actionAdExtracted(new Handlers.EventHandlers.AdExtractedEventArgs(_ad, _stopwatch.Elapsed, _pgItemIdx));
}
}
}
}
catch (Exception ex)
{
if (this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "ExtractAds"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
return(_ads);
}
/// <summary>
///
/// </summary>
/// <param name="htmlDocument"></param>
/// <param name="actionAdExtracted"></param>
/// <returns></returns>
public Models.Advertisement.Advertisements ExtractAds(string htmlDocument, Action <Handlers.EventHandlers.AdExtractedEventArgs> actionAdExtracted)
{
Models.Advertisement.Advertisements _ads = null;
try
{
HtmlAgilityPack.HtmlDocument _doc = new HtmlAgilityPack.HtmlDocument();
_doc.LoadHtml(htmlDocument);
var _nodes = HtmlUtil.GetNodeCollection(_doc.DocumentNode, "div", "class", "ypgListing clearfix");
if (_nodes != null && _nodes.Count() > 0)
{
_ads = new Models.Advertisement.Advertisements();
int _pgItemIdx = 0;
foreach (HtmlAgilityPack.HtmlNode _n in _nodes)
{
Models.Advertisement _ad = new Models.Advertisement();
_pgItemIdx++;
Stopwatch _stopwatch = new Stopwatch();
_stopwatch.Reset();
_stopwatch.Start();
_ad = this.EscrapeAdInfo(_n);
if (_ad != null)
{
_ad = this.EscrapeAdInfoExtend(_ad);
}
_stopwatch.Stop();
_ads.Add(_ad);
if (actionAdExtracted != null)
{
actionAdExtracted(new Handlers.EventHandlers.AdExtractedEventArgs(_ad, _stopwatch.Elapsed, _pgItemIdx));
}
}
}
}
catch (Exception ex)
{
if (this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "ExtractAds"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
return(_ads);
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public override Models.Advertisement.Advertisements StartSearchResultProcessCallback()
{
Models.Advertisement.Advertisements _ads = null;
if (base.SearchItem.searchResult.pointers != null && base.SearchItem.searchResult.pointers.Count > 0)
{
int _pgNo = 0;
_ads = new Models.Advertisement.Advertisements();
try
{
foreach (Models.SearchItem.SearchResult.UrlPointer _p in base.SearchItem.searchResult.pointers)
{
Thread.Sleep(5000);
_pgNo++;
string _htmlDoc = HtmlUtil.GetPageDocument(_p.SearchUrl);
if (!string.IsNullOrEmpty(_htmlDoc))
{
string _errMsg = string.Empty;
bool _pgInError = this.ypTool.CheckPageIfError(_htmlDoc, ref _errMsg);
_p.IsValid = !_pgInError;
if (_p.IsValid)
{
_p.SearchHtml = _htmlDoc;
Stopwatch _stopwatch = new Stopwatch();
_stopwatch.Reset();
_stopwatch.Start();
Models.Advertisement.Advertisements __ads = this.ypTool.ExtractAds(_htmlDoc, base.InvokeEventAdExtracted);
if (__ads != null && __ads.Count > 0)
{
_ads.AddRange(__ads.ToArray());
}
_stopwatch.Stop();
base.InvokeEventUrlPointerProcessed(new Handlers.EventHandlers.UrlPointerProcessedEventArgs(_p, _stopwatch.Elapsed));
if (base.SearchItem.pagesToProcess.HasValue && _pgNo == base.SearchItem.pagesToProcess)
{
break;
}
if (__ads != null && __ads.Count == base.SearchItem.searchResult.TotalResults)
{
break;
}
}
else
{
//correct Search result values here...
//base.SearchItem.searchResult.ResultsPerPage=base.SearchItem.searchResult.
//Strip the rest forward + all invalid objects
base.SearchItem.searchResult.pointers.RemoveAll(p => !p.IsValid);
}
}
}
}
catch (Exception ex)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.DirectoryProviderSetting.ServicedCountry.ToString(), "StartSearchResultProcessCallback"), ex);
base.InvokeEventFrameworkException(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
return(_ads);
}
/// <summary>
///
/// </summary>
/// <param name="htmlDocument"></param>
/// <param name="actionAdExtracted"></param>
/// <returns></returns>
public Models.Advertisement.Advertisements ExtractAds(string htmlDocument, Action<Handlers.EventHandlers.AdExtractedEventArgs> actionAdExtracted)
{
Models.Advertisement.Advertisements _ads = null;
try
{
HtmlAgilityPack.HtmlDocument _doc = new HtmlAgilityPack.HtmlDocument();
_doc.LoadHtml(htmlDocument);
var _nodes = HtmlUtil.GetNodeCollection(_doc.DocumentNode, "div", "class", "ypgListing clearfix");
if(_nodes != null && _nodes.Count() > 0)
{
_ads = new Models.Advertisement.Advertisements();
int _pgItemIdx = 0;
foreach(HtmlAgilityPack.HtmlNode _n in _nodes)
{
Models.Advertisement _ad = new Models.Advertisement();
_pgItemIdx++;
Stopwatch _stopwatch = new Stopwatch();
_stopwatch.Reset();
_stopwatch.Start();
_ad = this.EscrapeAdInfo(_n);
if(_ad != null)
_ad = this.EscrapeAdInfoExtend(_ad);
_stopwatch.Stop();
_ads.Add(_ad);
if(actionAdExtracted != null)
actionAdExtracted(new Handlers.EventHandlers.AdExtractedEventArgs(_ad, _stopwatch.Elapsed, _pgItemIdx));
}
}
}
catch(Exception ex)
{
if(this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "ExtractAds"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
return _ads;
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public override Models.Advertisement.Advertisements StartSearchResultProcessCallback()
{
Models.Advertisement.Advertisements _ads = null;
if(base.SearchItem.searchResult.pointers != null && base.SearchItem.searchResult.pointers.Count > 0)
{
int _pgNo = 0;
_ads = new Models.Advertisement.Advertisements();
try
{
foreach(Models.SearchItem.SearchResult.UrlPointer _p in base.SearchItem.searchResult.pointers)
{
Thread.Sleep(5000);
_pgNo++;
string _htmlDoc = HtmlUtil.GetPageDocument(_p.SearchUrl);
if(!string.IsNullOrEmpty(_htmlDoc))
{
string _errMsg = string.Empty;
bool _pgInError = this.ypTool.CheckPageIfError(_htmlDoc, ref _errMsg);
_p.IsValid = !_pgInError;
if(_p.IsValid)
{
_p.SearchHtml = _htmlDoc;
Stopwatch _stopwatch = new Stopwatch();
_stopwatch.Reset();
_stopwatch.Start();
Models.Advertisement.Advertisements __ads = this.ypTool.ExtractAds(_htmlDoc, base.InvokeEventAdExtracted);
if(__ads != null && __ads.Count > 0)
_ads.AddRange(__ads.ToArray());
_stopwatch.Stop();
base.InvokeEventUrlPointerProcessed(new Handlers.EventHandlers.UrlPointerProcessedEventArgs(_p, _stopwatch.Elapsed));
if(base.SearchItem.pagesToProcess.HasValue && _pgNo == base.SearchItem.pagesToProcess)
break;
if(__ads != null && __ads.Count == base.SearchItem.searchResult.TotalResults)
break;
}
else
{
//correct Search result values here...
//base.SearchItem.searchResult.ResultsPerPage=base.SearchItem.searchResult.
//Strip the rest forward + all invalid objects
base.SearchItem.searchResult.pointers.RemoveAll(p => !p.IsValid);
}
}
}
}
catch(Exception ex)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.DirectoryProviderSetting.ServicedCountry.ToString(), "StartSearchResultProcessCallback"), ex);
base.InvokeEventFrameworkException(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
return _ads;
}
/// <summary>
///
/// </summary>
/// <param name="htmlDocument"></param>
/// <param name="actionAdExtracted"></param>
/// <returns></returns>
public Models.Advertisement.Advertisements ExtractAds(string htmlDocument, Action<Handlers.EventHandlers.AdExtractedEventArgs> actionAdExtracted)
{
Models.Advertisement.Advertisements _ads = null;
try
{
HtmlAgilityPack.HtmlDocument _doc = new HtmlAgilityPack.HtmlDocument();
_doc.LoadHtml(htmlDocument);
var _N = HtmlUtil.GetNode(_doc.DocumentNode, "ul", "id", "searchResultListings");
HtmlAgilityPack.HtmlNode[] _nodes = _N.Descendants("li")
.Where(li => li.Attributes.Contains("class")
&& li.Attributes[@"class"].Value.Contains("listingContainer"))
.ToArray();
if(_nodes != null && _nodes.Count() > 0)
{
_ads = new Models.Advertisement.Advertisements();
int _pgItemIdx = 0;
foreach(HtmlAgilityPack.HtmlNode _n in _nodes)
{
Models.Advertisement _ad = new Models.Advertisement();
_pgItemIdx++;
Stopwatch _stopwatch = new Stopwatch();
_stopwatch.Reset();
_stopwatch.Start();
_ad = this.EscrapeAdInfo(_n);
if(_ad != null)
_ad = this.EscrapeAdInfoExtend(_ad);
_stopwatch.Stop();
_ads.Add(_ad);
if(actionAdExtracted != null)
actionAdExtracted(new Handlers.EventHandlers.AdExtractedEventArgs(_ad, _stopwatch.Elapsed, _pgItemIdx));
}
}
}
catch(Exception ex)
{
if(this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "ExtractAds"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
return _ads;
}