public void Delete(ExecutingInfoDTO info)
{
if (info == null)
{
return;
}
var ExInfo = uOW.ExecuteRepo.GetByID(info.Id);
if (ExInfo == null)
{
return;
}
uOW.ExecuteRepo.Delete(ExInfo);
uOW.Save();
}
public void Update(ExecutingInfoDTO info)
{
if (info == null)
{
return;
}
var ExInfo = uOW.ExecuteRepo.GetByID(info.Id);
if (ExInfo == null)
{
return;
}
ExInfo.Status = info.Status;
ExInfo.ParserTask = uOW.ParserRepo.GetByID(info.ParserTaskId);
ExInfo.GoodUrl = info.GoodUrl;
ExInfo.ErrorMessage = info.ErrorMessage;
uOW.Save();
}
public int Insert(ExecutingInfoDTO info)
{
lock (uOW)
{
if (info == null)
{
return(-1);
}
//var ExInfo = Mapper.Map<ExecutingInfo>(info);
var TaskToAdd = new ExecutingInfo()
{
Date = info.Date,
GoodUrl = info.GoodUrl,
ParserTaskId = info.ParserTaskId,
Status = info.Status
};
uOW.ExecuteRepo.Insert(TaskToAdd);
uOW.Save();
return(TaskToAdd.Id);
}
}
/// <summary>
/// Parses input url by configuration from parser task
/// </summary>
/// <param name="parsertaskid">id of parser task</param>
/// <param name="url">url to parse</param>
/// <returns>New parsed GoodDTO</returns>
public GoodDTO ExecuteTask(int parsertaskid, string url)
{
//clearing previous logs
//if (!isStarted)
// taskinfoManager.DeleteByStatus(ExecuteStatus.Executing);
//else
// isStarted = true;
//downloading page source using tor+phantomjs
ParserTaskDTO parsertask = parsermanager.Get(parsertaskid);
HtmlDocument doc = null;
//adding to local log storage
ExecutingInfoDTO taskinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.Executing,
Date = DateTime.Now,
ParserTaskId = parsertaskid
};
taskinfo.Id = taskinfoManager.Insert(taskinfo);
//getting page source due to method
string pageSource = "";
try
{
SiteDownloader sw = new SiteDownloader();
switch (parsertask.IteratorSettings.DownloadMethod)
{
case DownloadMethod.Direct:
pageSource = sw.GetPageSouceDirectly(url);
break;
case DownloadMethod.Tor:
pageSource = sw.GetPageSouce(url);
break;
default:
break;
}
//pageSource = htmlValidator.CheckHtml(pageSource);
doc = new HtmlDocument();
doc.LoadHtml(pageSource);
}
catch (Exception ex)
{
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't download url"
};
taskinfoManager.Insert(errorinfo);
taskinfoManager.Delete(taskinfo);
return(null);
}
//gets configuration from parsertask id
GrabberSettingsDTO grabbersettings = parsertask.GrabberSettings;
GoodDTO resultGood = new GoodDTO();
resultGood.WebShop_Id = parsertask.WebShopId;
resultGood.Category_Id = parsertask.CategoryId;
///////////////////////////////////Parcing name by list of xpathes
var xpathbuffer = "";
try
{
var name = "";
foreach (var nameprop in grabbersettings.Name)
{
xpathbuffer = nameprop;
HtmlNode value = doc.DocumentNode.SelectSingleNode(nameprop);
if (value != null)
{
name = value.InnerHtml;
break;
}
}
name = name.Trim();
resultGood.Name = StripHTML(name);
}
catch (Exception ex)
{
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't parse name,-xpath: " + xpathbuffer
};
taskinfoManager.Insert(errorinfo);
}
/////////////////////////////////////Parcing price by list of xpathes
try
{
var price = "";
foreach (var priceprop in grabbersettings.Price)
{
xpathbuffer = priceprop;
HtmlNode value = doc.DocumentNode.SelectSingleNode(priceprop);
if (value != null)
{
price = value.InnerHtml;
break;
}
}
if (price != "")
{
resultGood.Price = Convert.ToDecimal(this.RemoveAllLetters(price));
}
}
catch (Exception ex)
{
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't parse main price,-xpath: " + xpathbuffer
};
taskinfoManager.Insert(errorinfo);
}
//////////////////////////////////////Parcing old price by list of xpathes
try
{
var oldPrice = "";
foreach (var price in grabbersettings.OldPrice)
{
xpathbuffer = price;
HtmlNode value = doc.DocumentNode.SelectNodes(price).FirstOrDefault();
if (value != null)
{
oldPrice = value.InnerHtml;
break;
}
}
if (oldPrice != "")
{
resultGood.OldPrice = Convert.ToDecimal(this.RemoveAllLetters(oldPrice));
}
}
catch (Exception ex)
{
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't parse old price,-xpath: " + xpathbuffer
};
taskinfoManager.Insert(errorinfo);
}
//////////////////////////////Parcing image link by list of xpathes
try
{
var imagelink = "";
foreach (var imglink in grabbersettings.ImgLink)
{
xpathbuffer = imglink;
HtmlNode value = doc.DocumentNode.SelectNodes(imglink + "/@src").FirstOrDefault();
if (value != null)
{
imagelink = value.Attributes["src"].Value;
resultGood.ImgLink = imagelink;
break;
}
if (imagelink == "" || imagelink == null)
{
resultGood.ImgLink = @"http://www.kalahandi.info/wp-content/uploads/2016/05/sorry-image-not-available.png";
}
else
{
resultGood.ImgLink = imagelink;
}
if (resultGood.ImgLink == null)
{
resultGood.ImgLink = @"http://www.kalahandi.info/wp-content/uploads/2016/05/sorry-image-not-available.png";
}
}
}
catch (Exception ex)
{
resultGood.ImgLink = @"http://www.kalahandi.info/wp-content/uploads/2016/05/sorry-image-not-available.png";
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't parse image link,-xpath: " + xpathbuffer
};
taskinfoManager.Insert(errorinfo);
}
resultGood.UrlLink = url;
PropertyValuesDTO propertyValues = new PropertyValuesDTO();
propertyValues.DictDoubleProperties = new Dictionary <int, double>();
propertyValues.DictIntProperties = new Dictionary <int, int>();
propertyValues.DictStringProperties = new Dictionary <int, string>();
foreach (var propitem in grabbersettings.PropertyItems)
{
HtmlNode value = null;
PropertyDTO property = propmanager.Get(propitem.Id);
var htmlvalue = "";
try
{
foreach (var item in propitem.Value)
{
xpathbuffer = item;
value = doc.DocumentNode.SelectNodes(item).FirstOrDefault();
if (value != null)
{
htmlvalue = value.InnerHtml;
break;
}
}
}
catch (Exception ex)
{
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't parse property" + property.Name + ",-xpath: " + xpathbuffer
};
taskinfoManager.Insert(errorinfo);
}
try
{
switch (property.Type)
{
case PropertyType.Integer:
propertyValues.DictIntProperties.Add(propitem.Id, Convert.ToInt32(htmlvalue));
break;
case PropertyType.Double:
propertyValues.DictDoubleProperties.Add(propitem.Id, Convert.ToDouble(htmlvalue));
break;
case PropertyType.String:
propertyValues.DictStringProperties.Add(propitem.Id, StripHTML(htmlvalue));
break;
default:
break;
}
}
catch (Exception ex)
{
logger.Error(ex);
ExecutingInfoDTO errorinfo = new ExecutingInfoDTO()
{
GoodUrl = url,
Status = ExecuteStatus.ErrorInsert,
Date = DateTime.Now,
ParserTaskId = parsertaskid,
ErrorMessage = "Can't convert value " + htmlvalue + " of " + property.Name + ",-xpath: " + xpathbuffer
};
taskinfoManager.Insert(errorinfo);
}
}
resultGood.Status = true;
resultGood.PropertyValues = propertyValues;
goodwizardManager.InsertOrUpdate(resultGood);
//goodManager.Insert(resultGood);
var newPrice = new PriceHistoryDTO();
newPrice.Url = resultGood.UrlLink;
newPrice.Price = resultGood.Price;
newPrice.Date = DateTime.Now;
newPrice.Name = resultGood.Name;
priceManager.Insert(newPrice);
//deleting from local log storage
taskinfoManager.Delete(taskinfo);
return(resultGood);
}
