public override Task <ICrawlerResultSingle <LashinbangItem> > ParseDetail(string data, string id)
{
var doc = new HtmlDocument();
doc.LoadHtml(data);
var output = new CrawlerResultBase <LashinbangItem>();
try
{
var item = new LashinbangItem();
item.Id = id;
item.InternalId = $"lashin_{item.Id}";
item.ImageUrl = doc.FirstOfDescendantsWithId("img", "zoom_03").Attributes["data-zoom-image"].Value;
item.Name = WebUtility.HtmlDecode(doc.FirstOfDescendantsWithClass("div", "item_name").Descendants("h1")
.First().InnerText.Trim());
item.Price = float.Parse(doc.FirstOfDescendantsWithClass("div", "price red").InnerText.Split('円').First()
.Replace(",", "").Trim());
output.Success = true;
output.Result = item;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse item detail ({id}).");
}
return(Task.FromResult((ICrawlerResultSingle <LashinbangItem>)output));
}
protected override Task <ICrawlerResultList <YahooItem> > Parse(string data, YahooSourceParameters parameters)
{
var root = JsonConvert.DeserializeObject <RootObject>(data);
var parsedItems = new List <YahooItem>();
var output = new CrawlerResultBase <YahooItem>
{
Results = parsedItems,
};
try
{
foreach (var rootItem in root.Items)
{
if (IsItemExcluded(rootItem.Title, parameters))
{
continue;
}
YahooItem.ItemCondition condition = YahooItem.ItemCondition.Unknown;
if (rootItem.Condition == "中古")
{
condition = YahooItem.ItemCondition.Used;
}
else if (rootItem.Condition == "新品")
{
condition = YahooItem.ItemCondition.New;
}
var item = new YahooItem
{
Id = rootItem.Id,
InternalId = $"yahoo_{rootItem.Id}",
ImageUrl = rootItem.ImageUrl,
Name = rootItem.Title,
Price = rootItem.Price,
BuyoutPrice = rootItem.BuyItNowPrice,
EndTime = new DateTimeOffset(DateTime.Parse(rootItem.EndTime), TimeSpan.FromHours(9))
.ToUniversalTime().UtcDateTime,
IsShippingFree = rootItem.Postage == 0,
Condition = condition,
Tax = rootItem.Tax,
BidsCount = rootItem.Bids,
};
parsedItems.Add(item);
}
output.Success = true;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse list of items. ({parameters.SearchQuery})");
}
return(Task.FromResult((ICrawlerResultList <YahooItem>)output));
}
protected override Task <ICrawlerResultList <MercariItem> > Parse(string data, MercariSourceParameters parameters)
{
var doc = new HtmlDocument();
doc.LoadHtml(data);
var items = doc.DocumentNode.WhereOfDescendantsWithClass("div", "item");
var parsedItems = new List <MercariItem>();
var output = new CrawlerResultBase <MercariItem>
{
Results = parsedItems,
};
try
{
var itemNodes = doc
.FirstOfDescendantsWithClass("div", "items-box-content clearfix")
.WhereOfDescendantsWithClass("section", "items-box");
foreach (var itemNode in itemNodes)
{
var image = itemNode.Descendants("img").First();
var link = itemNode.Descendants("a").First();
var itemName = WebUtility.HtmlDecode(image.Attributes["alt"].Value.Trim());
if (IsItemExcluded(itemName, parameters))
{
continue;
}
var item = new MercariItem();
var idTemp = link.Attributes["href"].Value;
var pos = idTemp.IndexOf('?');
idTemp = idTemp.Substring(0, pos).Trim('/');
pos = idTemp.LastIndexOf('/');
item.Id = idTemp.Substring(pos + 1);
item.InternalId = $"mercari_{item.Id}";
item.Name = itemName;
item.Price = float.Parse(itemNode.FirstOfDescendantsWithClass("div", "items-box-price font-5")
.InnerText.Replace("¥", "").Replace(",", "").Trim());
item.ImageUrl = image.Attributes["data-src"].Value;
parsedItems.Add(item);
}
output.Success = true;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse list of items. ({parameters.SearchQuery})");
}
return(Task.FromResult((ICrawlerResultList <MercariItem>)output));
}
protected override Task <ICrawlerResultList <MandarakeItem> > Parse(string data, MandarakeSourceParameters parameters)
{
var doc = new HtmlDocument();
doc.LoadHtml(data);
var items = doc.DocumentNode.FirstOfDescendantsWithClass("div", "entry").WhereOfDescendantsWithClass("div", "block");
var parsedItems = new List <MandarakeItem>();
var output = new CrawlerResultBase <MandarakeItem>
{
Results = parsedItems,
};
try
{
foreach (var itemNode in items)
{
var itemName = WebUtility.HtmlDecode(itemNode.FirstOfDescendantsWithClass("div", "title").InnerText.Trim());
if (IsItemExcluded(itemName, parameters))
{
continue;
}
var item = new MandarakeItem();
var id = itemNode.FirstOfDescendantsWithClass("a", "addbasket").Attributes["data-index"].Value.Trim();
item.Id = id;
item.Name = itemName;
item.Price = float.Parse(itemNode.FirstOfDescendantsWithClass("div", "price").InnerText.Replace("円+税", "")
.Replace(",", ""));
item.ImageUrl = itemNode.Descendants("img").Last().Attributes["src"].Value;
item.InternalId = $"mandarake_{item.Id}";
item.Shop =
WebUtility.HtmlDecode(itemNode.FirstOfDescendantsWithClass("p", "shop").InnerText.Trim());
parsedItems.Add(item);
}
output.Success = true;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse list of items. ({parameters.SearchQuery})");
}
return(Task.FromResult((ICrawlerResultList <MandarakeItem>)output));
}
public override Task <ICrawlerResultSingle <MandarakeItem> > ParseDetail(string data, string id)
{
var doc = new HtmlDocument();
doc.LoadHtml(data);
var output = new CrawlerResultBase <MandarakeItem>();
try
{
var item = new MandarakeItem();
item.Id = id;
item.InternalId = $"mandarake_{item.Id}";
item.Shop = WebUtility.HtmlDecode(doc.FirstOfDescendantsWithClass("div", "shop").InnerText.Trim());
item.Name = WebUtility.HtmlDecode(doc.FirstOfDescendantsWithClass("div", "subject").InnerText.Trim());
if (doc.WhereOfDescendantsWithClass("a", "addalert").Any())
{
item.Price = CrawlerConstants.InvalidPrice;
}
else
{
item.Price = float.Parse(doc.FirstOfDescendantsWithClass("p", "__price").InnerText.Split('円')
.First()
.Replace(",", ""));
}
item.ImageUrl = doc.FirstOfDescendantsWithClass("img", "xzoom").Attributes["xoriginal"].Value;
item.ImageUrl = item.ImageUrl.Insert(item.ImageUrl.LastIndexOf('/') + 1, "s_");
output.Success = true;
output.Result = item;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse item detail ({id}).");
}
return(Task.FromResult((ICrawlerResultSingle <MandarakeItem>)output));
}
protected override Task <ICrawlerResultList <LashinbangItem> > Parse(string data,
LashinbangSourceParameters parameters)
{
var root = JsonConvert.DeserializeObject <RootObject>(data.Substring(9).Trim(';', ')'));
var parsedItems = new List <LashinbangItem>();
var output = new CrawlerResultBase <LashinbangItem>
{
Results = parsedItems,
};
try
{
foreach (var resultItem in root.Kotohaco.Result.Items)
{
if (IsItemExcluded(resultItem.Title, parameters))
{
continue;
}
var item = new LashinbangItem
{
Id = resultItem.Itemid,
InternalId = $"lashin_{resultItem.Itemid}",
Name = resultItem.Title,
Price = resultItem.Price,
ImageUrl = resultItem.Image
};
parsedItems.Add(item);
}
output.Success = true;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse list of items. ({parameters.SearchQuery})");
}
return(Task.FromResult((ICrawlerResultList <LashinbangItem>)output));
}
public override Task <ICrawlerResultSingle <MercariItem> > ParseDetail(string data, string id)
{
var doc = new HtmlDocument();
doc.LoadHtml(data);
var output = new CrawlerResultBase <MercariItem>();
try
{
var image = doc.FirstOfDescendantsWithClass("img", "owl-lazy");
var item = new MercariItem();
item.Id = id;
item.InternalId = $"mercari_{item.Id}";
item.Name = WebUtility.HtmlDecode(image.Attributes["alt"].Value.Trim());
item.ImageUrl = image.Attributes["data-src"].Value;
if (data.Contains("売り切れました"))
{
item.Price = CrawlerConstants.InvalidPrice;
}
else
{
item.Price = float.Parse(doc.FirstOfDescendantsWithClass("span", "item-price bold")
.InnerText.Replace("¥", "").Replace(",", "").Trim());
}
output.Success = true;
output.Result = item;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse item detail ({id}).");
}
return(Task.FromResult((ICrawlerResultSingle <MercariItem>)output));
}
public override Task <ICrawlerResultSingle <YahooItem> > ParseDetail(string data, string id)
{
var doc = new HtmlDocument();
doc.LoadHtml(data);
var output = new CrawlerResultBase <YahooItem>();
try
{
var image = doc.FirstOfDescendantsWithClass("p", "fjd_img img").Descendants("img").First();
var item = new YahooItem();
item.Id = id;
item.InternalId = $"yahoo_{item.Id}";
item.ImageUrl = image.Attributes["src"].Value;
item.Name = WebUtility.HtmlDecode(image.Attributes["alt"].Value);
if (data.Contains("Sorry: Auction of item URL or Auction ID that you filled in has been closed."))
{
item.Price = CrawlerConstants.InvalidPrice;
}
else
{
var container = doc.FirstOfDescendantsWithId("div", "dtl").WhereOfDescendantsWithClass("span", "num")
.ToList();
item.Price = float.Parse(container[0].InnerText.Replace(",", "").Trim());
if (container.Count == 2)
{
item.BuyoutPrice = float.Parse(container[1].InnerText.Replace(",", "").Trim());
}
}
var detailsGrid = doc.WhereOfDescendantsWithClass("div", "wr").First(node => node.InnerHtml.StartsWith("<table><tbody><tr><th"));
var datesRow = detailsGrid.Descendants("tr").First(node => node.InnerText.Contains("End time"));
item.EndTime =
new DateTimeOffset(
DateTime.Parse(WebUtility.HtmlDecode(datesRow.Descendants("td").Last().InnerText
.Replace("(Japan Time)", "").Trim())), TimeSpan.FromHours(9))
.ToUniversalTime().UtcDateTime;
var conditionRow = detailsGrid.Descendants("tr").First(node => node.InnerText.Contains("condition"));
var condition = conditionRow.Descendants("td").First().InnerText.Trim();
if (condition == "New")
{
item.Condition = YahooItem.ItemCondition.New;
}
else if (condition == "Used")
{
item.Condition = YahooItem.ItemCondition.Used;
}
var bidsRow = detailsGrid.Descendants("tr").First(node => node.InnerText.Contains("Current bids"));
var bids = bidsRow.Descendants("td").Last().InnerText.Trim();
item.BidsCount = int.Parse(bids);
output.Success = true;
output.Result = item;
}
catch (Exception e)
{
_logger.LogError(e, $"Failed to parse item detail ({id}).");
}
return(Task.FromResult((ICrawlerResultSingle <YahooItem>)output));
}