public object Query(FreeTask tsk, string recordId)
{
SnapSearchResult result = null;
result = new SnapSearchResult();
result = GetLinks(result, tsk, recordId);
return(result);
}
void Snapshot(FreeTask task, string conn, string recordId)
{
int itemCount = 0;
List <XListing> xListings = new List <XListing>();
// TaoBaoSnapshotQuery tbq = new TaoBaoSnapshotQuery();
TaoBaoQuery tbq = new TaoBaoQuery(task.TaskName);
SnapSearchResult result = null;
result = tbq.Query(task, recordId) as SnapSearchResult;
xListings = result.Listings;
//int pageSize = 2000;
//xListings = xListings.OrderByDescending(x => x.ItemSold30Days).ToList();
//for (int page = 0; page * pageSize < xListings.Count; page++)
//{
// SaveResult(xListings.Skip(page * pageSize).Take(pageSize).ToList(), conn, BotTypes.ItemSnapshot);
//}
// SaveKeyRecord(task, xListings);
}
SnapSearchResult GetLinks(SnapSearchResult result, FreeTask tsk, string recordId)
{
List <XListing> xListings = new List <XListing>();
string link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default".FormatStr(tsk.TaskName);
// string link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default".FormatStr("连衣裙冬");
int nohist_pages = 0;
int quried_pages = 1;
int Position = 1;
//最多搜索20页
while (!string.IsNullOrEmpty(link) && quried_pages <= 20)
{
log(link);
var html = TaobaoWebHelper.GetSnapshotHtml(link);;
try
{
if (html != null)
{
var tagslist = html.SubAfter("itemlist").SubBefore("recommendAuctions");
var tags = tagslist.SubAfter("p4pTags").SplitWith("p4pTags");
if (tags == null || tags.Length == 0)
{
log("BLOCKED " + tsk);
break;
}
bool nohit = true;
foreach (var tag in tags)
{
try
{
if (!tag.Contains("raw_title"))
{
Console.WriteLine(DateTime.Now);
break;
}
if (tag == null || tag == "" || tag.Trim().Length == 0)
{
Console.WriteLine(DateTime.Now);
break;
}
var a = tag.SubAfter("raw_title").SubBefore("pic_url");
string title = RemoveChar(a.GetLower());
string nid = RemoveChar(tag.SubAfter("nid").SubBefore("category"));
string pic_url = RemoveChar(tag.SubAfter("pic_url").SubBefore("detail_url"));
string detail_url = "https:" + RemoveChar(tag.SubAfter("detail_url").SubBefore("view_price"));
detail_url = Regex.Unescape(detail_url);
var view_price = RemoveChar(tag.SubAfter("view_price").SubBefore("view_fee"));
double price = 0;
if (!string.IsNullOrEmpty(view_price))
{
price = Convert.ToDouble(view_price);
}
string item_loc = RemoveChar(tag.SubAfter("item_loc").SubBefore("reserve_price"));
string view_sales = RemoveChar(tag.SubAfter("view_sales").SubBefore("comment_count")).Replace("人收货", "").Replace("人付款", "");
int days30 = 0;
if (!string.IsNullOrEmpty(view_sales))
{
days30 = Convert.ToInt32(view_sales);
}
string comment_count = RemoveChar(tag.SubAfter("comment_count").SubBefore("user_id"));
int commentcount = 0;
if (!string.IsNullOrEmpty(comment_count))
{
commentcount = Convert.ToInt32(comment_count);
}
//shop
string user_id = RemoveChar(tag.SubAfter("user_id").SubBefore("nick"));
string nick = RemoveChar(tag.SubAfter("nick").SubBefore("shopcard"));
string isTmall = RemoveChar(tag.SubAfter("isTmall").SubBefore("delivery"));
string delivery = RemoveChar(tag.SubAfter("delivery").SubBefore("description"));
string description = RemoveChar(tag.SubAfter("description").SubBefore("service"));
string sellerCredit = RemoveChar(tag.SubAfter("sellerCredit").SubBefore("totalRate"));
string siteName = "taobao";
Guid siteId = Guid.Parse("A00A672B-DD05-65FB-4EE0-CFA26EBF2ED5");
var totalRate = RemoveChar(tag.SubAfter("totalRate").SubBefore("icon").GetLower());
var shopLink = RemoveChar(tag.SubAfter("shopLink").SubBefore("}"));
shopLink = "https:" + shopLink;
shopLink = Regex.Unescape(shopLink);
XListing listing = new XListing
{
ShopContactUrl = shopLink,
ItemDetailUrl = detail_url,
ItemPrice = price,
ItemName = title,
ItemID = nid,
ItemLocation = item_loc,
ItemSold30Days = days30,
Itempic = pic_url,
ItemTotalCommentCount = commentcount,
UId = tsk.UId,
ShopID = user_id,
ShopName = nick,
ShopLocation = item_loc,
ShopIsTmall = isTmall == "true" ? true : false,
taskid = tsk._id,
taskName = tsk.TaskName,
SiteName = siteName,
SiteID = siteId,
usrid = tsk.UsrId,
ShopIsAuthorized = false,
Position = Position,
PageNum = quried_pages,
ProjectId = tsk.ProjectId
};
if (listing.ItemDetailUrl != null && listing.ItemName != null)
{
listing.ItemBotStatus = BotStatus.Ok;
}
result.Listings.Add(listing);
xListings.Add(listing);
nohit = false;
nohist_pages = 0;
Position++;
}
catch (Exception we)
{
Console.WriteLine(DateTime.Now + "错误：" + we.Message);
break;
}
}
if (nohit)
{
nohist_pages++;
}
//如果连续3页都没有结果，就跳出
if (nohist_pages > 3)
{
break;
}
quried_pages++;
pages++;
NextPage = NextPage + 44;
// link = "https://s.taobao.com/search?q={0}&sort=sale-desc&s={1}".FormatStr(tsk.TaskName, NextPage);
//
link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default&s={1}".FormatStr(tsk.TaskName, NextPage);
Console.WriteLine(DateTime.Now + "任务名：" + tsk.TaskName + "；开始搜索第" + quried_pages + "页");
SaveResult(xListings, BotTypes.ItemSnapshot, recordId, tsk);
xListings.Clear();
int n = new Random().Next(3000, 6000);
Thread.Sleep(n);
}
else
{
return(result);
}
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + "错误：" + ex.Message);
break;
}
}
return(result);
}