public object Query(FreeTask tsk, string recordId)
{
SnapSearchResult result = null;
result = new SnapSearchResult();
result = GetLinks(result, tsk, recordId);
return(result);
}
void Snapshot(FreeTask task, string conn, string recordId)
{
int itemCount = 0;
List <XListing> xListings = new List <XListing>();
// TaoBaoSnapshotQuery tbq = new TaoBaoSnapshotQuery();
TaoBaoQuery tbq = new TaoBaoQuery(task.TaskName);
SnapSearchResult result = null;
result = tbq.Query(task, recordId) as SnapSearchResult;
xListings = result.Listings;
//int pageSize = 2000;
//xListings = xListings.OrderByDescending(x => x.ItemSold30Days).ToList();
//for (int page = 0; page * pageSize < xListings.Count; page++)
//{
// SaveResult(xListings.Skip(page * pageSize).Take(pageSize).ToList(), conn, BotTypes.ItemSnapshot);
//}
// SaveKeyRecord(task, xListings);
}
public void Run()
{
while (true)
{
BotTaskService bt = new BotTaskService();
Random r = new Random();
Thread.Sleep(2000);
FreeTask keyTask = bt.GetBotTask(BotTypes.ItemSnapshot); //get_task();
if (keyTask == null || keyTask.TaskName == "" || keyTask.TaskName == null)
{
SetReady();
log("No search task !");
Thread.Sleep(1000);
var sssH = DateTime.Now.Hour;
if (sssH == 5)
{
var miuH = DateTime.Now.Minute;
if (miuH < 3)
{
var queryTask = new QueryDocument {
{ "BotIntervalHours", 24 }, { "IsStarted", true }
};
var colH = MongoDBHelper.Instance.Get_FreeTask();
var resultH = colH.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList();
foreach (var item in resultH)
{
var update1 = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", false }
} }
};
var Hresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", item._id }
}, update1);
}
}
}
var Zhou = DateTime.Now.DayOfWeek.ToString();
if (Zhou == "Sunday")
{
var Zsss = DateTime.Now.Hour;
if (Zsss == 5)
{
var miuZ = DateTime.Now.Minute;
if (miuZ < 3)
{
var queryTask = new QueryDocument {
{ "BotIntervalHours", 168 }, { "IsStarted", true }
};
var colZ = MongoDBHelper.Instance.Get_FreeTask();
var resultZ = colZ.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList();
foreach (var item in resultZ)
{
var update2 = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", false }
} }
};
var Zresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", item._id }
}, update2);
}
}
}
}
var Tian = DateTime.Now.Day;
if (Tian == 30)
{
var Tsss = DateTime.Now.Hour;
if (Tsss == 5)
{
var miuT = DateTime.Now.Minute;
if (miuT < 3)
{
var queryTask = new QueryDocument {
{ "BotIntervalHours", 720 }, { "IsStarted", true }
};
var colT = MongoDBHelper.Instance.Get_FreeTask();
var resultT = colT.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList();
foreach (var item in resultT)
{
var update3 = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", false }
} }
};
var Tresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", item._id }
}, update3);
}
}
}
}
DateTime start = Convert.ToDateTime(DateTime.Now.AddDays(0).AddHours(8).ToShortDateString());
DateTime end = Convert.ToDateTime(DateTime.Now.AddDays(1).AddHours(8));
continue;
}
SetBusy();
//string sql = @"update taskparameters set LastBotStartAt ='{0}',IsBot=1 where Id={1}".FormatStr(DateTime.Now, keyTask.Id);
//MySqlDbHelper.ExecuteSql(com, sql);
int?LinksNum = 0;
LinksNum = keyTask.recordNum + 1;
var update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", true }, { "LastBotStartAt", DateTime.Now.AddHours(8) },
{ "recordNum", LinksNum }
} }
};
// var result = MongoDBHelper<FreeTask>.Update("FreeTask", new QueryDocument { { "_id", keyTask._id } }, update);
var result = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", keyTask._id }
}, update);
List <FreeTaskRecord> list = new List <FreeTaskRecord>();
FreeTaskRecord ftr = new FreeTaskRecord();
ftr._id = new ObjectId();
ftr.IsStarted = keyTask.IsStarted;
ftr.IsBot = true;
ftr.Taskid = keyTask._id;
ftr.TaskName = keyTask.TaskName;
ftr.UsrId = keyTask.UsrId;
ftr.ServiceState = "Normal";
ftr.CreatedAt = DateTime.Now.AddHours(8);
ftr.LastBotStartAt = DateTime.Now.AddHours(8);
// ftr.LastBotEndAt = DateTime.Now.AddHours(8);
ftr.Dataquantity = 0;
ftr.LinksNum = 0;
ftr.ShopsNum = 0;
ftr.SiteId = Guid.Parse("A00A672B-DD05-65FB-4EE0-CFA26EBF2ED5");
ftr.SiteName = "taobao";
ftr.LanIP = "";//TaobaoWebHelper.GetLanIP();
ftr.UId = keyTask.UId;
ftr.ProjectId = keyTask.ProjectId;
// var savedq = MongoDBHelper<FreeTaskRecord>.InsertData(ftr, "FreeTaskRecord");
list.Add(ftr);
var col = MongoDBHelper.Instance.Get_FreeTaskRecord();
col.InsertMany(list);
string recordId = ftr._id.ToString();
Snapshot(keyTask, com, recordId);
try
{
// update = new UpdateDocument { { "$set", new QueryDocument { { "LastBotEndAt", DateTime.Now.AddHours(8) } } } };
// result = MongoDBHelper<FreeTask>.Update("FreeTask", new QueryDocument { { "_id", keyTask._id } }, update);
update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LastBotEndAt", DateTime.Now.AddHours(8) }
} }
};
result = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", keyTask._id }
}, update);
update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LastBotEndAt", DateTime.Now.AddHours(8) }
} }
};
// result = MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", new QueryDocument { { "_id", new ObjectId(recordId)} }, update);
result = MongoDBHelper.Instance.Get_FreeTaskRecord().UpdateOne(new QueryDocument {
{ "_id", new ObjectId(recordId) }
}, update);
}
catch (Exception ex)
{
log("get_proj_to_qry ERROR ." + ex.Message);
Thread.Sleep(5000);
}
//}
}
}
SnapSearchResult GetLinks(SnapSearchResult result, FreeTask tsk, string recordId)
{
List <XListing> xListings = new List <XListing>();
string link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default".FormatStr(tsk.TaskName);
// string link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default".FormatStr("连衣裙冬");
int nohist_pages = 0;
int quried_pages = 1;
int Position = 1;
//最多搜索20页
while (!string.IsNullOrEmpty(link) && quried_pages <= 20)
{
log(link);
var html = TaobaoWebHelper.GetSnapshotHtml(link);;
try
{
if (html != null)
{
var tagslist = html.SubAfter("itemlist").SubBefore("recommendAuctions");
var tags = tagslist.SubAfter("p4pTags").SplitWith("p4pTags");
if (tags == null || tags.Length == 0)
{
log("BLOCKED " + tsk);
break;
}
bool nohit = true;
foreach (var tag in tags)
{
try
{
if (!tag.Contains("raw_title"))
{
Console.WriteLine(DateTime.Now);
break;
}
if (tag == null || tag == "" || tag.Trim().Length == 0)
{
Console.WriteLine(DateTime.Now);
break;
}
var a = tag.SubAfter("raw_title").SubBefore("pic_url");
string title = RemoveChar(a.GetLower());
string nid = RemoveChar(tag.SubAfter("nid").SubBefore("category"));
string pic_url = RemoveChar(tag.SubAfter("pic_url").SubBefore("detail_url"));
string detail_url = "https:" + RemoveChar(tag.SubAfter("detail_url").SubBefore("view_price"));
detail_url = Regex.Unescape(detail_url);
var view_price = RemoveChar(tag.SubAfter("view_price").SubBefore("view_fee"));
double price = 0;
if (!string.IsNullOrEmpty(view_price))
{
price = Convert.ToDouble(view_price);
}
string item_loc = RemoveChar(tag.SubAfter("item_loc").SubBefore("reserve_price"));
string view_sales = RemoveChar(tag.SubAfter("view_sales").SubBefore("comment_count")).Replace("人收货", "").Replace("人付款", "");
int days30 = 0;
if (!string.IsNullOrEmpty(view_sales))
{
days30 = Convert.ToInt32(view_sales);
}
string comment_count = RemoveChar(tag.SubAfter("comment_count").SubBefore("user_id"));
int commentcount = 0;
if (!string.IsNullOrEmpty(comment_count))
{
commentcount = Convert.ToInt32(comment_count);
}
//shop
string user_id = RemoveChar(tag.SubAfter("user_id").SubBefore("nick"));
string nick = RemoveChar(tag.SubAfter("nick").SubBefore("shopcard"));
string isTmall = RemoveChar(tag.SubAfter("isTmall").SubBefore("delivery"));
string delivery = RemoveChar(tag.SubAfter("delivery").SubBefore("description"));
string description = RemoveChar(tag.SubAfter("description").SubBefore("service"));
string sellerCredit = RemoveChar(tag.SubAfter("sellerCredit").SubBefore("totalRate"));
string siteName = "taobao";
Guid siteId = Guid.Parse("A00A672B-DD05-65FB-4EE0-CFA26EBF2ED5");
var totalRate = RemoveChar(tag.SubAfter("totalRate").SubBefore("icon").GetLower());
var shopLink = RemoveChar(tag.SubAfter("shopLink").SubBefore("}"));
shopLink = "https:" + shopLink;
shopLink = Regex.Unescape(shopLink);
XListing listing = new XListing
{
ShopContactUrl = shopLink,
ItemDetailUrl = detail_url,
ItemPrice = price,
ItemName = title,
ItemID = nid,
ItemLocation = item_loc,
ItemSold30Days = days30,
Itempic = pic_url,
ItemTotalCommentCount = commentcount,
UId = tsk.UId,
ShopID = user_id,
ShopName = nick,
ShopLocation = item_loc,
ShopIsTmall = isTmall == "true" ? true : false,
taskid = tsk._id,
taskName = tsk.TaskName,
SiteName = siteName,
SiteID = siteId,
usrid = tsk.UsrId,
ShopIsAuthorized = false,
Position = Position,
PageNum = quried_pages,
ProjectId = tsk.ProjectId
};
if (listing.ItemDetailUrl != null && listing.ItemName != null)
{
listing.ItemBotStatus = BotStatus.Ok;
}
result.Listings.Add(listing);
xListings.Add(listing);
nohit = false;
nohist_pages = 0;
Position++;
}
catch (Exception we)
{
Console.WriteLine(DateTime.Now + "错误：" + we.Message);
break;
}
}
if (nohit)
{
nohist_pages++;
}
//如果连续3页都没有结果，就跳出
if (nohist_pages > 3)
{
break;
}
quried_pages++;
pages++;
NextPage = NextPage + 44;
// link = "https://s.taobao.com/search?q={0}&sort=sale-desc&s={1}".FormatStr(tsk.TaskName, NextPage);
//
link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default&s={1}".FormatStr(tsk.TaskName, NextPage);
Console.WriteLine(DateTime.Now + "任务名：" + tsk.TaskName + "；开始搜索第" + quried_pages + "页");
SaveResult(xListings, BotTypes.ItemSnapshot, recordId, tsk);
xListings.Clear();
int n = new Random().Next(3000, 6000);
Thread.Sleep(n);
}
else
{
return(result);
}
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + "错误：" + ex.Message);
break;
}
}
return(result);
}
private void update_level1_links(List <XListing> listings, BotTypes botType, string recordId, FreeTask tsk)
{
if (listings == null || listings.Count == 0)
{
log("SUCCESS update 0 apps Links for " + tsk.TaskName);
return;
}
var itemdt = GetItemList(listings);
for (int i = 0; i < itemdt.Count; i++)
{
var update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "PageNum", itemdt[i].PageNum }, { "TotalComments", itemdt[i].TotalComments }, { "Price", itemdt[i].Price },
{ "Recent30DaysSoldNum", itemdt[i].Recent30DaysSoldNum }, { "Position", itemdt[i].Position }
} }
};
var result = MongoDBHelper.Instance.Get_FreeBotItem().UpdateOne(new QueryDocument {
{ "ItemId", itemdt[i].ItemId }
}, update);
}
}
void SaveResult(List <XListing> listings, BotTypes botType, string recordId, FreeTask tsk)
{
listings.ToList().ForEach(x =>
{
if (string.IsNullOrEmpty(x.ShopName) && !string.IsNullOrEmpty(x.SiteName) && !x.BotShopID.HasValue &&
!x.SiteName.ToLower().Contains("taobao") && !x.SiteName.ToLower().Contains("alibaba"))
{
x.ShopName = x.SiteName;
x.BotShopID = x.SiteID;
}
if (!x.BotShopID.HasValue && !string.IsNullOrEmpty(x.ShopName))
{
x.BotShopID = IDHelper.GetGuid(string.Format("{0},{1},{2},{3}", x.ShopName, x.SiteName, tsk._id, x.ShopID));
}
});
var shopList = listings;
shopList = shopList.DistinctBy(x => x.BotShopID);
//var exists_ids = MySqlDbHelper.GetExsitsIds<Guid?>(com, "bot_shops", "Shop_id", shopList.Select(x => x.BotShopID).ToArray());
//if (exists_ids != null && exists_ids.Count > 0)
//{
// shopList = shopList.Where(x => !exists_ids.Contains(x.BotShopID)).ToList();
//}
FieldsDocument shopfd = new FieldsDocument();
shopfd.Add("BotShopID", 1);
// MongoCollection<Guid> shopcol = MongoDBHelper<Guid>.GetMongoDB().GetCollection<Guid>("FreeBotShop");
var shopcol = MongoDBHelper.Instance.Get_FreeBotShop();
var builder = Builders <FreeBotShop> .Filter;
List <Guid?> BotShopID = shopList.Select(x => x.BotShopID).ToList();
// var existsshop_objs = shopcol.Find(MongoDB.Driver.Builders.Query.In("BotShopID", new BsonArray(BotShopID))).SetFields(shopfd);
var existsshop_objs = shopcol.Find(builder.In(x => x.Shop_id, BotShopID)).Project(x => x.Shop_id).ToList();
List <Guid?> exists_ids = new List <Guid?>();
foreach (var result in existsshop_objs)
{
exists_ids.Add(result);
}
if (exists_ids != null && exists_ids.Count > 0)
{
shopList = shopList.Where(x => !exists_ids.Contains(x.BotShopID)).ToList();
}
if (shopList == null || shopList.Count == 0)
{
return;
}
List <FreeBotShop> dt = null;
if (shopList.Count > 0)
{
dt = GetShopList(shopList);
// var saved = MongoDBHelper<FreeBotShop>.BatchInsertData(dt, "FreeBotShop");
shopcol.InsertMany(dt);
log("to save bot_shops");
log("Done");
}
listings.ToList().ForEach(x =>
{
if (!x.BotItemID.HasValue && !string.IsNullOrEmpty(x.ItemName))
{
x.BotItemID = IDHelper.GetGuid(string.Format("{0},{1},{2},{3}", x.ItemName, x.SiteName, tsk._id, x.ItemID));
}
});
var itemList = listings;
itemList = itemList.DistinctBy(x => x.BotItemID);
//var exists_itemids = MySqlDbHelper.GetExsitsIds<Guid?>(com, "bot_items", "ItemId", itemList.Select(x => x.BotItemID).ToArray());
//if (exists_itemids != null && exists_itemids.Count > 0)
//{
// itemList = itemList.Where(x => !exists_itemids.Contains(x.BotItemID)).ToList();
//}
FieldsDocument fd = new FieldsDocument();
fd.Add("BotItemID", 1);
// MongoCollection<Guid> col = MongoDBHelper<Guid>.GetMongoDB().GetCollection<Guid>("FreeBotItem");
var col = MongoDBHelper.Instance.Get_FreeBotItem();
var itemBuilder = Builders <FreeBotItem> .Filter;
List <Guid?> BotItemID = itemList.Select(x => x.BotItemID).ToList();
// var exists_objs = col.Find(MongoDB.Driver.Builders.Query.In("BotItemID", new BsonArray(BotItemID))).SetFields(fd);
var exists_objs = col.Find(itemBuilder.In(x => x.ItemId, BotItemID)).Project(x => x.ItemId).ToList();
List <Guid?> existsitem_ids = new List <Guid?>();
foreach (var result in exists_objs)
{
existsitem_ids.Add(result);
}
if (existsitem_ids != null && existsitem_ids.Count > 0)
{
itemList = itemList.Where(x => !existsitem_ids.Contains(x.BotItemID)).ToList();
}
List <XListing> updatelinks = new List <XListing>();
if (existsitem_ids != null && existsitem_ids.Count > 0)
{
updatelinks = updatelinks.Where(x => existsitem_ids.Contains(x.BotItemID)).ToList();
}
update_level1_links(updatelinks, botType, recordId, tsk);
if (itemList == null || itemList.Count == 0)
{
return;
}
var itemdt = GetItemList(itemList);
// var savedListings = MySqlDbHelper.BatchInsert(con, "bot_items", itemdt);
// var savedListings = MongoDBHelper<FreeBotItem>.BatchInsertData(itemdt, "FreeBotItem");
col.InsertMany(itemdt);
// var wequery = new QueryDocument { { "_id", new ObjectId(recordId) } };
// FreeTaskRecord TaskList = MongoDBHelper<FreeTaskRecord>.Find1("FreeTaskRecord", wequery);
var colRecord = MongoDBHelper.Instance.Get_FreeTaskRecord();
var RecordBuilder = Builders <FreeTaskRecord> .Filter;
FreeTaskRecord TaskList = colRecord.Find(RecordBuilder.Eq(x => x._id, new ObjectId(recordId))).FirstOrDefault();
int LinksNum = 0;
int ShopsNum = 0;
LinksNum = TaskList.LinksNum + itemdt.Count;
ShopsNum = TaskList.ShopsNum + dt.Count;
var updateWebsiteCount = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LinksNum", LinksNum }, { "ShopsNum", ShopsNum }
} }
};
// MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", wequery, updateWebsiteCount);
MongoDBHelper.Instance.Get_FreeTaskRecord().UpdateOne(new QueryDocument {
{ "_id", new ObjectId(recordId) }
}, updateWebsiteCount);
var colTask = MongoDBHelper.Instance.Get_FreeTask();
var TaskBuilder = Builders <FreeTask> .Filter;
FreeTask Task2List = colTask.Find(TaskBuilder.Eq(x => x._id, itemdt[0].taskId)).FirstOrDefault();
int TaskLinksNum = 0;
int TaskShopsNum = 0;
TaskLinksNum = Task2List.LinksNum + itemdt.Count;
TaskShopsNum = Task2List.ShopsNum + dt.Count;
var TaskupdateWebsiteCount = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LinksNum", TaskLinksNum }, { "ShopsNum", TaskShopsNum }
} }
};
// MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", wequery, updateWebsiteCount);
MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", itemdt[0].taskId }
}, TaskupdateWebsiteCount);
log("to save listings");
log("Done");
}