public void Run() { while (true) { BotTaskService bt = new BotTaskService(); Random r = new Random(); Thread.Sleep(2000); FreeTask keyTask = bt.GetBotTask(BotTypes.ItemSnapshot); //get_task(); if (keyTask == null || keyTask.TaskName == "" || keyTask.TaskName == null) { SetReady(); log("No search task !"); Thread.Sleep(1000); var sssH = DateTime.Now.Hour; if (sssH == 5) { var miuH = DateTime.Now.Minute; if (miuH < 3) { var queryTask = new QueryDocument { { "BotIntervalHours", 24 }, { "IsStarted", true } }; var colH = MongoDBHelper.Instance.Get_FreeTask(); var resultH = colH.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList(); foreach (var item in resultH) { var update1 = new UpdateDocument { { "$set", new QueryDocument { { "IsBot", false } } } }; var Hresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument { { "_id", item._id } }, update1); } } } var Zhou = DateTime.Now.DayOfWeek.ToString(); if (Zhou == "Sunday") { var Zsss = DateTime.Now.Hour; if (Zsss == 5) { var miuZ = DateTime.Now.Minute; if (miuZ < 3) { var queryTask = new QueryDocument { { "BotIntervalHours", 168 }, { "IsStarted", true } }; var colZ = MongoDBHelper.Instance.Get_FreeTask(); var resultZ = colZ.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList(); foreach (var item in resultZ) { var update2 = new UpdateDocument { { "$set", new QueryDocument { { "IsBot", false } } } }; var Zresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument { { "_id", item._id } }, update2); } } } } var Tian = DateTime.Now.Day; if (Tian == 30) { var Tsss = DateTime.Now.Hour; if (Tsss == 5) { var miuT = DateTime.Now.Minute; if (miuT < 3) { var queryTask = new QueryDocument { { "BotIntervalHours", 720 }, { "IsStarted", true } }; var colT = MongoDBHelper.Instance.Get_FreeTask(); var resultT = colT.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList(); foreach (var item in resultT) { var update3 = new UpdateDocument { { "$set", new QueryDocument { { "IsBot", false } } } }; var Tresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument { { "_id", item._id } }, update3); } } } } DateTime start = Convert.ToDateTime(DateTime.Now.AddDays(0).AddHours(8).ToShortDateString()); DateTime end = Convert.ToDateTime(DateTime.Now.AddDays(1).AddHours(8)); continue; } SetBusy(); //string sql = @"update taskparameters set LastBotStartAt ='{0}',IsBot=1 where Id={1}".FormatStr(DateTime.Now, keyTask.Id); //MySqlDbHelper.ExecuteSql(com, sql); int?LinksNum = 0; LinksNum = keyTask.recordNum + 1; var update = new UpdateDocument { { "$set", new QueryDocument { { "IsBot", true }, { "LastBotStartAt", DateTime.Now.AddHours(8) }, { "recordNum", LinksNum } } } }; // var result = MongoDBHelper<FreeTask>.Update("FreeTask", new QueryDocument { { "_id", keyTask._id } }, update); var result = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument { { "_id", keyTask._id } }, update); List <FreeTaskRecord> list = new List <FreeTaskRecord>(); FreeTaskRecord ftr = new FreeTaskRecord(); ftr._id = new ObjectId(); ftr.IsStarted = keyTask.IsStarted; ftr.IsBot = true; ftr.Taskid = keyTask._id; ftr.TaskName = keyTask.TaskName; ftr.UsrId = keyTask.UsrId; ftr.ServiceState = "Normal"; ftr.CreatedAt = DateTime.Now.AddHours(8); ftr.LastBotStartAt = DateTime.Now.AddHours(8); // ftr.LastBotEndAt = DateTime.Now.AddHours(8); ftr.Dataquantity = 0; ftr.LinksNum = 0; ftr.ShopsNum = 0; ftr.SiteId = Guid.Parse("A00A672B-DD05-65FB-4EE0-CFA26EBF2ED5"); ftr.SiteName = "taobao"; ftr.LanIP = "";//TaobaoWebHelper.GetLanIP(); ftr.UId = keyTask.UId; ftr.ProjectId = keyTask.ProjectId; // var savedq = MongoDBHelper<FreeTaskRecord>.InsertData(ftr, "FreeTaskRecord"); list.Add(ftr); var col = MongoDBHelper.Instance.Get_FreeTaskRecord(); col.InsertMany(list); string recordId = ftr._id.ToString(); Snapshot(keyTask, com, recordId); try { // update = new UpdateDocument { { "$set", new QueryDocument { { "LastBotEndAt", DateTime.Now.AddHours(8) } } } }; // result = MongoDBHelper<FreeTask>.Update("FreeTask", new QueryDocument { { "_id", keyTask._id } }, update); update = new UpdateDocument { { "$set", new QueryDocument { { "LastBotEndAt", DateTime.Now.AddHours(8) } } } }; result = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument { { "_id", keyTask._id } }, update); update = new UpdateDocument { { "$set", new QueryDocument { { "LastBotEndAt", DateTime.Now.AddHours(8) } } } }; // result = MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", new QueryDocument { { "_id", new ObjectId(recordId)} }, update); result = MongoDBHelper.Instance.Get_FreeTaskRecord().UpdateOne(new QueryDocument { { "_id", new ObjectId(recordId) } }, update); } catch (Exception ex) { log("get_proj_to_qry ERROR ." + ex.Message); Thread.Sleep(5000); } //} } }
void SaveResult(List <XListing> listings, BotTypes botType, string recordId, FreeTask tsk) { listings.ToList().ForEach(x => { if (string.IsNullOrEmpty(x.ShopName) && !string.IsNullOrEmpty(x.SiteName) && !x.BotShopID.HasValue && !x.SiteName.ToLower().Contains("taobao") && !x.SiteName.ToLower().Contains("alibaba")) { x.ShopName = x.SiteName; x.BotShopID = x.SiteID; } if (!x.BotShopID.HasValue && !string.IsNullOrEmpty(x.ShopName)) { x.BotShopID = IDHelper.GetGuid(string.Format("{0},{1},{2},{3}", x.ShopName, x.SiteName, tsk._id, x.ShopID)); } }); var shopList = listings; shopList = shopList.DistinctBy(x => x.BotShopID); //var exists_ids = MySqlDbHelper.GetExsitsIds<Guid?>(com, "bot_shops", "Shop_id", shopList.Select(x => x.BotShopID).ToArray()); //if (exists_ids != null && exists_ids.Count > 0) //{ // shopList = shopList.Where(x => !exists_ids.Contains(x.BotShopID)).ToList(); //} FieldsDocument shopfd = new FieldsDocument(); shopfd.Add("BotShopID", 1); // MongoCollection<Guid> shopcol = MongoDBHelper<Guid>.GetMongoDB().GetCollection<Guid>("FreeBotShop"); var shopcol = MongoDBHelper.Instance.Get_FreeBotShop(); var builder = Builders <FreeBotShop> .Filter; List <Guid?> BotShopID = shopList.Select(x => x.BotShopID).ToList(); // var existsshop_objs = shopcol.Find(MongoDB.Driver.Builders.Query.In("BotShopID", new BsonArray(BotShopID))).SetFields(shopfd); var existsshop_objs = shopcol.Find(builder.In(x => x.Shop_id, BotShopID)).Project(x => x.Shop_id).ToList(); List <Guid?> exists_ids = new List <Guid?>(); foreach (var result in existsshop_objs) { exists_ids.Add(result); } if (exists_ids != null && exists_ids.Count > 0) { shopList = shopList.Where(x => !exists_ids.Contains(x.BotShopID)).ToList(); } if (shopList == null || shopList.Count == 0) { return; } List <FreeBotShop> dt = null; if (shopList.Count > 0) { dt = GetShopList(shopList); // var saved = MongoDBHelper<FreeBotShop>.BatchInsertData(dt, "FreeBotShop"); shopcol.InsertMany(dt); log("to save bot_shops"); log("Done"); } listings.ToList().ForEach(x => { if (!x.BotItemID.HasValue && !string.IsNullOrEmpty(x.ItemName)) { x.BotItemID = IDHelper.GetGuid(string.Format("{0},{1},{2},{3}", x.ItemName, x.SiteName, tsk._id, x.ItemID)); } }); var itemList = listings; itemList = itemList.DistinctBy(x => x.BotItemID); //var exists_itemids = MySqlDbHelper.GetExsitsIds<Guid?>(com, "bot_items", "ItemId", itemList.Select(x => x.BotItemID).ToArray()); //if (exists_itemids != null && exists_itemids.Count > 0) //{ // itemList = itemList.Where(x => !exists_itemids.Contains(x.BotItemID)).ToList(); //} FieldsDocument fd = new FieldsDocument(); fd.Add("BotItemID", 1); // MongoCollection<Guid> col = MongoDBHelper<Guid>.GetMongoDB().GetCollection<Guid>("FreeBotItem"); var col = MongoDBHelper.Instance.Get_FreeBotItem(); var itemBuilder = Builders <FreeBotItem> .Filter; List <Guid?> BotItemID = itemList.Select(x => x.BotItemID).ToList(); // var exists_objs = col.Find(MongoDB.Driver.Builders.Query.In("BotItemID", new BsonArray(BotItemID))).SetFields(fd); var exists_objs = col.Find(itemBuilder.In(x => x.ItemId, BotItemID)).Project(x => x.ItemId).ToList(); List <Guid?> existsitem_ids = new List <Guid?>(); foreach (var result in exists_objs) { existsitem_ids.Add(result); } if (existsitem_ids != null && existsitem_ids.Count > 0) { itemList = itemList.Where(x => !existsitem_ids.Contains(x.BotItemID)).ToList(); } List <XListing> updatelinks = new List <XListing>(); if (existsitem_ids != null && existsitem_ids.Count > 0) { updatelinks = updatelinks.Where(x => existsitem_ids.Contains(x.BotItemID)).ToList(); } update_level1_links(updatelinks, botType, recordId, tsk); if (itemList == null || itemList.Count == 0) { return; } var itemdt = GetItemList(itemList); // var savedListings = MySqlDbHelper.BatchInsert(con, "bot_items", itemdt); // var savedListings = MongoDBHelper<FreeBotItem>.BatchInsertData(itemdt, "FreeBotItem"); col.InsertMany(itemdt); // var wequery = new QueryDocument { { "_id", new ObjectId(recordId) } }; // FreeTaskRecord TaskList = MongoDBHelper<FreeTaskRecord>.Find1("FreeTaskRecord", wequery); var colRecord = MongoDBHelper.Instance.Get_FreeTaskRecord(); var RecordBuilder = Builders <FreeTaskRecord> .Filter; FreeTaskRecord TaskList = colRecord.Find(RecordBuilder.Eq(x => x._id, new ObjectId(recordId))).FirstOrDefault(); int LinksNum = 0; int ShopsNum = 0; LinksNum = TaskList.LinksNum + itemdt.Count; ShopsNum = TaskList.ShopsNum + dt.Count; var updateWebsiteCount = new UpdateDocument { { "$set", new QueryDocument { { "LinksNum", LinksNum }, { "ShopsNum", ShopsNum } } } }; // MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", wequery, updateWebsiteCount); MongoDBHelper.Instance.Get_FreeTaskRecord().UpdateOne(new QueryDocument { { "_id", new ObjectId(recordId) } }, updateWebsiteCount); var colTask = MongoDBHelper.Instance.Get_FreeTask(); var TaskBuilder = Builders <FreeTask> .Filter; FreeTask Task2List = colTask.Find(TaskBuilder.Eq(x => x._id, itemdt[0].taskId)).FirstOrDefault(); int TaskLinksNum = 0; int TaskShopsNum = 0; TaskLinksNum = Task2List.LinksNum + itemdt.Count; TaskShopsNum = Task2List.ShopsNum + dt.Count; var TaskupdateWebsiteCount = new UpdateDocument { { "$set", new QueryDocument { { "LinksNum", TaskLinksNum }, { "ShopsNum", TaskShopsNum } } } }; // MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", wequery, updateWebsiteCount); MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument { { "_id", itemdt[0].taskId } }, TaskupdateWebsiteCount); log("to save listings"); log("Done"); }