public void Run()
{
while (true)
{
BotTaskService bt = new BotTaskService();
Random r = new Random();
Thread.Sleep(2000);
FreeTask keyTask = bt.GetBotTask(BotTypes.ItemSnapshot); //get_task();
if (keyTask == null || keyTask.TaskName == "" || keyTask.TaskName == null)
{
SetReady();
log("No search task !");
Thread.Sleep(1000);
var sssH = DateTime.Now.Hour;
if (sssH == 5)
{
var miuH = DateTime.Now.Minute;
if (miuH < 3)
{
var queryTask = new QueryDocument {
{ "BotIntervalHours", 24 }, { "IsStarted", true }
};
var colH = MongoDBHelper.Instance.Get_FreeTask();
var resultH = colH.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList();
foreach (var item in resultH)
{
var update1 = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", false }
} }
};
var Hresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", item._id }
}, update1);
}
}
}
var Zhou = DateTime.Now.DayOfWeek.ToString();
if (Zhou == "Sunday")
{
var Zsss = DateTime.Now.Hour;
if (Zsss == 5)
{
var miuZ = DateTime.Now.Minute;
if (miuZ < 3)
{
var queryTask = new QueryDocument {
{ "BotIntervalHours", 168 }, { "IsStarted", true }
};
var colZ = MongoDBHelper.Instance.Get_FreeTask();
var resultZ = colZ.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList();
foreach (var item in resultZ)
{
var update2 = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", false }
} }
};
var Zresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", item._id }
}, update2);
}
}
}
}
var Tian = DateTime.Now.Day;
if (Tian == 30)
{
var Tsss = DateTime.Now.Hour;
if (Tsss == 5)
{
var miuT = DateTime.Now.Minute;
if (miuT < 3)
{
var queryTask = new QueryDocument {
{ "BotIntervalHours", 720 }, { "IsStarted", true }
};
var colT = MongoDBHelper.Instance.Get_FreeTask();
var resultT = colT.Find(queryTask).SortByDescending(x => x.CreatedAt).ToList();
foreach (var item in resultT)
{
var update3 = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", false }
} }
};
var Tresult = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", item._id }
}, update3);
}
}
}
}
DateTime start = Convert.ToDateTime(DateTime.Now.AddDays(0).AddHours(8).ToShortDateString());
DateTime end = Convert.ToDateTime(DateTime.Now.AddDays(1).AddHours(8));
continue;
}
SetBusy();
//string sql = @"update taskparameters set LastBotStartAt ='{0}',IsBot=1 where Id={1}".FormatStr(DateTime.Now, keyTask.Id);
//MySqlDbHelper.ExecuteSql(com, sql);
int?LinksNum = 0;
LinksNum = keyTask.recordNum + 1;
var update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "IsBot", true }, { "LastBotStartAt", DateTime.Now.AddHours(8) },
{ "recordNum", LinksNum }
} }
};
// var result = MongoDBHelper<FreeTask>.Update("FreeTask", new QueryDocument { { "_id", keyTask._id } }, update);
var result = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", keyTask._id }
}, update);
List <FreeTaskRecord> list = new List <FreeTaskRecord>();
FreeTaskRecord ftr = new FreeTaskRecord();
ftr._id = new ObjectId();
ftr.IsStarted = keyTask.IsStarted;
ftr.IsBot = true;
ftr.Taskid = keyTask._id;
ftr.TaskName = keyTask.TaskName;
ftr.UsrId = keyTask.UsrId;
ftr.ServiceState = "Normal";
ftr.CreatedAt = DateTime.Now.AddHours(8);
ftr.LastBotStartAt = DateTime.Now.AddHours(8);
// ftr.LastBotEndAt = DateTime.Now.AddHours(8);
ftr.Dataquantity = 0;
ftr.LinksNum = 0;
ftr.ShopsNum = 0;
ftr.SiteId = Guid.Parse("A00A672B-DD05-65FB-4EE0-CFA26EBF2ED5");
ftr.SiteName = "taobao";
ftr.LanIP = "";//TaobaoWebHelper.GetLanIP();
ftr.UId = keyTask.UId;
ftr.ProjectId = keyTask.ProjectId;
// var savedq = MongoDBHelper<FreeTaskRecord>.InsertData(ftr, "FreeTaskRecord");
list.Add(ftr);
var col = MongoDBHelper.Instance.Get_FreeTaskRecord();
col.InsertMany(list);
string recordId = ftr._id.ToString();
Snapshot(keyTask, com, recordId);
try
{
// update = new UpdateDocument { { "$set", new QueryDocument { { "LastBotEndAt", DateTime.Now.AddHours(8) } } } };
// result = MongoDBHelper<FreeTask>.Update("FreeTask", new QueryDocument { { "_id", keyTask._id } }, update);
update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LastBotEndAt", DateTime.Now.AddHours(8) }
} }
};
result = MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", keyTask._id }
}, update);
update = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LastBotEndAt", DateTime.Now.AddHours(8) }
} }
};
// result = MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", new QueryDocument { { "_id", new ObjectId(recordId)} }, update);
result = MongoDBHelper.Instance.Get_FreeTaskRecord().UpdateOne(new QueryDocument {
{ "_id", new ObjectId(recordId) }
}, update);
}
catch (Exception ex)
{
log("get_proj_to_qry ERROR ." + ex.Message);
Thread.Sleep(5000);
}
//}
}
}
void SaveResult(List <XListing> listings, BotTypes botType, string recordId, FreeTask tsk)
{
listings.ToList().ForEach(x =>
{
if (string.IsNullOrEmpty(x.ShopName) && !string.IsNullOrEmpty(x.SiteName) && !x.BotShopID.HasValue &&
!x.SiteName.ToLower().Contains("taobao") && !x.SiteName.ToLower().Contains("alibaba"))
{
x.ShopName = x.SiteName;
x.BotShopID = x.SiteID;
}
if (!x.BotShopID.HasValue && !string.IsNullOrEmpty(x.ShopName))
{
x.BotShopID = IDHelper.GetGuid(string.Format("{0},{1},{2},{3}", x.ShopName, x.SiteName, tsk._id, x.ShopID));
}
});
var shopList = listings;
shopList = shopList.DistinctBy(x => x.BotShopID);
//var exists_ids = MySqlDbHelper.GetExsitsIds<Guid?>(com, "bot_shops", "Shop_id", shopList.Select(x => x.BotShopID).ToArray());
//if (exists_ids != null && exists_ids.Count > 0)
//{
// shopList = shopList.Where(x => !exists_ids.Contains(x.BotShopID)).ToList();
//}
FieldsDocument shopfd = new FieldsDocument();
shopfd.Add("BotShopID", 1);
// MongoCollection<Guid> shopcol = MongoDBHelper<Guid>.GetMongoDB().GetCollection<Guid>("FreeBotShop");
var shopcol = MongoDBHelper.Instance.Get_FreeBotShop();
var builder = Builders <FreeBotShop> .Filter;
List <Guid?> BotShopID = shopList.Select(x => x.BotShopID).ToList();
// var existsshop_objs = shopcol.Find(MongoDB.Driver.Builders.Query.In("BotShopID", new BsonArray(BotShopID))).SetFields(shopfd);
var existsshop_objs = shopcol.Find(builder.In(x => x.Shop_id, BotShopID)).Project(x => x.Shop_id).ToList();
List <Guid?> exists_ids = new List <Guid?>();
foreach (var result in existsshop_objs)
{
exists_ids.Add(result);
}
if (exists_ids != null && exists_ids.Count > 0)
{
shopList = shopList.Where(x => !exists_ids.Contains(x.BotShopID)).ToList();
}
if (shopList == null || shopList.Count == 0)
{
return;
}
List <FreeBotShop> dt = null;
if (shopList.Count > 0)
{
dt = GetShopList(shopList);
// var saved = MongoDBHelper<FreeBotShop>.BatchInsertData(dt, "FreeBotShop");
shopcol.InsertMany(dt);
log("to save bot_shops");
log("Done");
}
listings.ToList().ForEach(x =>
{
if (!x.BotItemID.HasValue && !string.IsNullOrEmpty(x.ItemName))
{
x.BotItemID = IDHelper.GetGuid(string.Format("{0},{1},{2},{3}", x.ItemName, x.SiteName, tsk._id, x.ItemID));
}
});
var itemList = listings;
itemList = itemList.DistinctBy(x => x.BotItemID);
//var exists_itemids = MySqlDbHelper.GetExsitsIds<Guid?>(com, "bot_items", "ItemId", itemList.Select(x => x.BotItemID).ToArray());
//if (exists_itemids != null && exists_itemids.Count > 0)
//{
// itemList = itemList.Where(x => !exists_itemids.Contains(x.BotItemID)).ToList();
//}
FieldsDocument fd = new FieldsDocument();
fd.Add("BotItemID", 1);
// MongoCollection<Guid> col = MongoDBHelper<Guid>.GetMongoDB().GetCollection<Guid>("FreeBotItem");
var col = MongoDBHelper.Instance.Get_FreeBotItem();
var itemBuilder = Builders <FreeBotItem> .Filter;
List <Guid?> BotItemID = itemList.Select(x => x.BotItemID).ToList();
// var exists_objs = col.Find(MongoDB.Driver.Builders.Query.In("BotItemID", new BsonArray(BotItemID))).SetFields(fd);
var exists_objs = col.Find(itemBuilder.In(x => x.ItemId, BotItemID)).Project(x => x.ItemId).ToList();
List <Guid?> existsitem_ids = new List <Guid?>();
foreach (var result in exists_objs)
{
existsitem_ids.Add(result);
}
if (existsitem_ids != null && existsitem_ids.Count > 0)
{
itemList = itemList.Where(x => !existsitem_ids.Contains(x.BotItemID)).ToList();
}
List <XListing> updatelinks = new List <XListing>();
if (existsitem_ids != null && existsitem_ids.Count > 0)
{
updatelinks = updatelinks.Where(x => existsitem_ids.Contains(x.BotItemID)).ToList();
}
update_level1_links(updatelinks, botType, recordId, tsk);
if (itemList == null || itemList.Count == 0)
{
return;
}
var itemdt = GetItemList(itemList);
// var savedListings = MySqlDbHelper.BatchInsert(con, "bot_items", itemdt);
// var savedListings = MongoDBHelper<FreeBotItem>.BatchInsertData(itemdt, "FreeBotItem");
col.InsertMany(itemdt);
// var wequery = new QueryDocument { { "_id", new ObjectId(recordId) } };
// FreeTaskRecord TaskList = MongoDBHelper<FreeTaskRecord>.Find1("FreeTaskRecord", wequery);
var colRecord = MongoDBHelper.Instance.Get_FreeTaskRecord();
var RecordBuilder = Builders <FreeTaskRecord> .Filter;
FreeTaskRecord TaskList = colRecord.Find(RecordBuilder.Eq(x => x._id, new ObjectId(recordId))).FirstOrDefault();
int LinksNum = 0;
int ShopsNum = 0;
LinksNum = TaskList.LinksNum + itemdt.Count;
ShopsNum = TaskList.ShopsNum + dt.Count;
var updateWebsiteCount = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LinksNum", LinksNum }, { "ShopsNum", ShopsNum }
} }
};
// MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", wequery, updateWebsiteCount);
MongoDBHelper.Instance.Get_FreeTaskRecord().UpdateOne(new QueryDocument {
{ "_id", new ObjectId(recordId) }
}, updateWebsiteCount);
var colTask = MongoDBHelper.Instance.Get_FreeTask();
var TaskBuilder = Builders <FreeTask> .Filter;
FreeTask Task2List = colTask.Find(TaskBuilder.Eq(x => x._id, itemdt[0].taskId)).FirstOrDefault();
int TaskLinksNum = 0;
int TaskShopsNum = 0;
TaskLinksNum = Task2List.LinksNum + itemdt.Count;
TaskShopsNum = Task2List.ShopsNum + dt.Count;
var TaskupdateWebsiteCount = new UpdateDocument {
{ "$set", new QueryDocument {
{ "LinksNum", TaskLinksNum }, { "ShopsNum", TaskShopsNum }
} }
};
// MongoDBHelper<FreeTaskRecord>.Update("FreeTaskRecord", wequery, updateWebsiteCount);
MongoDBHelper.Instance.Get_FreeTask().UpdateOne(new QueryDocument {
{ "_id", itemdt[0].taskId }
}, TaskupdateWebsiteCount);
log("to save listings");
log("Done");
}
