/// <summary>
/// Mains the specified arguments.
/// </summary>
/// <param name="args">The arguments.</param>
private static void Main(string[] args)
{
var tSessionDefender = new TerminalSessionDefender();
tSessionDefender.OnLogginEvent += WriteLog;
tSessionDefender.Start();
/*tSessionDefender.Stop();
* tSessionDefender.OnLogginEvent -= WriteLog;*/
// 启用日志记录
X.CommLib.Logs.LogHelper.StartLogSystem();
var govCrawler = new GovCrawler();
govCrawler.OnLogginEvent += WriteLog;
//由于安全原因 此处不给出工作环境
var govDbHelperRenwu = new GovDbHelper("host", "tableName", "userName", "password");
var govDbHelperResult = new GovDbHelper("host", "tableName", "userName", "password");
////优先
//RunTask(govCrawler, govDbHelperRenwu, "renwu_gongshang_x315_vip_attr", govDbHelperResult, "yuanshi_gongshang_x315_vip_attr");
////其次 继续 不退出
//RunTask(govCrawler, govDbHelperRenwu, "renwu_gongshang_x315_attr", govDbHelperResult, "yuanshi_gongshang_x315_attr", true);
RunVipTaskUntilFinish(govCrawler, govDbHelperRenwu, "renwu_gongshang_x315_vip_attr", govDbHelperResult, "yuanshi_gongshang_x315_vip_attr");
RunTaskAndCheckVipTask(govCrawler, govDbHelperRenwu, "renwu_gongshang_x315_vip_attr", "renwu_gongshang_x315_attr",
govDbHelperResult, "yuanshi_gongshang_x315_vip_attr", "yuanshi_gongshang_x315_attr");
}
/// <summary>
/// RunVipTask
/// </summary>
/// <param name="govCrawler"></param>
/// <param name="govDbHelperRenwu"></param>
/// <param name="renwuTableName"></param>
/// <param name="govDbHelperResult"></param>
/// <param name="resultTableName"></param>
private static void RunVipTaskUntilFinish(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName)
{
var hasTask = true;
while (hasTask)
{
hasTask = HasTask(govCrawler, govDbHelperRenwu, renwuTableName, govDbHelperResult, resultTableName);
}
}
/// <summary>
/// RunTaskAndCheckVipTask
/// </summary>
/// <param name="govCrawler"></param>
/// <param name="govDbHelperRenwu"></param>
/// <param name="renwuVipTableName"></param>
/// <param name="renwuTableName"></param>
/// <param name="govDbHelperResult"></param>
/// <param name="resultVipTableName"></param>
/// <param name="resultTableName"></param>
private static void RunTaskAndCheckVipTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuVipTableName, string renwuTableName, GovDbHelper govDbHelperResult, string resultVipTableName, string resultTableName)
{
var startTime = DateTime.Now;
while (true)
{
//跑一个普通任务
HasTask(govCrawler, govDbHelperRenwu, renwuTableName, govDbHelperResult, resultTableName);
if ((DateTime.Now - startTime).Minutes <= 30)
{
continue;
}
//跑vip任务
RunVipTaskUntilFinish(govCrawler, govDbHelperRenwu, renwuVipTableName, govDbHelperResult, resultVipTableName);
//更新开始时间
startTime = DateTime.Now;
}
}
/// <summary>
/// RunTask
/// </summary>
/// <param name="govCrawler"></param>
/// <param name="govDbHelperRenwu"></param>
/// <param name="renwuTableName"></param>
/// <param name="govDbHelperResult"></param>
/// <param name="resultTableName"></param>
/// <param name="isContinue"></param>
private static void RunTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName, bool isContinue = false)
{
while (true)
{
try
{
//先取任务为1的
var dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 1 LIMIT 3", renwuTableName);
var valueList = dic["companyname"];
//为空 取任务为6的 尝试次数小于等于5的
if (valueList.Count == 0)
{
//重试次数30次
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 6 AND taskTryNumber<30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
}
//为空 取任务为100 时间大于半小时的
if (valueList.Count == 0)
{
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue=100 AND TIMESTAMPDIFF(MINUTE,DispatchTime,NOW())>30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
if (valueList.Count == 0)
{
if (isContinue)
{
//休息50s
Console.WriteLine("休息50s");
Thread.Sleep(1000 * 50);
continue;
}
return;
}
}
foreach (var value in valueList)
{
var companyName = value.ToString();
var companyInfoDic = new Dictionary <string, string>();
try
{
try
{
companyInfoDic = govCrawler.GetCompanyInfoDicByKeyWord(companyName);
}
catch (CompanyNotFoundException)
{
companyInfoDic.Add("companyName", companyName);
// 台州市黄岩南陌商贸有限公司
companyInfoDic.Add("notice", "公司信息不存在。");
}
//加入采集时间
companyInfoDic.Add("InfoGatherDate", DateTime.Now.ToString(CultureInfo.CurrentCulture));
// 插入结果表
govDbHelperResult.InsertTableWithDic(companyInfoDic, resultTableName);
// 更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, true);
}
catch (Exception e)
{
Console.WriteLine($"异常信息：{e.Message}");
// 更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, false);
}
Console.WriteLine($"companyname:{value}");
foreach (var info in companyInfoDic)
{
Console.WriteLine($"{info.Key}:{info.Value}");
}
}
}
catch (Exception e)
{
Console.WriteLine($"出现异常：{e.Message}");
Console.WriteLine("休息1分钟");
Thread.Sleep(1000 * 60);
}
}
}
/// <summary>
/// HasTask
/// </summary>
/// <param name="govCrawler"></param>
/// <param name="govDbHelperRenwu"></param>
/// <param name="renwuTableName"></param>
/// <param name="govDbHelperResult"></param>
/// <param name="resultTableName"></param>
private static bool HasTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName)
{
try
{
//先取任务为1的
var dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 1 LIMIT 3", renwuTableName);
var valueList = dic["companyname"];
//为空 取任务为6的 尝试次数小于等于5的
if (valueList.Count == 0)
{
//重试次数30次
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 6 AND taskTryNumber<30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
}
//为空 取任务为100 时间大于半小时的
if (valueList.Count == 0)
{
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue=100 AND TIMESTAMPDIFF(MINUTE,DispatchTime,NOW())>30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
if (valueList.Count == 0)
{
return(false);
}
}
foreach (var value in valueList)
{
var companyName = value.ToString();
//公司信息是否存在
var infoExist = true;
var companyInfoDic = new Dictionary <string, string>();
var publishmentDetailInfoDicList = new List <Dictionary <string, string> >();
try
{
try
{
companyInfoDic = govCrawler.GetCompanyInfoDicByKeyWord(companyName);
}
catch (CompanyNotFoundException)
{
companyInfoDic.Add("companyName", companyName);
// 台州市黄岩南陌商贸有限公司
companyInfoDic.Add("notice", "公司信息不存在。");
// 公司信息不存在
infoExist = false;
}
//加入采集时间
companyInfoDic.Add("InfoGatherDate", DateTime.Now.ToString(CultureInfo.CurrentCulture));
if (infoExist)
{
publishmentDetailInfoDicList = govCrawler.GetPunishmentDetailInfoDicList(companyName);
}
// 插入结果表
govDbHelperResult.InsertTableWithDic(companyInfoDic, resultTableName);
//先处理 基础信息 再处理处罚信息 防止超时需要重新滑动滑块
if (infoExist)
{
foreach (Dictionary <string, string> publishmentDetailInfoDic in publishmentDetailInfoDicList)
{
govDbHelperResult.InsertTableWithDic(publishmentDetailInfoDic, "yuanshi_gongshang_x315_punishmentdetailinfo");
}
}
//更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, infoExist);
}
catch (Exception e)
{
Console.WriteLine($"出现异常：{e.StackTrace}{Environment.NewLine}{e.Message}");
// 更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, false);
}
Console.WriteLine($"companyname:{value}");
foreach (var info in companyInfoDic)
{
Console.WriteLine($"{info.Key}:{info.Value}");
}
Console.WriteLine($"publishmentDetailInfo:");
foreach (Dictionary <string, string> publishmentDetailInfoDic in publishmentDetailInfoDicList)
{
foreach (var info in publishmentDetailInfoDic)
{
Console.WriteLine($"{info.Key}:{info.Value}");
}
Console.WriteLine();
}
}
}
catch (Exception e)
{
Console.WriteLine($"出现异常：{e.StackTrace}{Environment.NewLine}{e.Message}");
Console.WriteLine("休息1分钟");
Thread.Sleep(1000 * 60);
}
return(true);
}