/// <summary>
/// RunTask
/// </summary>
/// <param name="govCrawler"></param>
/// <param name="govDbHelperRenwu"></param>
/// <param name="renwuTableName"></param>
/// <param name="govDbHelperResult"></param>
/// <param name="resultTableName"></param>
/// <param name="isContinue"></param>
private static void RunTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName, bool isContinue = false)
{
while (true)
{
try
{
//先取任务为1的
var dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 1 LIMIT 3", renwuTableName);
var valueList = dic["companyname"];
//为空 取任务为6的 尝试次数小于等于5的
if (valueList.Count == 0)
{
//重试次数30次
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 6 AND taskTryNumber<30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
}
//为空 取任务为100 时间大于半小时的
if (valueList.Count == 0)
{
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue=100 AND TIMESTAMPDIFF(MINUTE,DispatchTime,NOW())>30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
if (valueList.Count == 0)
{
if (isContinue)
{
//休息50s
Console.WriteLine("休息50s");
Thread.Sleep(1000 * 50);
continue;
}
return;
}
}
foreach (var value in valueList)
{
var companyName = value.ToString();
var companyInfoDic = new Dictionary <string, string>();
try
{
try
{
companyInfoDic = govCrawler.GetCompanyInfoDicByKeyWord(companyName);
}
catch (CompanyNotFoundException)
{
companyInfoDic.Add("companyName", companyName);
// 台州市黄岩南陌商贸有限公司
companyInfoDic.Add("notice", "公司信息不存在。");
}
//加入采集时间
companyInfoDic.Add("InfoGatherDate", DateTime.Now.ToString(CultureInfo.CurrentCulture));
// 插入结果表
govDbHelperResult.InsertTableWithDic(companyInfoDic, resultTableName);
// 更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, true);
}
catch (Exception e)
{
Console.WriteLine($"异常信息：{e.Message}");
// 更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, false);
}
Console.WriteLine($"companyname:{value}");
foreach (var info in companyInfoDic)
{
Console.WriteLine($"{info.Key}:{info.Value}");
}
}
}
catch (Exception e)
{
Console.WriteLine($"出现异常：{e.Message}");
Console.WriteLine("休息1分钟");
Thread.Sleep(1000 * 60);
}
}
}
/// <summary>
/// HasTask
/// </summary>
/// <param name="govCrawler"></param>
/// <param name="govDbHelperRenwu"></param>
/// <param name="renwuTableName"></param>
/// <param name="govDbHelperResult"></param>
/// <param name="resultTableName"></param>
private static bool HasTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName)
{
try
{
//先取任务为1的
var dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 1 LIMIT 3", renwuTableName);
var valueList = dic["companyname"];
//为空 取任务为6的 尝试次数小于等于5的
if (valueList.Count == 0)
{
//重试次数30次
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 6 AND taskTryNumber<30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
}
//为空 取任务为100 时间大于半小时的
if (valueList.Count == 0)
{
dic =
govDbHelperRenwu.GetSelectDicBySqlWithLock(
$"SELECT companyname FROM {renwuTableName} WHERE TaskStatue=100 AND TIMESTAMPDIFF(MINUTE,DispatchTime,NOW())>30 LIMIT 3", renwuTableName, false);
valueList = dic["companyname"];
if (valueList.Count == 0)
{
return(false);
}
}
foreach (var value in valueList)
{
var companyName = value.ToString();
//公司信息是否存在
var infoExist = true;
var companyInfoDic = new Dictionary <string, string>();
var publishmentDetailInfoDicList = new List <Dictionary <string, string> >();
try
{
try
{
companyInfoDic = govCrawler.GetCompanyInfoDicByKeyWord(companyName);
}
catch (CompanyNotFoundException)
{
companyInfoDic.Add("companyName", companyName);
// 台州市黄岩南陌商贸有限公司
companyInfoDic.Add("notice", "公司信息不存在。");
// 公司信息不存在
infoExist = false;
}
//加入采集时间
companyInfoDic.Add("InfoGatherDate", DateTime.Now.ToString(CultureInfo.CurrentCulture));
if (infoExist)
{
publishmentDetailInfoDicList = govCrawler.GetPunishmentDetailInfoDicList(companyName);
}
// 插入结果表
govDbHelperResult.InsertTableWithDic(companyInfoDic, resultTableName);
//先处理 基础信息 再处理处罚信息 防止超时需要重新滑动滑块
if (infoExist)
{
foreach (Dictionary <string, string> publishmentDetailInfoDic in publishmentDetailInfoDicList)
{
govDbHelperResult.InsertTableWithDic(publishmentDetailInfoDic, "yuanshi_gongshang_x315_punishmentdetailinfo");
}
}
//更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, infoExist);
}
catch (Exception e)
{
Console.WriteLine($"出现异常：{e.StackTrace}{Environment.NewLine}{e.Message}");
// 更新任务表状态
govDbHelperRenwu.UpdateTable(companyName, renwuTableName, false);
}
Console.WriteLine($"companyname:{value}");
foreach (var info in companyInfoDic)
{
Console.WriteLine($"{info.Key}:{info.Value}");
}
Console.WriteLine($"publishmentDetailInfo:");
foreach (Dictionary <string, string> publishmentDetailInfoDic in publishmentDetailInfoDicList)
{
foreach (var info in publishmentDetailInfoDic)
{
Console.WriteLine($"{info.Key}:{info.Value}");
}
Console.WriteLine();
}
}
}
catch (Exception e)
{
Console.WriteLine($"出现异常：{e.StackTrace}{Environment.NewLine}{e.Message}");
Console.WriteLine("休息1分钟");
Thread.Sleep(1000 * 60);
}
return(true);
}