Beispiel #1
0
        /// <summary>
        /// RunTask
        /// </summary>
        /// <param name="govCrawler"></param>
        /// <param name="govDbHelperRenwu"></param>
        /// <param name="renwuTableName"></param>
        /// <param name="govDbHelperResult"></param>
        /// <param name="resultTableName"></param>
        /// <param name="isContinue"></param>
        private static void RunTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName, bool isContinue = false)
        {
            while (true)
            {
                try
                {
                    //先取任务为1的
                    var dic =
                        govDbHelperRenwu.GetSelectDicBySqlWithLock(
                            $"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 1 LIMIT 3", renwuTableName);
                    var valueList = dic["companyname"];

                    //为空 取任务为6的 尝试次数小于等于5的
                    if (valueList.Count == 0)
                    {
                        //重试次数30次
                        dic =
                            govDbHelperRenwu.GetSelectDicBySqlWithLock(
                                $"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 6 AND taskTryNumber<30 LIMIT 3", renwuTableName, false);
                        valueList = dic["companyname"];
                    }


                    //为空 取任务为100 时间大于半小时的
                    if (valueList.Count == 0)
                    {
                        dic =
                            govDbHelperRenwu.GetSelectDicBySqlWithLock(
                                $"SELECT companyname FROM {renwuTableName} WHERE TaskStatue=100 AND TIMESTAMPDIFF(MINUTE,DispatchTime,NOW())>30 LIMIT 3", renwuTableName, false);
                        valueList = dic["companyname"];
                        if (valueList.Count == 0)
                        {
                            if (isContinue)
                            {
                                //休息50s
                                Console.WriteLine("休息50s");
                                Thread.Sleep(1000 * 50);
                                continue;
                            }
                            return;
                        }
                    }

                    foreach (var value in valueList)
                    {
                        var companyName = value.ToString();

                        var companyInfoDic = new Dictionary <string, string>();
                        try
                        {
                            try
                            {
                                companyInfoDic = govCrawler.GetCompanyInfoDicByKeyWord(companyName);
                            }
                            catch (CompanyNotFoundException)
                            {
                                companyInfoDic.Add("companyName", companyName);

                                // 台州市黄岩南陌商贸有限公司
                                companyInfoDic.Add("notice", "公司信息不存在。");
                            }

                            //加入采集时间
                            companyInfoDic.Add("InfoGatherDate", DateTime.Now.ToString(CultureInfo.CurrentCulture));

                            // 插入结果表
                            govDbHelperResult.InsertTableWithDic(companyInfoDic, resultTableName);

                            // 更新任务表状态
                            govDbHelperRenwu.UpdateTable(companyName, renwuTableName, true);
                        }
                        catch (Exception e)
                        {
                            Console.WriteLine($"异常信息:{e.Message}");

                            // 更新任务表状态
                            govDbHelperRenwu.UpdateTable(companyName, renwuTableName, false);
                        }

                        Console.WriteLine($"companyname:{value}");
                        foreach (var info in companyInfoDic)
                        {
                            Console.WriteLine($"{info.Key}:{info.Value}");
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine($"出现异常:{e.Message}");
                    Console.WriteLine("休息1分钟");
                    Thread.Sleep(1000 * 60);
                }
            }
        }
Beispiel #2
0
        /// <summary>
        /// HasTask
        /// </summary>
        /// <param name="govCrawler"></param>
        /// <param name="govDbHelperRenwu"></param>
        /// <param name="renwuTableName"></param>
        /// <param name="govDbHelperResult"></param>
        /// <param name="resultTableName"></param>
        private static bool HasTask(GovCrawler govCrawler, GovDbHelper govDbHelperRenwu, string renwuTableName, GovDbHelper govDbHelperResult, string resultTableName)
        {
            try
            {
                //先取任务为1的
                var dic =
                    govDbHelperRenwu.GetSelectDicBySqlWithLock(
                        $"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 1 LIMIT 3", renwuTableName);
                var valueList = dic["companyname"];

                //为空 取任务为6的 尝试次数小于等于5的
                if (valueList.Count == 0)
                {
                    //重试次数30次
                    dic =
                        govDbHelperRenwu.GetSelectDicBySqlWithLock(
                            $"SELECT companyname FROM {renwuTableName} WHERE TaskStatue = 6 AND taskTryNumber<30 LIMIT 3", renwuTableName, false);
                    valueList = dic["companyname"];
                }


                //为空 取任务为100 时间大于半小时的
                if (valueList.Count == 0)
                {
                    dic =
                        govDbHelperRenwu.GetSelectDicBySqlWithLock(
                            $"SELECT companyname FROM {renwuTableName} WHERE TaskStatue=100 AND TIMESTAMPDIFF(MINUTE,DispatchTime,NOW())>30 LIMIT 3", renwuTableName, false);
                    valueList = dic["companyname"];
                    if (valueList.Count == 0)
                    {
                        return(false);
                    }
                }

                foreach (var value in valueList)
                {
                    var companyName = value.ToString();
                    //公司信息是否存在
                    var infoExist      = true;
                    var companyInfoDic = new Dictionary <string, string>();
                    var publishmentDetailInfoDicList = new List <Dictionary <string, string> >();
                    try
                    {
                        try
                        {
                            companyInfoDic = govCrawler.GetCompanyInfoDicByKeyWord(companyName);
                        }
                        catch (CompanyNotFoundException)
                        {
                            companyInfoDic.Add("companyName", companyName);

                            // 台州市黄岩南陌商贸有限公司
                            companyInfoDic.Add("notice", "公司信息不存在。");
                            // 公司信息不存在
                            infoExist = false;
                        }

                        //加入采集时间
                        companyInfoDic.Add("InfoGatherDate", DateTime.Now.ToString(CultureInfo.CurrentCulture));

                        if (infoExist)
                        {
                            publishmentDetailInfoDicList = govCrawler.GetPunishmentDetailInfoDicList(companyName);
                        }

                        // 插入结果表
                        govDbHelperResult.InsertTableWithDic(companyInfoDic, resultTableName);

                        //先处理 基础信息 再处理处罚信息 防止超时需要重新滑动滑块
                        if (infoExist)
                        {
                            foreach (Dictionary <string, string> publishmentDetailInfoDic in publishmentDetailInfoDicList)
                            {
                                govDbHelperResult.InsertTableWithDic(publishmentDetailInfoDic, "yuanshi_gongshang_x315_punishmentdetailinfo");
                            }
                        }


                        //更新任务表状态
                        govDbHelperRenwu.UpdateTable(companyName, renwuTableName, infoExist);
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine($"出现异常:{e.StackTrace}{Environment.NewLine}{e.Message}");

                        // 更新任务表状态
                        govDbHelperRenwu.UpdateTable(companyName, renwuTableName, false);
                    }

                    Console.WriteLine($"companyname:{value}");
                    foreach (var info in companyInfoDic)
                    {
                        Console.WriteLine($"{info.Key}:{info.Value}");
                    }

                    Console.WriteLine($"publishmentDetailInfo:");
                    foreach (Dictionary <string, string> publishmentDetailInfoDic in publishmentDetailInfoDicList)
                    {
                        foreach (var info in publishmentDetailInfoDic)
                        {
                            Console.WriteLine($"{info.Key}:{info.Value}");
                        }
                        Console.WriteLine();
                    }
                }
            }
            catch (Exception e)
            {
                Console.WriteLine($"出现异常:{e.StackTrace}{Environment.NewLine}{e.Message}");
                Console.WriteLine("休息1分钟");
                Thread.Sleep(1000 * 60);
            }

            return(true);
        }