public void ExecPageDBData()
{
DbHelp.Utilities util = new DbHelp.Utilities();
Log.ClsLog clsLog = new Log.ClsLog();
Common.UrlContorl urlContorl = new Common.UrlContorl();
while (true)
{
Parallel.ForEach <DBEntity>(Program.dbList.GetConsumingEnumerable(), dbEntity =>
//foreach (DBEntity dbEntity in Program.dbList.GetConsumingEnumerable())
{
try
{
//插入SQL
util.ExecNonQuery(dbEntity.myCom, "");
}
catch (Exception ex)
{
urlContorl.SaveUrl(dbEntity.pageContentEntity, ex.ToString());
clsLog.AddLog(DateTime.Now.ToString(), "执行数据失败" + ex.ToString());
clsLog.AddLog(DateTime.Now.ToString(), dbEntity.pageContentEntity.SType + ";" + dbEntity.pageContentEntity.PID + ";" + dbEntity.pageContentEntity.Url + "");
}
Interlocked.Decrement(ref Program.clsDBSignal);
});
}
}
public void AnalyseData()
{
string SDate = Program.SpiderDate;
Log.ClsLog clsLog = new Log.ClsLog();
Common.UrlContorl urlContorl = new Common.UrlContorl();
MainContorl mainContorl = new MainContorl();
Spider.ClsPageUrl clsPageUrl = new Spider.ClsPageUrl();
//new ParallelOptions() { MaxDegreeOfParallelism = 5 } 设置最大并行数量
//Parallel.ForEach<PageContentEntity>(Program.pageContentList.GetConsumingEnumerable(), pageContentEntity =>
//找出要抓取的Url
foreach (PageContentEntity pageContentEntity in Program.pageContentList.GetConsumingEnumerable())
{
//开始抓取数据
try
{
#region 分析数据
if (pageContentEntity.PContent.Trim() != "" || pageContentEntity.MContent != null)
{
//分析数据
switch (pageContentEntity.SType)
{
case "Portal":
mainContorl.HousePortalAnalysis(pageContentEntity);
break;
default: break;
}
}
else
{
//保存抓取失败数据
urlContorl.SaveUrl(pageContentEntity, "页面数据为空");
clsLog.AddLog(DateTime.Now.ToString(), "抓取数据失败");
clsLog.AddLog(DateTime.Now.ToString(), pageContentEntity.SType + ";" + pageContentEntity.Url);
}
#endregion
}
catch (Exception ex)
{
clsLog.AddLog(DateTime.Now.ToString(), "分析数据失败" + ex.ToString());
clsLog.AddLog(DateTime.Now.ToString(), pageContentEntity.SType + ";" + pageContentEntity.PID + ";" + pageContentEntity.Url);
urlContorl.SaveUrl(pageContentEntity, ex.ToString());
}
Interlocked.Decrement(ref Program.clsContentSignal);
}
}