public static string GetProvinceHtmlContent()
{
if (NationRegionOperaSql.IsExistsByLevel(1))
{
return("0");
}
string htmlString = WebHandler.GetHtmlStr(Const.WebSiteUri, "Default");
if (htmlString == "")
{
if (!NationRegionOperaSql.IsExistsExpetionByUri(Const.WebSiteUri, 1))
{
NationRegionOperaSql.InsertExpetion(Const.WebSiteUri, 1);
}
return("");
}
List <BasicModel> provinceModel = AnalysisProvHtmlstr.GetProvinceModel(htmlString);
string insertStr = string.Format("INSERT INTO [dbo].[UCML_NationRegion]([ParentNode],[Node],[Code],[Name],[Href],[IsLowerUp],[Level]) VALUES ");
foreach (BasicModel model in provinceModel)
{
insertStr += string.Format("('{0}','{1}','{2}','{3}','{4}','{5}','{6}'),",
model.ParentNode, model.Node, model.Code,
model.Name, model.Href, 0, model.Level);
}
NationRegionOperaSql.ExecuteSql(insertStr.Substring(0, insertStr.Length - 1));
return("");
}
public static string GetCityHtmlContent(int sleepTime)
{
List <BasicModel> list = NationRegionOperaSql.GetTopBasicModelByLevel(1);
Thread.Sleep(sleepTime); /*我好累,休息一下!*/
foreach (BasicModel model in list)
{
string htmlString = WebHandler.GetHtmlStr(Const.WebSiteUri + "//" + model.Href + ".html", "Default");
if (htmlString == "")
{
if (!NationRegionOperaSql.IsExistsExpetionByUri(Const.WebSiteUri + "//" + model.Href + ".html", 2))
{
NationRegionOperaSql.InsertExpetion(Const.WebSiteUri + "//" + model.Href + ".html", 2);
}
continue;
}
List <BasicModel> cityModels = AnalysisCityHtmlstr.GetCityModel(htmlString);
string insertStr = string.Format("INSERT INTO [dbo].[UCML_NationRegion]([ParentNode],[Node],[Code],[Name],[Href],[IsLowerUp],[Level]) VALUES ");
foreach (BasicModel cityModel in cityModels)
{
insertStr += string.Format("('{0}','{1}','{2}','{3}','{4}','{5}','{6}'),",
cityModel.ParentNode, cityModel.Node, cityModel.Code,
cityModel.Name, cityModel.Href, 0, cityModel.Level);
}
NationRegionOperaSql.ExecuteSql(insertStr.Substring(0, insertStr.Length - 1));
}
return("");
}
/// <summary>
/// 处理全部异常数据
/// </summary>
public static void GetAllExpetionData()
{
DataTable dt = NationRegionOperaSql.GetAllExpetionData();
foreach (DataRow dr in dt.Rows)
{
GetExceptionHtmlContent(dr["Uri"].ToString(), int.Parse(dr["Level"].ToString()));
}
}
/// <summary>
/// 处理异常
/// </summary>
/// <param name="uri"></param>
/// <returns></returns>
public static string GetExceptionVillageHtmlContent(string uri)
{
string htmlString = "";
try
{
//这个比较特殊点
htmlString = WebHandler.GetHtmlStr(uri, "Default");
if (htmlString == "")
{
if (!NationRegionOperaSql.IsExistsExpetionByUri(uri, 5))
{
NationRegionOperaSql.InsertExpetion(uri, 5);
}
return("");
}
int startIndex = uri.LastIndexOf('/') + 1;
string ParentNode = uri.Substring(startIndex, uri.Length - startIndex).Split('.')[0];
List <VillageModel> villageModels = AnalysisVillageHtmlstr.GetVillageModel(htmlString, ParentNode);
string insertStr = string.Format(@"
CREATE TABLE #Temp_NationRegion(
[ParentNode] [varchar](50) NULL,
[Node] [varchar](50) NULL,
[Code] [varchar](50) NULL,
[TypeCode] [varchar](50) NULL,
[Name] [varchar](200) NULL,
[Href] [varchar](200) NULL,
[IsLowerUp] [int] NULL,
[Level] [int] NULL)
INSERT INTO #Temp_NationRegion([ParentNode],[Node],[Code],[TypeCode],[Name],[Href],[IsLowerUp],[Level]) VALUES ");
foreach (VillageModel villageModel in villageModels)
{
insertStr += string.Format("('{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}'),",
villageModel.ParentNode, villageModel.Node, villageModel.Code,
villageModel.TypeCode, villageModel.Name, villageModel.Href, 0, villageModel.Level);
}
insertStr = insertStr.Substring(0, insertStr.Length - 1);
insertStr += @"
INSERT INTO [dbo].[UCML_NationRegion]([ParentNode],[Node],[Code],[TypeCode],[Name],[Href],[IsLowerUp],[Level])
SELECT [ParentNode],[Node],[Code],[TypeCode],[Name],[Href],[IsLowerUp],[Level] FROM #Temp_NationRegion a
WHERE NOT EXISTS(SELECT 1 FROM dbo.UCML_NationRegion b WHERE a.Code=b.Code)
DROP TABLE #Temp_NationRegion ";
NationRegionOperaSql.ExecuteSql(insertStr.Substring(0, insertStr.Length - 1));
}
catch (Exception ex)
{
LogUtil.WriteInfo(ex.Message);
LogUtil.WriteInfo(ex.StackTrace);
LogUtil.WriteInfo(htmlString);
}
return("");
}
/// <summary>
/// 处理异常
/// </summary>
/// <param name="sleepTime"></param>
/// <returns></returns>
public static string GetExceptionCityHtmlContent(string Uri)
{
string htmlString = WebHandler.GetHtmlStr(Uri, "Default");
if (htmlString == "")
{
if (!NationRegionOperaSql.IsExistsExpetionByUri(Uri, 2))
{
NationRegionOperaSql.InsertExpetion(Uri, 2);
}
return("");
}
List <BasicModel> cityModels = AnalysisCityHtmlstr.GetCityModel(htmlString);
string insertStr = string.Format(@"
CREATE TABLE #Temp_NationRegion(
[ParentNode] [varchar](50) NULL,
[Node] [varchar](50) NULL,
[Code] [varchar](50) NULL,
[TypeCode] [varchar](50) NULL,
[Name] [varchar](50) NULL,
[Href] [varchar](200) NULL,
[IsLowerUp] [int] NULL,
[Level] [int] NULL)
INSERT INTO #Temp_NationRegion([ParentNode],[Node],[Code],[Name],[Href],[IsLowerUp],[Level]) VALUES ");
foreach (BasicModel cityModel in cityModels)
{
insertStr += string.Format("('{0}','{1}','{2}','{3}','{4}','{5}','{6}'),",
cityModel.ParentNode, cityModel.Node, cityModel.Code,
cityModel.Name, cityModel.Href, 0, cityModel.Level);
}
insertStr = insertStr.Substring(0, insertStr.Length - 1);
insertStr += @"
INSERT INTO [dbo].[UCML_NationRegion]([ParentNode],[Node],[Code],[Name],[Href],[IsLowerUp],[Level])
SELECT [ParentNode],[Node],[Code],[Name],[Href],[IsLowerUp],[Level] FROM #Temp_NationRegion a
WHERE NOT EXISTS(SELECT 1 FROM dbo.UCML_NationRegion b WHERE a.Code=b.Code)
DROP TABLE #Temp_NationRegion ";
NationRegionOperaSql.ExecuteSql(insertStr.Substring(0, insertStr.Length - 1));
return("");
}
public static string GetAreaCountyHtmlContent(int sleepTime)
{
List <BasicModel> list = NationRegionOperaSql.GetTopBasicModelByLevel(2);
Thread.Sleep(sleepTime); /*我好累,休息一下!*/
foreach (BasicModel model in list)
{
string htmlString = "";
try
{
htmlString = WebHandler.GetHtmlStr(Const.WebSiteUri + "//" + model.Href + ".html", "Default");
if (htmlString == "")
{
if (!NationRegionOperaSql.IsExistsExpetionByUri(Const.WebSiteUri + "//" + model.Href + ".html", 3))
{
NationRegionOperaSql.InsertExpetion(Const.WebSiteUri + "//" + model.Href + ".html", 3);
}
continue;
}
List <BasicModel> areaModels = AnalysisAreaCountyHtmlstr.GetAreaCountyModel(htmlString, model.Node);
string insertStr = string.Format("INSERT INTO [dbo].[UCML_NationRegion]([ParentNode],[Node],[Code],[Name],[Href],[IsLowerUp],[Level]) VALUES ");
foreach (BasicModel areaModel in areaModels)
{
insertStr += string.Format("('{0}','{1}','{2}','{3}','{4}','{5}','{6}'),",
areaModel.ParentNode, areaModel.Node, areaModel.Code,
areaModel.Name, areaModel.Href, 0, areaModel.Level);
}
NationRegionOperaSql.ExecuteSql(insertStr.Substring(0, insertStr.Length - 1));
}
catch (Exception ex)
{
LogUtil.WriteInfo(ex.Message);
LogUtil.WriteInfo(ex.StackTrace);
LogUtil.WriteInfo(htmlString);
}
}
return("");
}
public static string GetVillageHtmlContent(int sleepTime, int index)
{
List <BasicModel> list = NationRegionOperaSql.GetTopBasicModelByLevel(4);
if (list.Count <= 0)
{
Form1.readVillageHtmlThreadCount[index] = 1;
}
//Thread.Sleep(sleepTime);/*我好累,休息一下!*/
foreach (BasicModel model in list)
{
string htmlString = "";
try
{
//这个比较特殊点
string uri = Const.WebSiteUri + "//" + model.Node.Substring(0, 2) + "//" +
model.Node.Substring(2, 2) + "//" + model.Href + ".html";
htmlString = WebHandler.GetHtmlStr(uri, "Default");
if (htmlString == "")
{
if (!NationRegionOperaSql.IsExistsExpetionByUri(uri, 5))
{
NationRegionOperaSql.InsertExpetion(uri, 5);
}
continue;
}
List <VillageModel> villageModels = AnalysisVillageHtmlstr.GetVillageModel(htmlString, model.Node);
string insertStr = string.Format(@"
CREATE TABLE #Temp_NationRegion(
[ParentNode] [varchar](50) NULL,
[Node] [varchar](50) NULL,
[Code] [varchar](50) NULL,
[TypeCode] [varchar](50) NULL,
[Name] [varchar](200) NULL,
[Href] [varchar](200) NULL,
[IsLowerUp] [int] NULL,
[Level] [int] NULL)
INSERT INTO #Temp_NationRegion([ParentNode],[Node],[Code],[TypeCode],[Name],[Href],[IsLowerUp],[Level]) VALUES ");
foreach (VillageModel villageModel in villageModels)
{
insertStr += string.Format("('{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}'),",
villageModel.ParentNode, villageModel.Node, villageModel.Code,
villageModel.TypeCode, villageModel.Name, villageModel.Href, 0, villageModel.Level);
}
insertStr = insertStr.Substring(0, insertStr.Length - 1);
insertStr += @"
INSERT INTO [dbo].[UCML_NationRegion]([ParentNode],[Node],[Code],[TypeCode],[Name],[Href],[IsLowerUp],[Level])
SELECT [ParentNode],[Node],[Code],[TypeCode],[Name],[Href],[IsLowerUp],[Level] FROM #Temp_NationRegion a
WHERE NOT EXISTS(SELECT 1 FROM dbo.UCML_NationRegion b WHERE a.Code=b.Code)
DROP TABLE #Temp_NationRegion ";
NationRegionOperaSql.ExecuteSql(insertStr.Substring(0, insertStr.Length - 1));
}
catch (Exception ex)
{
LogUtil.WriteInfo(ex.Message);
LogUtil.WriteInfo(ex.StackTrace);
LogUtil.WriteInfo(htmlString);
}
}
return("");
}
