public void Handle(string url)
{
List <Base_Towns> towns = new List <Base_Towns>();
using (IDbConnection conn = DBHelper.Connection)
{
string sQuery = "SELECT Id,Code,TownId,TownName,CountyId,County_Id,CountyName,CityId,City_Id,CityName,ProvinceId,Province_Id,ProvinceName FROM Base_Towns where IsCompleted!=1";
conn.Open();
towns = conn.Query <Base_Towns>(sQuery).ToList();
Parallel.ForEach(towns, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, (town) =>
{
List <Base_Villages> villages = new List <Base_Villages>();
var getUrl = string.Empty;
if (string.IsNullOrWhiteSpace(town.County_Id))
{
getUrl = $"{url}{town.ProvinceId}/{town.CityId.Substring(2, 2)}/{town.Id}.html";
}
else
{
getUrl = $"{url}{town.ProvinceId}/{town.City_Id.Substring(2, 2)}/{town.County_Id.Substring(4, 2)}/{town.Id}.html";
}
Console.WriteLine($"villageUrl:{getUrl}");
HtmlDocument doc = new HtmlDocument();
var html = HttpServiceHelper.PolicyGet(getUrl);
doc.LoadHtml(html);
HtmlNode rootNode = doc.DocumentNode;
var villagetrs = rootNode.SelectNodes("//tr[@class='villagetr']");
foreach (var tr in villagetrs)
{
var tds = tr.SelectNodes("./td");
var code = tds[0].InnerText;
var name = tds[2].InnerText;
Console.WriteLine($"village:{code},{name}");
villages.Add(new Base_Villages
{
VillageId = code,
VillageName = name,
TownId = town.TownId,
TownName = town.TownName,
CountyId = town.CountyId,
CountyName = town.CountyName,
CityId = town.CityId,
CityName = town.CityName,
ProvinceId = town.ProvinceId,
ProvinceName = town.ProvinceName
});
}
if (villages.Count > 0)
{
SqlBulkCopyHelper db = new SqlBulkCopyHelper();
db.CommonBulkCopy(villages, null);
string updateTown = $"update Base_Towns set IsCompleted =1 where TownId= '{town.TownId}'";
conn.Execute(updateTown);
}
});
Console.WriteLine("村结束");
}
}
public void Handle(string url)
{
List <Base_Provinces> provinces = new List <Base_Provinces>();
using (IDbConnection conn = DBHelper.Connection)
{
string sQuery = "SELECT Id,Code,ProvinceId,ProvinceName FROM Base_Provinces where IsCompleted!=1";
conn.Open();
provinces = conn.Query <Base_Provinces>(sQuery).ToList();
Parallel.ForEach(provinces, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, (province) =>
{
List <Base_Cities> citys = new List <Base_Cities>();
var getUrl = $"{url}{province.Id}.html";
Console.WriteLine($"cityUrl:{getUrl}");
HtmlDocument doc = new HtmlDocument();
var html = HttpServiceHelper.PolicyGet(getUrl);
doc.LoadHtml(html);
HtmlNode rootNode = doc.DocumentNode;
var citytrs = rootNode.SelectNodes("//tr[@class='citytr']");
foreach (var citytr in citytrs)
{
var cityas = citytr.SelectNodes("./td/a[@href]");
var href = cityas[0].Attributes["href"].Value;
var id = Regex.Match(href, @"[0-9]{4}").Value;
var code = cityas[0].InnerText;
var name = cityas[1].InnerText;
Console.WriteLine($"city:{id},{code},{name}");
citys.Add(new Base_Cities
{
Id = id,
Code = code,
CityId = code,
CityName = name,
ProvinceId = province.ProvinceId,
Province_Id = province.Id,
ProvinceName = province.ProvinceName,
IsCompleted = false
});
}
if (citys.Count > 0)
{
SqlBulkCopyHelper db = new SqlBulkCopyHelper();
db.CommonBulkCopy(citys, null);
string udpateProvince = $"update Base_Provinces set IsCompleted =1 where provinceId= '{province.ProvinceId}'";
conn.Execute(udpateProvince);
}
});
Console.WriteLine("城市结束");
}
}
public void Handle(string url)
{
List <Models.Base_Provinces> provinces = new List <Base_Provinces>();
using (IDbConnection conn = DBHelper.Connection)
{
string sQuery = "SELECT Id,Code,ProvinceId,ProvinceName FROM Base_Provinces";
conn.Open();
provinces = conn.Query <Base_Provinces>(sQuery).ToList();
}
if (provinces.Count == 0)
{
HtmlDocument doc = new HtmlDocument();
var html = HttpServiceHelper.PolicyGet($"{url}index.html");
doc.LoadHtml(html);
HtmlNode rootNode = doc.DocumentNode;
var provinceas = rootNode.SelectNodes("//tr[@class='provincetr']/td/a[@href]");
foreach (var provincea in provinceas)
{
var href = provincea.Attributes["href"].Value;
var id = Regex.Match(href, @"[0-9]{2}").Value;
var name = provincea.InnerText;
var code = id;
Console.WriteLine($"province:{id},{code},{name}");
provinces.Add(new Base_Provinces
{
Id = id,
Code = code,
ProvinceId = code,
ProvinceName = name,
IsCompleted = false
});
}
SqlBulkCopyHelper db = new SqlBulkCopyHelper();
db.CommonBulkCopy(provinces, null);
Console.WriteLine("省份结束");
}
}
public void Handle(string url)
{
List <Base_Cities> citys = new List <Base_Cities>();
using (IDbConnection conn = DBHelper.Connection)
{
string sQuery = "SELECT Id,Code,CityId,CityName,ProvinceId,Province_Id,ProvinceName FROM Base_Cities where IsCompleted!=1";
conn.Open();
citys = conn.Query <Base_Cities>(sQuery).ToList();
Parallel.ForEach(citys, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, (city) =>
{
List <Base_Counties> countys = new List <Base_Counties>();
var getUrl = $"{url}{city.ProvinceId}/{city.Id}.html";
Console.WriteLine($"countyUrl:{getUrl}");
HtmlDocument doc = new HtmlDocument();
var html = HttpServiceHelper.PolicyGet(getUrl);
doc.LoadHtml(html);
HtmlNode rootNode = doc.DocumentNode;
var countytrs = rootNode.SelectNodes("//tr[@class='countytr']");
var towntrs = rootNode.SelectNodes("//tr[@class='towntr']");
if (countytrs != null)
{
foreach (var tr in countytrs)
{
var tdas = tr.SelectNodes("./td/a[@href]");
if (tdas != null)
{
var href = tdas[0].Attributes["href"].Value;
var id = Regex.Match(href, @"[0-9]{6}").Value;
var code = tdas[0].InnerText;
var name = tdas[1].InnerText;
Console.WriteLine($"county:{id},{code},{name}");
countys.Add(new Base_Counties
{
Id = id,
Code = code,
CountyId = code,
CountyName = name,
ProvinceId = city.ProvinceId,
Province_Id = city.Province_Id,
ProvinceName = city.ProvinceName,
CityId = city.CityId,
City_Id = city.Id,
CityName = city.CityName,
IsHasChildren = true,
IsCompleted = false
});
}
else
{
var code = tr.ChildNodes[0].InnerText;
var name = tr.ChildNodes[1].InnerText;
Console.WriteLine($"county:{code},{name}");
countys.Add(new Base_Counties
{
Id = "",
Code = code,
CountyId = code,
CountyName = name,
ProvinceId = city.ProvinceId,
Province_Id = city.Province_Id,
ProvinceName = city.ProvinceName,
CityId = city.CityId,
City_Id = city.Id,
CityName = city.CityName,
IsHasChildren = false,
IsCompleted = false
});
}
}
}
else if (towntrs != null)
{
var code = $"{city.Id}01000000";
var name = "市辖区";
Console.WriteLine($"county:{code},市辖区");
countys.Add(new Base_Counties
{
Id = "",
Code = code,
CountyId = code,
CountyName = name,
ProvinceId = city.ProvinceId,
Province_Id = city.Province_Id,
ProvinceName = city.ProvinceName,
CityId = city.CityId,
City_Id = city.Id,
CityName = city.CityName,
IsHasChildren = true,
IsCompleted = false
});
}
if (countys.Count > 0)
{
SqlBulkCopyHelper db = new SqlBulkCopyHelper();
db.CommonBulkCopy(countys, null);
string updateCity = $"update Base_Cities set IsCompleted =1 where CityId= '{city.CityId}'";
conn.Execute(updateCity);
}
});
Console.WriteLine("区县结束");
}
}