public override List<string> GetUrlsFromPages(int total)
{
var urls = new List<string>();
try
{
var html = Client.RetrieveHtmlStrGet(URL_TOTALPAGES).Result;
var doc = new HtmlDocument();
doc.LoadHtml(html);
var odd = doc.DocumentNode.Descendants("tr").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("odd")).Select(x=>x.InnerHtml).ToList();
var even = doc.DocumentNode.Descendants("tr").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("even")).Select(x=>x.InnerHtml).ToList();
odd.AddRange(even);
var info = odd.Select(x => x.SplitTag()).ToList();
urls = odd.Select(x => x.GetMatchStr(@"(properties\/prop_detail.php\?page=home&ID=[0-9]{0,10})").FirstOrDefault()).ToList();
//urls = RegexHelper.GetMatchStr(html, @"(\/properties\/prop_detail.php\?page=home&ID=[0-9]{0,10})").Distinct().Select(x => URL_MAIN + x).ToList();
var res = new List<RealEstateE>();
for (int i = 0; i < info.Count; i++)
{
var json = new RealEstateObj();
json.Code = info[i][1].Trim();
json.PropertyType = info[i][2].Trim();
json.SizeTotal = info[i][3].Trim();
json.Price = info[i][5].ToDecimal();
json.Source = this.GetType().Name;
res.Add(new RealEstateE()
{
Url = URL_MAIN + urls[i].Trim(),
State = 0,
RecordStatus = 1,
Source = SourceName,
RealEstateJson = json.Serialize(true)
});
}
DataHelper.InsertRealEstateE(res);
}
catch (Exception ex)
{
lock (sync)
{
File.AppendAllText("D:/RE/A_" + this.GetType().Name + ".log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + "" + "," + ex.GetBaseException().Message + "\r\n");
}
}
return urls;
}
public List<string> GetUrlsFromPage(string pageUrl)
{
var urls = new List<string>();
try
{
var html = Client.RetrieveHtmlStrGet(pageUrl).Result;
//urls = RegexHelper.GetMatchStr(html, @"(APropertyDetail.html\?id=[0-9]{0,20})").Distinct().Select(x => URL_MAIN + x).ToList();
var doc = new HtmlDocument();
doc.LoadHtml(html);
var tmp = doc.DocumentNode.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("SummaryPanel")).Select(x => x.InnerHtml).ToList();
urls = tmp.Select(x => x.GetMatchStr(@"(APropertyDetail.html\?id=[0-9]{0,20})").Distinct().Select(y => URL_MAIN + y).FirstOrDefault()).ToList();
var info = tmp.Select(x => x.SplitTag()).ToList();
var res = new List<RealEstateE>();
for (int i = 0; i < urls.Count; i++)
{
var json = new RealEstateObj();
json.Price = info[i][0].Replace("ราคาเริ่มต้น", string.Empty).Replace("บ.", string.Empty).ToDecimal();
json.PropertyType = info[i][1].SplitRemoveEmpty(" ")[0].Trim();
json.Source = this.GetType().Name;
res.Add(new RealEstateE()
{
Url = urls[i].Trim(),
State = 0,
RecordStatus = 1,
Source = SourceName,
RealEstateJson = json.Serialize(true)
});
}
DataHelper.InsertRealEstateE(res);
}
catch (Exception ex)
{
lock (sync)
{
File.AppendAllText("D:/RE/A_" + this.GetType().Name + ".log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + pageUrl + "," + ex.GetBaseException().Message + "\r\n");
}
}
return urls;
}
public override void GetDetails(SourceObj obj)
{
Console.WriteLine(obj.Url);
var re = new RealEstateObj();
var doc = new HtmlAgilityPack.HtmlDocument();
try
{
if (string.IsNullOrEmpty(obj.Data))
{
obj.Data = Client.RetrieveHtmlStrGet(obj.Url).Result;
}
doc.LoadHtml(obj.Data);
if (string.IsNullOrEmpty(obj.RealEstateJson))
re = new RealEstateObj();
else
re = obj.RealEstateJson.Deserialize<RealEstateObj>();
re.Url = obj.Url;
var isSoldout = doc.DocumentNode.Descendants("a")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("thumb-label label-sold-s")).FirstOrDefault();
if (isSoldout != null)
{
re.IsSoldOut = true;
}
re.Url = obj.Url;
re.Source = SourceName;
re.Icon = URL_MAIN + doc.DocumentNode.Descendants("p")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("Thumb-s"))
.FirstOrDefault().ChildNodes["img"].GetAttributeValue("src", string.Empty);
var html = doc.DocumentNode.Descendants("div").Where(x => x.Id == "Lname").FirstOrDefault().ParentNode.ParentNode.InnerHtml;
html = WebUtility.HtmlDecode(html);
var titleDetails = RegexHelper.SplitTag(html);
re.Name = titleDetails[0];
re.PropertyType = titleDetails[0].Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
if (titleDetails[1].Contains("ไร่-งาน-วา"))
{
var tmp = titleDetails[1].Split(new char[] { '-', ' ' }).Select(x => x.Replace('(', ' ').Replace(')', ' ')).ToList();
if (tmp.Count == 6)
{
var rai = decimal.Parse(tmp[0]) * 400;
var ngn = decimal.Parse(tmp[1]) * 100;
var war = decimal.Parse(tmp[2]);
re.SizeTotal = (rai + ngn + war).ToString();
re.SizeTotalUnit = "ตร.ว.";
}
}
else if (titleDetails[1].Contains("ตร.ม."))
{
var tmp = titleDetails[1].Split(new char[] { '-', ' ' }).Select(x => x.Replace('(', ' ').Replace(')', ' ')).ToList();
if (tmp.Count == 4)
{
re.SizeTotal = tmp[2].GetMatchStr(RegexHelper.REGEX_NUMBER).FirstOrDefault();
re.SizeTotalUnit = "ตร.ม.";
}
}
re.SizeTotalText = titleDetails[1];
if (titleDetails.Length > 3)
{
re.BedRooom = titleDetails[2];
re.BathRoom = titleDetails[4];
re.ParkingSpace = titleDetails[6];
re.Price = titleDetails[8].GetMatchStr(RegexHelper.REGEX_MONEY).FirstOrDefault().ToDecimal();
}
else
{
re.Price = titleDetails[2].GetMatchStr(RegexHelper.REGEX_MONEY).FirstOrDefault().ToDecimal();
}
var detailTitles = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("col2"))
.Select(x => WebUtility.HtmlDecode(RegexHelper.StripHTML(x.PreviousSibling.InnerHtml))).ToList();
var detailDescs = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("col2"))
.Select(x => WebUtility.HtmlDecode(RegexHelper.StripHTML(x.InnerHtml))).ToList();
var details = new Dictionary<string, string>();
for (int i = 0; i < detailTitles.Count(); i++)
{
details.Add(detailTitles[i], detailDescs[i]);
}
//var price = doc.DocumentNode.Descendants("div")
// .Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("price")).FirstOrDefault().InnerHtml;
re.Code = details.ContainsKey("รหัสทรัพย์:") ? details["รหัสทรัพย์:"] : string.Empty;
re.SizeTotal = string.IsNullOrEmpty(re.SizeTotal) ? details.ContainsKey("เนื้อที่:") ? details["เนื้อที่:"].GetMatchStr(RegexHelper.REGEX_NUMBER).FirstOrDefault() : re.SizeTotal : re.SizeTotal;
re.Map.Desc = details.ContainsKey("ที่ตั้งทรัพย์:") ? details["ที่ตั้งทรัพย์:"].CleanNewLine() : string.Empty;
re.Storeys = details.ContainsKey("จำนวนชั้น:") ? details["จำนวนชั้น:"].Replace("ชั้น", string.Empty).Trim() : string.Empty;
var parcels = details.ContainsKey("โฉนดเลขที่:") ? RegexHelper.GetMatchStr(details["โฉนดเลขที่:"], RegexHelper.REGEX_NUMBER).ToArray() : new string[] { };
if (parcels.Count() == 0)
{
parcels = details.ContainsKey("กรรมสิทธิ์เลขที่:") ? RegexHelper.GetMatchStr(details["กรรมสิทธิ์เลขที่:"], RegexHelper.REGEX_NUMBER).ToArray() : new string[] { };
}
if (parcels.Count() == 0)
{
parcels = details.ContainsKey("น.ส. 3ก.เลขที่:") ? RegexHelper.GetMatchStr(details["น.ส. 3ก.เลขที่:"], RegexHelper.REGEX_NUMBER).ToArray() : new string[] { };
}
if (parcels.Count() == 0)
{
parcels = details.ContainsKey("น.ส. 3เลขที่:") ? RegexHelper.GetMatchStr(details["น.ส. 3เลขที่:"], RegexHelper.REGEX_NUMBER).ToArray() : new string[] { };
}
if (parcels.Count() == 0)
{
parcels = details.ContainsKey("อื่นๆเลขที่:") ? RegexHelper.GetMatchStr(details["อื่นๆเลขที่:"], RegexHelper.REGEX_NUMBER).ToArray() : new string[] { };
}
if (parcels.Count() > 0)
{
re.Map.ParcelNumber = parcels.Select(x => x.Trim()).ToArray();
}
//details.Add("price", RegexHelper.StripHTML(price));
//re.Details = details;
var gallery = doc.DocumentNode.Descendants("ul").Where(x => x.Id == "imageGallery").FirstOrDefault();
if (gallery != null)
{
re.Images = gallery.Elements("li")
.Select(x => URL_MAIN + x.Descendants("img").FirstOrDefault()
.GetAttributeValue("src", string.Empty)).Where(x => !x.Contains("M0")).ToList();
}
var contact = new Contact();
var contactStr = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("C_label")).FirstOrDefault()
.Element("div").InnerHtml.Trim();
var contactTells = RegexHelper.GetMatchStr(contactStr, RegexHelper.REGEX_TELL_NO);
contact.TellNo = contactTells.ToList<string>();
contact.Name = details.ContainsKey("สอบถามรายละเอียด:") ? details["สอบถามรายละเอียด:"].CleanNewLine() : string.Empty;
re.Contacts.Add(contact);
var loc = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("chromemenu")).FirstOrDefault().InnerHtml.Trim();
if (loc.Contains("พิกัด Latitude(X):"))
{
var arr = loc.Replace("พิกัด Latitude(X):", string.Empty).Replace("Longitude(Y):", string.Empty).Substring(0, loc.IndexOf("<br>")).Replace("<br>", string.Empty).Trim().Split(',');
//re.Map.Lat = double.Parse(arr[0].Trim());
//re.Map.Lon = double.Parse(arr[1].Trim());
re.Map.Coordinate = new Point(double.Parse(arr[1].Trim()), double.Parse(arr[0].Trim()));
}
var mapImg = URL_MAIN + doc.DocumentNode.Descendants("div").Where(x => x.Id == "imagetab").FirstOrDefault().Element("img").GetAttributeValue("src", string.Empty);
re.Map.Images.Add(mapImg);
var province = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_PROVINCE);
var district = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_DISTRICT);
var subProvince = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_SUB_DISTRICT);
var village = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_VILLAGE);
var road = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_ROAD);
var number = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_PROPERTYNUMBER);
var villageNo = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_VILLAGENO);
var alley = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_ALLEY);
var lane = RegexHelper.GetMatchStr(details["ที่ตั้งทรัพย์:"], RegexHelper.REGEX_LANE);
if (number.Count > 0)
re.Map.Number = number[0].Trim();
if (province.Count > 0)
re.Map.Province = province[0].Replace("จังหวัด", string.Empty).Trim();
if (district.Count > 0)
re.Map.District = district[0].Replace("อำเภอ", string.Empty).Replace("เขต", string.Empty).Trim();
if (subProvince.Count > 0)
re.Map.SubDistrict = subProvince[0].Replace("ตำบล", string.Empty).Replace("แขวง", string.Empty).Trim();
if (village.Count > 0)
re.Map.Village = village[0].Replace("หมู่บ้าน", string.Empty).Replace("อาคาร", string.Empty).Replace("ชุด", string.Empty).Replace("ถนน", string.Empty).Trim();
if (road.Count > 0)
re.Map.Road = road[0].Replace("ถนน", string.Empty).Replace("ถ.", string.Empty).Trim();
if (villageNo.Count > 0)
re.Map.VillageNo = villageNo[0].Replace("หมู่", string.Empty).Trim();
if (alley.Count > 0)
re.Map.Alley = alley[0].Replace("ตรอก", string.Empty).Trim();
if (lane.Count > 0)
re.Map.Lane = lane[0].Replace("ซอย", string.Empty).Trim();
var t = new RealEstateT
{
Data = JsonHelper.Serialize(re, true),
Province = re.Map.Province,
District = re.Map.District,
ParcelNo = JsonHelper.Serialize(re.Map.ParcelNumber),
Url = re.Url.Trim(),
Lat = re.Map.Coordinate.Position.Latitude,
Lon = re.Map.Coordinate.Position.Longitude,
State = 0,
RecordStatus = 1,
Source = "SCB"
};
DataHelper.InsertRealEstateT(t);
}
catch (Exception ex)
{
File.AppendAllText("D:/RE/T_SCB.log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + obj.Url + "," + ex.GetBaseException().Message + "\r\n");
}
}
public override void GetDetails(SourceObj obj)
{
Console.WriteLine(obj.Url);
var re = new RealEstateObj();
var doc = new HtmlAgilityPack.HtmlDocument();
try
{
if (string.IsNullOrEmpty(obj.Data))
{
obj.Data = Client.RetrieveHtmlStrGet(obj.Url, Encoding.GetEncoding(874)).Result;
}
doc.LoadHtml(obj.Data);
if (string.IsNullOrEmpty(obj.RealEstateJson))
re = new RealEstateObj();
else
re = obj.RealEstateJson.Deserialize<RealEstateObj>();
re.Url = obj.Url;
re.Source = SourceName;
var info = doc.DocumentNode.Descendants("td")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("font_normal2"))
.Select(x => WebUtility.HtmlDecode(x.InnerHtml.Trim()).StripHTML()).ToList();
if (info.Count > 0)
{
re.Code = info[0];
re.Name = info[1];
re.Project = info[2];
re.Map.Desc = info[3];
re.DocumentOfRightType = info[4];
re.Map.ParcelNumber = RegexHelper.GetMatchStr(info[5], RegexHelper.REGEX_NUMBER).ToArray();
var tmp = info[6].Split('-');
if (tmp.Length == 3)
{
var rai = decimal.Parse(tmp[0]) * 400;
var ngn = decimal.Parse(tmp[1]) * 100;
var war = decimal.Parse(tmp[2]);
re.SizeTotal = (rai + ngn + war).ToString();
re.SizeTotalUnit = "ตร.ว.";
}
re.SizeTotalText = info[6].Replace(" ", string.Empty);
re.SizeUtility = info[7];
re.Width = info[8];
re.Depth = info[9];
re.WidthRoadFrontage = info[10];
re.Age = info[11];
re.Desc = info[12];
var province = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_PROVINCE);
var district = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_DISTRICT);
var subProvince = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_SUB_DISTRICT);
re.Map.Village = info[1];
var road = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_ROAD);
var number = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_PROPERTYNUMBER);
var villageNo = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_VILLAGENO);
var alley = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_ALLEY);
var lane = RegexHelper.GetMatchStr(info[3], RegexHelper.REGEX_LANE);
if (number.Count > 0)
re.Map.Number = number[0].Trim();
if (province.Count > 0)
re.Map.Province = province[0].Replace("จังหวัด", string.Empty).Trim();
if (district.Count > 0)
re.Map.District = district[0].Replace("อำเภอ", string.Empty).Replace("เขต", string.Empty).Trim();
if (subProvince.Count > 0)
re.Map.SubDistrict = subProvince[0].Replace("ตำบล", string.Empty).Replace("แขวง", string.Empty).Trim();
if (road.Count > 0)
re.Map.Road = road[0].Replace("ถนน", string.Empty).Replace("ถ.", string.Empty).Trim();
if (villageNo.Count > 0)
re.Map.VillageNo = villageNo[0].Replace("หมู่", string.Empty).Trim();
if (alley.Count > 0)
re.Map.Alley = alley[0].Replace("ตรอก", string.Empty).Trim();
if (lane.Count > 0)
re.Map.Lane = lane[0].Replace("ซอย", string.Empty).Trim();
}
//re.Price = obj.Data.GetMatchStr(@"((<b>)(\d{1,3}(,\d{3})*)?(\.\d+)(<\/b>))").FirstOrDefault().StripHTML();
var images = RegexHelper.GetMatchStr(obj.Data, "(\\/asset_images\\/(.*?)\")").Distinct().Select(x => URL_MAIN + x.TrimEnd('"')).ToList();
if (images.Count > 1)
{
re.Images.Add(images[0]);
re.Map.Images.Add(images[1]);
}
var contact = new Contact();
contact.TellNo.Add("0-2260-6100");
contact.Email.Add("*****@*****.**");
re.Contacts.Add(contact);
var t = new RealEstateT
{
Data = JsonHelper.Serialize(re, true),
Province = re.Map.Province,
District = re.Map.District,
ParcelNo = JsonHelper.Serialize(re.Map.ParcelNumber),
Url = re.Url.Trim(),
Lat = re.Map.Coordinate.Position.Latitude,
Lon = re.Map.Coordinate.Position.Longitude,
State = 0,
RecordStatus = 1,
Source = "TNB"
};
DataHelper.InsertRealEstateT(t);
}
catch (Exception ex)
{
File.AppendAllText("D:/RE/T_TNB.log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + obj.Url + "," + ex.GetBaseException().Message + "\r\n");
}
}
public override void GetDetails(SourceObj obj)
{
Console.WriteLine(obj.Url);
var re = new RealEstateObj();
var doc = new HtmlAgilityPack.HtmlDocument();
try
{
if (string.IsNullOrEmpty(obj.Data))
{
obj.Data = Client.RetrieveHtmlStrGet(obj.Url, Encoding.GetEncoding(874)).Result;
}
if (string.IsNullOrEmpty(obj.RealEstateJson))
re = new RealEstateObj();
else
re = obj.RealEstateJson.Deserialize<RealEstateObj>();
re.Url = obj.Url;
re.Source = SourceName;
var idxStart = obj.Data.IndexOf("<!-- Start Content -->");
var idxEnd = obj.Data.IndexOf("<!-- End Content -->");
if (idxStart < 0 || idxEnd < 0)
{
obj.Data = Client.RetrieveHtmlStrGet(obj.Url, Encoding.GetEncoding(874)).Result;
}
var html = obj.Data.GetStrBtw("<!-- Start Content -->", "<!-- End Content -->");
var info = html.SplitTag();
re.IsSoldOut = info[7] != "รอการขาย";
re.Map.ParcelNumber = info[11].SplitRemoveEmpty(" ");
re.Desc = info[24].StripHTML();
var province = info[20].GetMatchStr(RegexHelper.REGEX_PROVINCE);
var district = info[20].GetMatchStr(RegexHelper.REGEX_DISTRICT);
var subDistrict = info[20].GetMatchStr(RegexHelper.REGEX_SUB_DISTRICT);
var village = info[20].GetMatchStr(RegexHelper.REGEX_VILLAGE);
var road = info[20].GetMatchStr(RegexHelper.REGEX_ROAD);
var number = info[20].GetMatchStr(RegexHelper.REGEX_PROPERTYNUMBER);
var villageNo = info[20].GetMatchStr(RegexHelper.REGEX_VILLAGENO);
var alley = info[20].GetMatchStr(RegexHelper.REGEX_ALLEY);
var lane = info[20].GetMatchStr(RegexHelper.REGEX_LANE);
if (number.Count > 0)
re.Map.Number = number[0].Trim();
if (province.Count > 0)
re.Map.Province = province[0].RemovePrefix_Province();
if (district.Count > 0)
re.Map.District = district[0].RemovePrefix_District();
if (subDistrict.Count > 0)
re.Map.SubDistrict = subDistrict[0].RemovePrefix_SubDistrict();
if (village.Count > 0)
re.Map.Village = village[0].RemovePrefix_Village();
if (road.Count > 0)
re.Map.Road = road[0].RemovePrefix_Road();
if (villageNo.Count > 0)
re.Map.VillageNo = villageNo[0].RemovePrefix_VillageNo();
if (alley.Count > 0)
re.Map.Alley = alley[0].Replace("ตรอก", string.Empty).Trim();
if (lane.Count > 0)
re.Map.Lane = lane[0].RemovePrefix_Lane();
doc.LoadHtml(html);
var rows = doc.DocumentNode.Descendants("tr").ToList();
var imageRows = rows.Where(x => x.InnerHtml.Contains(@"/upload/properties/")).ToList();
if (imageRows.Count > 0)
{
var images = imageRows[1].InnerHtml.GetMatchStr(@"(\/upload\/properties\/)[a-zA-z0-9/-]{5,25}.(jpg|JPG)").Distinct().ToList();
re.Images = images;
}
if (imageRows.Count > 1)
{
var maps = imageRows[2].InnerHtml.GetMatchStr(@"(\/upload\/properties\/)[a-zA-z0-9/-]{5,25}.(jpg|JPG)").Distinct().ToList();
re.Map.Images = maps;
}
var t = new RealEstateT
{
Data = JsonHelper.Serialize(re, true),
Province = re.Map.Province,
District = re.Map.District,
ParcelNo = JsonHelper.Serialize(re.Map.ParcelNumber),
Url = re.Url.Trim(),
Lat = re.Map.Coordinate.Position.Latitude,
Lon = re.Map.Coordinate.Position.Longitude,
State = 0,
RecordStatus = 1,
Source = this.GetType().Name
};
DataHelper.InsertRealEstateT(t);
}
catch (Exception ex)
{
File.AppendAllText("D:/RE/T_GSB.log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + obj.Url + "," + ex.GetBaseException().Message + "\r\n");
}
}
protected List<string> GetUrlsFromPage(string pageUrl, string pageNumber)
{
var urls = new List<string>();
try
{
var nvc = new List<KeyValuePair<string, string>>() { new KeyValuePair<string, string>("numPage", pageNumber) };
var doc = Client.RetrieveHtmlPost(pageUrl, nvc).Result;
urls = doc.DocumentNode.Descendants("a").Where(x => x.Attributes.Contains("href") && x.Attributes["href"].Value.Contains("ViewPropServlet") && x.Attributes["href"].Value.Contains("&check=")).Select(x => URL_DTLS + x.GetAttributeValue("href", string.Empty)).ToList();
var info = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("property_all_column1"))
.Select(x => x.InnerHtml)
.Select(x => x.Replace(" ", string.Empty).Trim())
.Select(x => x.Replace("ราคาพิเศษ", string.Empty).Replace("ราคา", string.Empty).Trim().SplitTag()
).ToList();
var res = new List<RealEstateE>();
for (int i = 0; i < urls.Count; i++)
{
var json = new RealEstateObj();
//[0]: "รหัสรายการทรัพย์"
//[1]: "58CC010300798"
//[2]: "ที่ตั้งทรัพย์"
//[3]: "พระนครศรีอยุธยา"
//[4]: "อำเภอบางปะอิน"
//[5]: "ตำบลบ้านพลับ"
//[6]: "เนื้อที่"
//[7]: "0-0-80"
//[8]: "ประเภททรัพย์"
//[9]: "บ้านเดี่ยว"
//[10]: "เลขที่เอกสารสิทธิ์"
//[11]: "61919"
//[12]: "2,782,000บาท"
//[13]: "ใช้ได้ถึง"
//[14]: "31ส.ค.2559"
json.Code = info[i][1];
json.Map.Province = info[i][3].RemovePrefix_Province();
json.Map.District = info[i][4].RemovePrefix_District();
json.Map.SubDistrict = info[i][5].RemovePrefix_SubDistrict();
json.SizeTotalText = info[i][7];
json.PropertyType = info[i][9];
json.Map.ParcelNumber = info[i][11].Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
json.Price = info[i][12].RemovePrefix_Currency().ToDecimal();
json.ValidUntil = info[i][14];
json.Source = this.GetType().Name;
res.Add(new RealEstateE()
{
Url = urls[i].Trim(),
State = 0,
RecordStatus = 1,
Source = SourceName,
RealEstateJson = json.Serialize(true)
});
}
DataHelper.InsertRealEstateE(res);
}
catch (Exception ex)
{
lock (sync)
{
File.AppendAllText("D:/RE/A_" + this.GetType().Name + ".log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + pageUrl + "," + ex.GetBaseException().Message + "\r\n");
}
}
return urls;
}
public override void GetDetails(SourceObj obj)
{
var re = new RealEstateObj();
var doc = new HtmlAgilityPack.HtmlDocument();
try
{
if (string.IsNullOrEmpty(obj.Data))
{
obj.Data = Client.RetrieveHtmlStrGet(obj.Url).Result;
}
doc.LoadHtml(obj.Data);
if (string.IsNullOrEmpty(obj.RealEstateJson))
re = new RealEstateObj();
else
re = obj.RealEstateJson.Deserialize<RealEstateObj>();
re.Url = obj.Url;
re.Source = SourceName;
var detailTitles = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("property_detail_col1"))
.Select(x => RegexHelper.StripHTML(x.FirstChild.InnerHtml)).ToList();
var detailDescs = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("property_detail_col2"))
.Select(x => RegexHelper.StripHTML(x.FirstChild.InnerHtml)).ToList();
var details = new Dictionary<string, string>();
for (int i = 0; i < detailTitles.Count(); i++)
{
details.Add(detailTitles[i], detailDescs[i]);
}
//re.Code = details.ContainsKey("รหัสรายการทรัพย์") ? details["รหัสรายการทรัพย์"] : string.Empty;
//re.PropertyType = details.ContainsKey("ประเภททรัพย์") ? details["ประเภททรัพย์"] : string.Empty;
//re.Map.Province = details.ContainsKey("จังหวัด") ? details["จังหวัด"] : string.Empty;
if (details.ContainsKey("เนื้อที่(ไร่-งาน-วา)"))
{
var tmp = re.SizeTotalText.Split('-'); ;
//var tmp = details["เนื้อที่(ไร่-งาน-วา)"].Split('-');
if (tmp.Length == 3)
{
var rai = decimal.Parse(tmp[0]) * 400;
var ngn = decimal.Parse(tmp[1]) * 100;
var war = decimal.Parse(tmp[2]);
re.SizeTotal = (rai + ngn + war).ToString();
re.SizeTotalUnit = "ตร.ว.";
}
re.SizeTotalText = details["เนื้อที่(ไร่-งาน-วา)"];
}
re.DocumentOfRightType = details.ContainsKey("ประเภทเอกสารสิทธิ์") ? details["ประเภทเอกสารสิทธิ์"] : string.Empty;
re.DocumentOfRightDesc = details.ContainsKey("รายละเอียดเลขที่เอกสารสิทธิ์") ? details["รายละเอียดเลขที่เอกสารสิทธิ์"] : string.Empty;
//re.Price = details.ContainsKey("ราคาพิเศษ") ? details["ราคาพิเศษ"] : string.Empty;
re.BedRooom = details.ContainsKey("ห้องนอน") ? details["ห้องนอน"] : string.Empty;
re.BathRoom = details.ContainsKey("ห้องน้ำ") ? details["ห้องน้ำ"] : string.Empty;
re.Map.Desc = details.ContainsKey("ที่ตั้งรหัสทรัพย์") ? details["ที่ตั้งรหัสทรัพย์"] : string.Empty;
//re.BedRooom = details.Where(x => x.Key.Contains("ห้องนอน")).Count() > 0 ? string.Empty : details.Where(x => x.Key == "ห้องนอน").FirstOrDefault().Value;
//re.BathRoom = details.Where(x => x.Key.Contains("ห้องน้ำ")).Count() > 0 ? string.Empty : details.Where(x => x.Key == "ห้องน้ำ").FirstOrDefault().Value;
//re.ParkingSpace = details.Where(x => x.Key.Contains("ที่จอดรถ")).Count() > 0 ? string.Empty : details.Where(x => x.Key == "ที่จอดรถ").FirstOrDefault().Value;
//re.Price = details.Where(x => x.Key == "ราคาพิเศษ").Count() > 0 ? string.Empty : details.Where(x => x.Key == "ราคาพิเศษ").FirstOrDefault().Value;
//re.PropertyType = details.Where(x => x.Key == "ประเภททรัพย์").Count() > 0 ? string.Empty : details.Where(x => x.Key == "ประเภททรัพย์").FirstOrDefault().Value;
var contactNames = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("box_contact_col2"))
.Select(x => RegexHelper.StripHTML(x.FirstChild.InnerHtml)).ToList();
var contactTells = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("box_contact_col3"))
.Select(x => RegexHelper.StripHTML(x.FirstChild.InnerHtml)).ToList();
var contactEmails = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("box_contact_col4"))
.Select(x => RegexHelper.StripHTML(x.InnerHtml).Replace("คลิกที่นี่เพื่อติดต่อเจ้าหน้าที่", string.Empty)).ToList();
var contacts = new List<Contact>();
for (int i = 0; i < contactNames.Count(); i++)
{
contacts.Add(new Contact() { Name = contactNames[i], TellNo = contactTells.ToList(), Email = contactEmails.ToList() });
}
re.Contacts = contacts;
re.Images = doc.DocumentNode.Descendants("img").Where(x => x.Id == "image")
.Select(x => x.GetAttributeValue("src", string.Empty))
.Distinct().ToList();
//var remark = doc.DocumentNode.Descendants("div")
// .Where(x => x.Attributes.Contains("class") && (x.Attributes["class"].Value.Contains("text_contact_detail1")))
// .FirstOrDefault().InnerHtml;
//re.Remark = RegexHelper.StripHTML(WebUtility.HtmlDecode(remark));
var mapImage = doc.DocumentNode
.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("map_detail_col1")).FirstOrDefault()
.Descendants("a").FirstOrDefault();
if (mapImage != null)
{
re.Map.Images.Add(mapImage.GetAttributeValue("href", string.Empty));
}
var propId = HttpUtility.ParseQueryString(obj.Url.Split('?')[1])["propId"];
var mapUrl = USL_MAPS + propId;
var nvc = new List<KeyValuePair<string, string>>();
var mapStr = Client.RetrieveHtmlStrPost(mapUrl, nvc).Result;
var mapObj = JsonHelper.Deserialize<KTBMap>(mapStr);
if (mapObj.poi.Count > 0 && !string.IsNullOrEmpty(mapObj.poi[0].lat) && !string.IsNullOrEmpty(mapObj.poi[0].lon))
{
//re.Map.Lat = double.Parse(mapObj.poi[0].lat);
//re.Map.Lon = double.Parse(mapObj.poi[0].lon);
re.Map.Coordinate = new Point(double.Parse(mapObj.poi[0].lon), double.Parse(mapObj.poi[0].lat));
}
//var amphur = RegexHelper.GetMatchStr(obj.Data, RegexHelper.REGEX_DISTRICT);
//re.Map.ParcelNumber = RegexHelper.GetMatchStr(details["เลขที่เอกสารสิทธิ์"], RegexHelper.REGEX_NUMBER).ToArray();
//if (amphur.Count > 0)
//{
// re.Map.District = amphur[0].Replace("อำเภอ", string.Empty).Replace("เขต", string.Empty).Trim();
//}
var reT = new RealEstateT
{
Data = JsonHelper.Serialize(re, true),
Province = re.Map.Province,
District = re.Map.District,
ParcelNo = JsonHelper.Serialize(re.Map.ParcelNumber),
Url = re.Url.Trim(),
Lat = re.Map.Coordinate.Position.Latitude,
Lon = re.Map.Coordinate.Position.Longitude,
State = 0,
RecordStatus = 1,
Source = "KTB"
};
DataHelper.InsertRealEstateT(reT);
}
catch (Exception ex)
{
File.AppendAllText("D:/RE/T_KTB.log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + obj.Url + "," + ex.GetBaseException().Message + "\r\n");
}
}
public override void GetDetails(SourceObj obj)
{
Console.WriteLine(obj.Url);
var re = new RealEstateObj();
var doc = new HtmlAgilityPack.HtmlDocument();
try
{
if (string.IsNullOrEmpty(obj.Data))
{
obj.Data = Client.RetrieveHtmlStrGet(obj.Url).Result;
}
doc.LoadHtml(obj.Data);
if (string.IsNullOrEmpty(obj.RealEstateJson))
re = new RealEstateObj();
else
re = obj.RealEstateJson.Deserialize<RealEstateObj>();
re.Url = obj.Url;
re.Source = SourceName;
var titlesHtml = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("asset_name_detail"))
.Select(x => x.InnerHtml).FirstOrDefault();
if (titlesHtml != null)
{
var titles = titlesHtml.Split(new string[] { "<br>" }, StringSplitOptions.None);
re.Name = titles[0] == null ? string.Empty : titles[0].StripHTML();
}
re.Code = obj.Data.GetStrBtw("ทรัพย์รอการขาย รหัส", "<");
var imageHtml = doc.DocumentNode.Descendants("a")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("highslide")).FirstOrDefault();
if (imageHtml != null)
{
var image = URL_MAIN + imageHtml.InnerHtml.GetStrBtw("src=\"", "\"");
re.Images.Add(image);
}
var promotionHtml = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("promotion_detail_section"))
.Select(x => x.InnerHtml).FirstOrDefault();
re.Promotion = promotionHtml == null ? string.Empty : promotionHtml.StripHTML();
var detailsHtml = doc.DocumentNode.Descendants("div")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("normal_detail_section"))
.Select(x => x.InnerHtml).FirstOrDefault();
if (detailsHtml != null)
{
var details = detailsHtml.Split(new string[] { "</li>" }, StringSplitOptions.None).Select(x => x.StripHTML().CleanNewLineChar()).ToArray();
re.Name = details[1].Length < 3 ? re.Name : details[1];
re.Map.Number = details[3];
re.Map.Lane = details[5];
re.Map.Road = details[7];
re.Map.SubDistrict = details[9];
re.Map.District = details[11];
re.Map.Province = details[13];
var type = details.Where(x => x.Contains("ประเภท")).FirstOrDefault();
re.PropertyType = type == null ? string.Empty : type.CleanInfo();
var size = details.Where(x => x.Contains("เนื้อที่")).FirstOrDefault();
re.SizeTotal = size == null ? string.Empty : size.CleanInfo();
var storeys = details.Where(x => x.Contains("จำนวนชั้น")).FirstOrDefault();
re.Storeys = storeys == null ? string.Empty : storeys.CleanInfo();
var reCondition = details.Where(x => x.Contains("สภาพ")).FirstOrDefault();
re.RealEstateCondition = reCondition == null ? string.Empty : reCondition.CleanInfo();
var desc = details.Where(x => x.Contains("รายละเอียดเพิ่มเติม")).FirstOrDefault();
re.Desc = desc == null ? string.Empty : desc.CleanInfo();
var owner = details.Where(x => x.Contains("เจ้าของทรัพย์")).FirstOrDefault();
re.Owner = owner == null ? string.Empty : owner.CleanInfo();
var contact = new Contact();
var contactName = details.Where(x => x.Contains("ข้อมูลการติดต่อ")).FirstOrDefault();
var contactTel = details.Where(x => x.Contains("เบอร์ติดต่อ")).FirstOrDefault();
contact.Name = contactName == null ? string.Empty : contactName.CleanInfo();
contact.TellNo = contactTel == null ? new List<string>() : contactTel.CleanInfo().Split(new string[] { ":" }, StringSplitOptions.RemoveEmptyEntries).ToList();
re.Contacts.Add(contact);
var timestamp = details.Where(x => x.Contains("ประกาศเมื่อวันที่")).FirstOrDefault();
re.Timestamp = timestamp == null ? string.Empty : timestamp.CleanInfo();
}
var mapImageHtml = doc.DocumentNode
.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("map_boder")).FirstOrDefault();
if (mapImageHtml != null)
{
re.Map.Images.Add(URL_MAIN + mapImageHtml.InnerHtml.GetStrBtw("<img src=\"", "\""));
}
var t = new RealEstateT
{
Data = JsonHelper.Serialize(re, true),
Url = re.Url.Trim(),
Province = re.Map.Province,
District = re.Map.District,
ParcelNo = JsonHelper.Serialize(re.Map.ParcelNumber),
Lat = re.Map.Coordinate.Position.Latitude,
Lon = re.Map.Coordinate.Position.Longitude,
State = 0,
RecordStatus = 1,
Source = "GHB"
};
DataHelper.InsertRealEstateT(t);
}
catch (Exception ex)
{
File.AppendAllText("D:/RE/T_GHB.log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + obj.Url + "," + ex.GetBaseException().Message + "\r\n");
}
}
protected List<string> GetUrlsFromPage(string pageUrl)
{
var urls = new List<string>();
try
{
var html = Client.RetrieveHtmlStrGet(pageUrl, Encoding.GetEncoding(874)).Result;
urls = RegexHelper.GetMatchStr(html, "(assetdetail.asp\\?code=(.*?)\")").Distinct().Select(x => URL_MAIN + x.TrimEnd('"')).ToList();
var doc = new HtmlDocument();
doc.LoadHtml(html);
var tmp = doc.DocumentNode.Descendants("tr")
.Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.Contains("font_normal1"))
.Select(x => x.InnerHtml.StripHTML())
.Select(x => x.Replace("\r\n", string.Empty).Trim())
.ToList();
var info = tmp.Select(x => x.Split(new string[] { "\t" }, StringSplitOptions.RemoveEmptyEntries)).ToList();
//var res = urls.AsParallel().WithDegreeOfParallelism(ParallismDegree).Select(x =>
// new RealEstateE()
// {
// Url = x.Trim(),
// State = 0,
// RecordStatus = 1,
// Source = SourceName
// }
//).ToList();
var res = new List<RealEstateE>();
for (int i = 0; i < urls.Count; i++)
{
var json = new RealEstateObj()
{
Code = info[i][0],
PropertyType = info[i][1],
Name = info[i][2],
SizeTotalText = info[i][3]
};
json.Map.District = info[i][4];
json.Map.Province = info[i][5];
json.Price = info[i][7].ToDecimal();
json.Source = this.GetType().Name;
res.Add(new RealEstateE()
{
Url = urls[i].Trim(),
State = 0,
RecordStatus = 1,
Source = SourceName,
RealEstateJson = json.Serialize(true)
});
}
DataHelper.InsertRealEstateE(res);
}
catch (Exception ex)
{
lock (sync)
{
File.AppendAllText("D:/RE/A_" + this.GetType().Name + ".log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + pageUrl + "," + ex.GetBaseException().Message + "\r\n");
}
}
return urls;
}
private async Task CreateRealEstate(string databaseName, string collectionName, RealEstateObj realEstateObj)
{
try
{
await this.client.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri(databaseName, collectionName), realEstateObj);
this.WriteToConsoleAndPromptToContinue("Created RE {0}", realEstateObj.Url);
}
catch (DocumentClientException de)
{
throw de;
}
}
private async Task CreateFamilyDocumentIfNotExists(string databaseName, string collectionName, RealEstateObj realEstateObj)
{
try
{
await this.client.ReadDocumentAsync(UriFactory.CreateDocumentUri(databaseName, collectionName, realEstateObj.Url));
this.WriteToConsoleAndPromptToContinue("Found {0}", realEstateObj.Url);
}
catch (DocumentClientException de)
{
if (de.StatusCode == HttpStatusCode.NotFound)
{
await this.client.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri(databaseName, collectionName), realEstateObj);
this.WriteToConsoleAndPromptToContinue("Created Family {0}", realEstateObj.Url);
}
else
{
throw;
}
}
}