public static void ExececuteSoldUrl(string url)
{
MongoClient mongoClient = new MongoClient(MongoUrl.Create("mongodb://localhost:27017"));
IMongoDatabase mongoDatabase = mongoClient.GetDatabase("RealEstate");
IMongoCollection <BsonDocument> collection = mongoDatabase.GetCollection <BsonDocument>("SaleMap");
string responseContent = OwnHttpClient.GetResponseContent(url).Result;
int count = 0;
try
{
if (responseContent != "[]")
{
List <MongohouseMapProperty> mongoHouseMaps = Newtonsoft.Json.JsonConvert.DeserializeObject <List <MongohouseMapProperty> >(
responseContent, new JsonSerializerSettings()
{
ContractResolver = new MapContractResolver()
});
count = mongoHouseMaps.Count;
foreach (MongohouseMapProperty x in mongoHouseMaps)
{
MapJaJa map = new MapJaJa()
{
_id = ObjectId.GenerateNewId().ToString(),
nodeId = "C-" + x.MLSNo,
lat = x.Twodsphere.coordinates[0],
lng = x.Twodsphere.coordinates[1]
};
IMongoCollection <BsonDocument> pcollection = mongoDatabase.GetCollection <BsonDocument>("WaitProcessingForSoldJaJa");
var filter = new BsonDocument("nodeId", map.nodeId);
var FindResult = pcollection.Find(filter);
var bsonDoc = BsonSerializer.Deserialize <BsonDocument>(Newtonsoft.Json.JsonConvert.SerializeObject(map));
if (FindResult.CountDocuments() == 0)
{
pcollection.InsertOne(bsonDoc);
}
}
}
IMongoCollection <BsonDocument> exeCollection = mongoDatabase.GetCollection <BsonDocument>("MongoHouseSoldUrls");
var mfilter = new BsonDocument("Url", url);
List <BsonDocument> bdocs = exeCollection.Find(mfilter).ToList();
foreach (BsonDocument doc in bdocs)
{
doc["QueryResult"] = doc["QueryResult"] + "-" + count.ToString();
exeCollection.ReplaceOne(mfilter, doc);
System.Diagnostics.Debug.WriteLine(doc["Url"].ToString());
}
}
catch (Exception) { }
}
public static void GetLeaseHtml(MapJaJa mls, string updateDate)
{
mls.Html = Details(mls.nodeId);
#region Save to MongoDB
MongoClient mongoClient = new MongoClient(MongoUrl.Create("mongodb://localhost:27017"));
IMongoDatabase mongoDatabase = mongoClient.GetDatabase("RealEstate");
IMongoCollection <MapJaJa> collection = mongoDatabase.GetCollection <MapJaJa>("WaitProcessingForLeaseJaJa");
IMongoCollection <MapJaJa> pcollection = mongoDatabase.GetCollection <MapJaJa>("WaitProcessingForLeaseJaJaHTML");
//var FindResult = collection.Find(c => c.nodeId == mls.nodeId).First();
collection.DeleteOne(c => c.nodeId == mls.nodeId);
pcollection.InsertOne(mls);
#endregion
}
public static void GetSoldHtml(MapJaJa mls, string updateDate)
{
mls.Html = RealJaJa.Details(mls.nodeId);
mls.MongohouseData = MLSSearch(mls.nodeId.Substring(2));
#region Save to MongoDB
MongoClient mongoClient = new MongoClient(MongoUrl.Create("mongodb://localhost:27017"));
IMongoDatabase mongoDatabase = mongoClient.GetDatabase("RealEstate");
IMongoCollection <MapJaJa> collection = mongoDatabase.GetCollection <MapJaJa>("WaitProcessingForSoldJaJa");
IMongoCollection <MapJaJa> pcollection = mongoDatabase.GetCollection <MapJaJa>("WaitProcessingForSoldJaJaHTML");
collection.DeleteOne(c => c.nodeId == mls.nodeId);
if (mls.MongohouseData.Trim() != "{}")
{
pcollection.InsertOne(mls);
}
#endregion
}
public static void GetSaleData(MapJaJa mls, string updateDate)
{
try
{
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(mls.Html);
PropertyDetailByJaJa houseDetail = new PropertyDetailByJaJa();
MongoClient mongoClient = new MongoClient(MongoUrl.Create("mongodb://localhost:27017"));
IMongoDatabase db = mongoClient.GetDatabase("RealEstate");
#region Data Processing
houseDetail._id = ObjectId.GenerateNewId().ToString();
houseDetail.JmlsNo = mls.nodeId;
List <HtmlNode> mapLocation = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "text-white").ToList <HtmlNode>();
houseDetail.Location = new Twodsphere();
houseDetail.Location = new Twodsphere()
{
type = "Point", coordinates = new List <double>()
{
mls.lat, mls.lng
}
};
houseDetail.UpdateDate = updateDate;
// Address
List <HtmlNode> divAddress = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "div-address").ToList <HtmlNode>();
if (divAddress.Count > 0)
{
houseDetail.Address = divAddress[0].InnerText.Replace("\n", "").Trim();
}
// Price
List <HtmlNode> divPrice = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "text-ui").ToList <HtmlNode>();
if (divPrice.Count > 0)
{
houseDetail.ListingPrice = divPrice[0].InnerText.Replace("CAD", "").Replace("$", "").Replace(",", "").Trim();
}
List <HtmlNode> divBedroom = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "fa-bed").ToList <HtmlNode>();
List <HtmlNode> divDetail = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "div-detail").ToList <HtmlNode>();
if (divDetail.Count == 0)
{
return;
}
List <HtmlNode> properties = HtmlParserHelper.GetElementsWithClass(divDetail[0], "col-xs-6").ToList <HtmlNode>();
houseDetail.Properties = new List <Property>();
for (int i = 0; i < properties.Count; i = i + 2)
{
houseDetail.Properties.Add(new Property()
{
propertyName = properties[i].InnerText.Replace("\n", "").Trim(), Value = properties[i + 1].InnerText.Replace("\n", "").Trim()
});
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("City"))
{
houseDetail.City = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Community"))
{
houseDetail.Community = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("District"))
{
houseDetail.District = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("MLS"))
{
houseDetail.MlsNo = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Bed Rooms"))
{
houseDetail.Bedrooms = properties[i + 1].InnerText.Replace("\n", "").Trim().Replace(" ", "");
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Bath Rooms"))
{
houseDetail.Washrooms = properties[i + 1].InnerText.Replace("\n", "").Trim().Replace(" ", "");
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Gar Spaces"))
{
houseDetail.Garage = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Listed Date"))
{
houseDetail.ListingDate = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Delisted Date"))
{
houseDetail.TransactionDate = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Appr. Footage"))
{
houseDetail.Footage = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
if (properties[i].InnerText.Replace("\n", "").Trim().StartsWith("Property Type"))
{
houseDetail.HouseType = properties[i + 1].InnerText.Replace("\n", "").Trim();
}
}
if (houseDetail.MlsNo == null)
{
return;
}
List <HtmlNode> Type = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "rm_out").ToList <HtmlNode>();
List <HtmlNode> Level = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "rm_lvl").ToList <HtmlNode>();
List <HtmlNode> Dimensions = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "rm_dem").ToList <HtmlNode>();
List <HtmlNode> Desc1 = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "rm_dc1").ToList <HtmlNode>();
List <HtmlNode> Desc2 = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "rm_dc2").ToList <HtmlNode>();
List <HtmlNode> Desc3 = HtmlParserHelper.GetElementsWithClass(htmlDoc.DocumentNode, "rm_dc3").ToList <HtmlNode>();
houseDetail.RoomTypes = new List <RoomType>();
for (int i = 0; i < Type.Count; i = i + 1)
{
houseDetail.RoomTypes.Add(new RoomType()
{
Type = Type[i].InnerText.Replace("\n", "").Trim(),
Level = Level[i].InnerText.Replace("\n", "").Trim(),
Dimensions = Dimensions[i].InnerText.Replace("\n", "").Trim(),
Desc1 = Desc1[i].InnerText.Replace("\n", "").Trim(),
Desc2 = Desc2[i].InnerText.Replace("\n", "").Trim(),
Desc3 = Desc3[i].InnerText.Replace("\n", "").Trim(),
});
}
if (houseDetail.RoomTypes.Count > 0)
{
houseDetail.RoomTypes.RemoveAt(0);
}
List <HtmlNode> schools = HtmlParserHelper.GetElementsByID(htmlDoc.DocumentNode, "school_collapse").ToList <HtmlNode>();
if (schools.Count > 0)
{
List <HtmlNode> schoolMaster = HtmlParserHelper.GetElementsWithClass(schools[0], "col-xs-9").ToList <HtmlNode>();
List <HtmlNode> schoolRank = HtmlParserHelper.GetElementsWithClass(schools[0], "col-xs-3").ToList <HtmlNode>();
houseDetail.Schools = new List <School>();
for (int i = 0; i < schoolMaster.Count; i = i + 1)
{
string name = schoolMaster[i].ChildNodes[1].InnerText.Replace("\n", "").Trim();
string detail = schoolMaster[i].ChildNodes[3].InnerText.Replace("\n", "").Trim();
string rank = schoolRank[i].InnerText.Replace("\n", "").Trim();
houseDetail.Schools.Add(new School()
{
Name = name, Detail = detail, Ranking = rank
});
}
}
List <HtmlNode> picturesSection = HtmlParserHelper.GetElementsByID(htmlDoc.DocumentNode, "swiper_gallery").ToList <HtmlNode>();
if (picturesSection.Count == 0)
{
return;
}
List <HtmlNode> pictures = HtmlParserHelper.GetElementsByElementType(picturesSection[0], "img").ToList <HtmlNode>();
houseDetail.Pictures = new List <Picture>();
for (int i = 0; i < pictures.Count; i = i + 1)
{
string name = pictures[i].GetAttributeValue("src", null);
if (name != null)
{
name = name.Substring(name.LastIndexOf("/"));
name = name.Substring(1);
if (name.LastIndexOf("-") > 0)
{
name = name.Substring(0, name.LastIndexOf("-"));
}
if (name.LastIndexOf("_") > 0)
{
name = name.Substring(0, name.LastIndexOf("_"));
}
name = string.Format("http://photos.v3.torontomls.net/Live/photos/FULL/{0}/{1}/{2}{3}.jpg",
i + 1, houseDetail.MlsNo.Substring(houseDetail.MlsNo.Length - 3), houseDetail.MlsNo, (i == 0) ? "" : "_" + (i + 1).ToString());
houseDetail.Pictures.Add(new Picture()
{
url = name
});
}
}
#endregion
try
{
string rawJson = mls.MongohouseData;
dynamic task = JObject.Parse(rawJson);
houseDetail.LastStatus = task["Last Status"];
houseDetail.TransactionDate = task["Sold Date"];
houseDetail.TransactionPrice = task["Sold"];
houseDetail.TransactionPrice = houseDetail.TransactionPrice.Replace("$", "").Replace(",", "");
houseDetail.MCity = task["City"];
houseDetail.MAddress1 = task["Address1"];
houseDetail.MAddress2 = task["Address2"];
}
catch (Exception)
{
}
IMongoCollection <PropertyDetailByJaJa> collection = db.GetCollection <PropertyDetailByJaJa>("ForSale");
var obj = collection.Find(c => c.MlsNo == houseDetail.MlsNo).FirstOrDefault();
if (obj == null)
{
collection.InsertOne(houseDetail);
}
else
{
houseDetail._id = obj._id;
collection.ReplaceOne(c => c.MlsNo == houseDetail.MlsNo, houseDetail);
}
IMongoCollection <MapJaJa> mapcollection = db.GetCollection <MapJaJa>("WaitProcessingForSaleJaJaHTML");
var mfilter = new BsonDocument("nodeId", mls.nodeId);
mapcollection.DeleteOne(mfilter);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
