public void Update(TEntity entity, int key)
{
if (entity == null)
{
return;
}
var existing = Context.Set <TEntity>().Find(key);
if (existing != null)
{
if (entity.GetType().GetProperty("DateModified") != null)
{
entity.GetType().GetProperty("DateModified").SetValue(entity, DateTime.UtcNow);
}
if (entity.GetType().GetProperty("Active") != null)
{
entity.GetType().GetProperty("Active").SetValue(entity, true);
}
Context.Entry(existing).CurrentValues.SetValues(entity);
}
}
public void ProcessProperty(Portal portal, Url url, UrlType urlType, ProcessPostalCode processPostalCode, ProxyIp proxyIp)
{
var _appContext = new PropertyCrawler.Data.AppContext(true);
try
{
var proxy = new WebProxy()
{
Address = new Uri($"http://{proxyIp.Ip}"),/*:{proxyPort}*/
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
// *** These creds are given to the proxy server, not the web server ***
Credentials = new NetworkCredential(
userName: proxyIp.Username,
password: proxyIp.Password)
};
// Now create a client handler which uses that proxy
var httpClientHandler = new HttpClientHandler()
{
Proxy = proxy,
};
using (var client = new HttpClient(handler: httpClientHandler, disposeHandler: true))
{
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36");
client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9,it;q=0.8,sq;q=0.7");
var links = client.GetStringAsync(portal.Url + urlType.UrlPortion + url.PropertyCode + ".html").Result;
HtmlDocument document = new HtmlDocument();
document.LoadHtml(links);
if (document.DocumentNode.InnerText.Contains("This property has been removed by the agent.") ||
document.DocumentNode.InnerText.Contains("We are sorry but we could not find the property you have requested."))
{
url.DateModified = DateTime.UtcNow;
url.Active = false;
_appContext.Entry(url).State = EntityState.Modified;
_appContext.Urls.Update(url);
_appContext.SaveChanges();
}
else
{
var propertyInfo = @"'property',";
var branchInfo = @"'branch',";
var imagesInfo = @"images : ";
var data = document.DocumentNode.SelectNodes("//script").FirstOrDefault(x => x.InnerHtml.Contains(propertyInfo));
var jsonString = data.InnerText.Split(@"'property',")[1].Trim();
var jsonData = jsonString.Remove(jsonString.Length - 3, 3);
var propertyData = JsonConvert.DeserializeObject <Details>(jsonData);
var branchData = document.DocumentNode.SelectNodes("//script").FirstOrDefault(x => x.InnerText.Contains(branchInfo));
jsonString = branchData.InnerText.Split(@"'branch',")[1].Trim();
jsonData = jsonString.Remove(jsonString.Length - 3, 3);
var agentData = JsonConvert.DeserializeObject <Branch>(jsonData);
var imagesData = document.DocumentNode.SelectNodes("//script").FirstOrDefault(x => x.InnerText.Contains(imagesInfo));
jsonString = imagesData.InnerText.Split(imagesInfo)[1].Trim();
jsonData = jsonString.Split("\"}],")[0] + "\"}]";
var imageData = jsonString.Split("\"}],").Length > 1 ? JsonConvert.DeserializeObject <List <Images> >(jsonData) : new List <Images>();
//Description itemprop="description" itemprop="description"
var description = document.DocumentNode.SelectSingleNode(".//p[@itemprop=\"description\"]").InnerText.Trim();
//Full address //address itemprop=address
var address = document.DocumentNode.SelectSingleNode(".//address[@itemprop=\"address\"]").InnerText.Trim();
//agent phone number// class branch-telephone-number
var agentPhoneNumber = document.DocumentNode.SelectSingleNode(".//a[@class=\"branch-telephone-number\"]").InnerText.Trim();
var dateNow = DateTime.UtcNow;
var priceValue = !string.IsNullOrEmpty(propertyData.propertyInfo.price) ? (Decimal.TryParse(propertyData.propertyInfo.price, out decimal tem) ? tem : 0) : 0;
//agent
var existAgent = _appContext.Agents.FirstOrDefault(x => x.AgentCode == agentData.branchId && x.Active);
if (existAgent == null)
{
existAgent = new Agent
{
DateAdded = dateNow,
DateModified = dateNow,
DisplayAddress = agentData.displayAddress,
Active = true,
AgentCode = agentData.branchId,
AgentType = agentData.agentType,
BranchName = agentData.branchName,
BranchPostcode = agentData.branchPostcode,
BrandName = agentData.brandName,
CompanyName = agentData.companyName,
CompanyType = agentData.companyType,
PhoneNumber = agentPhoneNumber,
};
}
//description
var propertyDescription = new PropertyDescription
{
DateAdded = dateNow,
DateModified = dateNow,
Description = description,
Active = true
};
//price
var price = new PropertyPrice
{
DateAdded = dateNow,
DateModified = dateNow,
Price = priceValue,
PriceQualifier = propertyData.propertyInfo.priceQualifier,
Active = true
};
//Images
var images = imageData.Select(x => new PropertyCrawler.Data.Image
{
DateAdded = dateNow,
DateModified = dateNow,
Active = true,
Caption = x.caption,
Url = x.masterUrl
}).ToList();
var postC = propertyData.location.postcode?.Trim();
//property
CultureInfo enUS = new CultureInfo("en-US");
var property = new PropertyCrawler.Data.Property
{
DateAdded = dateNow,
DateModified = dateNow,
Active = true,
LettingType = propertyData.propertyInfo.lettingType,
PropertyType = propertyData.propertyInfo.propertyType,
PostalCode = propertyData.location.postcode,
PostalCodeFull = postC.Substring(0, postC.Length - 2),
PostalCodePrefix = postC.Substring(0, 2),
PostalCodeExtended = postC.Substring(0, postC.Length - 3).Trim(),
Type = (PropertyType)url.Type,
PropertyAdded = DateTime.TryParseExact(propertyData.propertyInfo.added, "yyyyMMdd", enUS, DateTimeStyles.AssumeLocal, out DateTime temp) ? temp : DateTime.MinValue,
Latitude = propertyData.location.latitude,
Longtitude = propertyData.location.longitude,
Address = address,
Price = priceValue,
BedroomsCount = (byte)propertyData.propertyInfo.beds,
FloorPlanCount = propertyData.floorplanCount,
PropertySubType = propertyData.propertyInfo.propertySubType,
UrlId = url.Id
};
property.Images.AddRange(images);
property.PropertyPrices.Add(price);
if (existAgent.Id != 0)
{
property.AgentId = existAgent.Id;
}
else
{
property.Agent = existAgent;
}
property.PropertyDescription = propertyDescription;
_appContext.Properties.Add(property);
// _appContext.PropertyDescriptions.Add(propertyDescription);
_appContext.SaveChanges();
}
}
}
catch (Exception ex)
{
var dateNow = DateTime.UtcNow;
var processPostalCodeUrlFailed = new ProcessPostalCodeUrlFailed
{
DateAdded = dateNow,
Active = true,
DateModified = dateNow,
UrlId = url.Id,
ProcessPostalCodeId = processPostalCode.Id,
FailReason = ex.Message
};
_appContext.ProcessPostalCodeUrlFails.Add(processPostalCodeUrlFailed);
_appContext.SaveChanges();
}
}
static void ProcessDetails(Url url)
{
var _appContext = new Data.AppContext(true);
//sale example
//var basedUrl = "https://www.rightmove.co.uk/property-for-sale/property-56553534.html";
var basedUrl = "https://www.rightmove.co.uk";
//rent example
try
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36");
client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9,it;q=0.8,sq;q=0.7");
var links = client.GetStringAsync(basedUrl /*+url.PropertyUrl*/).Result;
HtmlDocument document = new HtmlDocument();
document.LoadHtml(links);
if (document.DocumentNode.InnerText.Contains("This property has been removed by the agent.") ||
document.DocumentNode.InnerText.Contains("We are sorry but we could not find the property you have requested."))
{
url.DateModified = DateTime.UtcNow;
url.Active = false;
_appContext.Entry(url).State = EntityState.Modified;
_appContext.Urls.Update(url);
_appContext.SaveChanges();
}
else
{
var propertyInfo = @"'property',";
var branchInfo = @"'branch',";
var imagesInfo = @"images : ";
var data = document.DocumentNode.SelectNodes("//script").FirstOrDefault(x => x.InnerHtml.Contains(propertyInfo));
var jsonString = data.InnerText.Split(@"'property',")[1].Trim();
var jsonData = jsonString.Remove(jsonString.Length - 3, 3);
var propertyData = JsonConvert.DeserializeObject <Details>(jsonData);
var branchData = document.DocumentNode.SelectNodes("//script").FirstOrDefault(x => x.InnerText.Contains(branchInfo));
jsonString = branchData.InnerText.Split(@"'branch',")[1].Trim();
jsonData = jsonString.Remove(jsonString.Length - 3, 3);
var agentData = JsonConvert.DeserializeObject <Branch>(jsonData);
var imagesData = document.DocumentNode.SelectNodes("//script").FirstOrDefault(x => x.InnerText.Contains(imagesInfo));
jsonString = imagesData.InnerText.Split(imagesInfo)[1].Trim();
jsonData = jsonString.Split("\"}],")[0] + "\"}]";
var imageData = jsonString.Split("\"}],").Length > 1? JsonConvert.DeserializeObject <List <Images> >(jsonData): new List <Images>();
//Description itemprop="description" itemprop="description"
var description = document.DocumentNode.SelectSingleNode(".//p[@itemprop=\"description\"]").InnerText.Trim();
//Full address //address itemprop=address
var address = document.DocumentNode.SelectSingleNode(".//address[@itemprop=\"address\"]").InnerText.Trim();
//agent phone number// class branch-telephone-number
var agentPhoneNumber = document.DocumentNode.SelectSingleNode(".//a[@class=\"branch-telephone-number\"]").InnerText.Trim();
var dateNow = DateTime.UtcNow;
var priceValue = !string.IsNullOrEmpty(propertyData.propertyInfo.price) ? (Decimal.TryParse(propertyData.propertyInfo.price, out decimal tem) ? tem : 0) : 0;
//agent
var existAgent = _appContext.Agents.FirstOrDefault(x => x.AgentCode == agentData.branchId && x.Active);
if (existAgent == null)
{
existAgent = new Agent
{
DateAdded = dateNow,
DateModified = dateNow,
DisplayAddress = agentData.displayAddress,
Active = true,
AgentCode = agentData.branchId,
AgentType = agentData.agentType,
BranchName = agentData.branchName,
BranchPostcode = agentData.branchPostcode,
BrandName = agentData.brandName,
CompanyName = agentData.companyName,
CompanyType = agentData.companyType,
PhoneNumber = agentPhoneNumber,
};
}
//description
var propertyDescription = new PropertyDescription
{
DateAdded = dateNow,
DateModified = dateNow,
Description = description,
Active = true
};
//price
var price = new PropertyPrice
{
DateAdded = dateNow,
DateModified = dateNow,
Price = priceValue,
PriceQualifier = propertyData.propertyInfo.priceQualifier,
Active = true
};
//Images
var images = imageData.Select(x => new Image
{
DateAdded = dateNow,
DateModified = dateNow,
Active = true,
Caption = x.caption,
Url = x.masterUrl
}).ToList();
var postC = propertyData.location.postcode?.Trim();
//property
var property = new Property
{
DateAdded = dateNow,
DateModified = dateNow,
Active = true,
LettingType = propertyData.propertyInfo.lettingType,
PropertyType = propertyData.propertyInfo.propertyType,
PostalCode = propertyData.location.postcode,
PostalCodeFull = postC.Substring(0, postC.Length - 2),
PostalCodePrefix = postC.Substring(0, 2),
PostalCodeExtended = postC.Substring(0, postC.Length - 3).Trim(),
Type = (PropertyType)url.Type,
PropertyAdded = DateTime.TryParse(propertyData.propertyInfo.added, out DateTime temp)?temp:DateTime.MinValue,
Latitude = propertyData.location.latitude,
Longtitude = propertyData.location.longitude,
Added = propertyData.propertyInfo.added,
Address = address,
Price = priceValue,
BedroomsCount = (byte)propertyData.propertyInfo.beds,
FloorPlanCount = propertyData.floorplanCount,
PropertySubType = propertyData.propertyInfo.propertySubType,
UrlId = url.Id
};
property.Images.AddRange(images);
property.PropertyPrices.Add(price);
if (existAgent.Id != 0)
{
property.AgentId = existAgent.Id;
}
else
{
property.Agent = existAgent;
}
property.PropertyDescription = propertyDescription;
_appContext.Properties.Add(property);
_appContext.SaveChanges();
}
}
}
catch (Exception ex)
{
}
}