private void LoadCars(CarMaxStore store)
{
var searchUrl = store.InventoryUrl;
while (!String.IsNullOrEmpty(searchUrl))
{
var content = WebHandler.DownloadContent(searchUrl);
var xmlDocument = WebHandler.DownloadDocument(content);
var aNodes = xmlDocument.SelectNodes("//div[@id='resultsList']/div[@class='car']");
if (aNodes != null)
{
foreach (XmlNode node in aNodes)
{
var carId = Convert.ToInt64(node.Attributes["sn"].Value);
try
{
_commonManagementForm.UpdateCarMaxVehicleStore(carId, store.StoreId);
}
catch (Exception ex)
{
_logging.Error("CARMAX - InsertVehicleToDatabase", ex);
Console.WriteLine("CARMAX - InsertVehicleToDatabase {0}", ex.Message);
}
}
}
var nextNode = xmlDocument.SelectSingleNode("//li[@id='next']");
searchUrl = nextNode != null && nextNode.Attributes["data-url"] != null ? store.InventoryUrl + "&" + nextNode.Attributes["data-url"].Value : string.Empty;
}
}
private List <CarMaxMake> LoadMakes()
{
var list = new List <CarMaxMake>();
var content = WebHandler.DownloadContent("http://www.carmax.com/enus/car-search/used-cars.html");
var xmlDocument = WebHandler.DownloadDocument(content);
//var nsManager = WebHandler.GetXmlNamespaceManager(xmlDocument);
var makeNodes = xmlDocument.SelectNodes("//*[@id='make']/*[.]");
if (makeNodes != null)
{
foreach (XmlNode node in makeNodes)
{
var makeName = node.InnerText.Trim();
if (new string[] { "All Makes", "All Domestic", "All Imports" }.Contains(makeName))
{
continue;
}
var make = new CarMaxMake();
make.Value = Convert.ToInt64(node.Attributes["value"].Value);
make.Name = makeName;
make.Models = LoadModels(make.Value);
list.Add(make);
}
}
return(list);
}
public void InsertStoresIntoDatabase()
{
//_logging.Info("*** START to scrap Car Max Stores ***");
Console.WriteLine("*** START to scrap Car Max Stores ***");
var states = LoadStates();
foreach (var state in states)
{
var stateUrl = String.Format("https://www.carmax.com/stores/states/{0}", state.Value);
var content = WebHandler.DownloadContent(stateUrl);
var xmlDocument = WebHandler.DownloadDocument(content);
var storeNodes = xmlDocument.SelectNodes("//div[@class='store-locator--result']");
if (storeNodes != null)
{
foreach (XmlNode node in storeNodes)
{
try
{
var newStore = GetStoreDetail(node);
_commonManagementForm.AddNewCarMaxStore(newStore);
Console.WriteLine("{0}", newStore.Name);
}
catch (Exception ex)
{
//_logging.Error(String.Format("Failed to insert store at {0}", stateUrl), ex);
Console.WriteLine(String.Format("Failed to insert store at {0}", stateUrl), ex);
}
}
}
}
//_logging.Info("*** END to scrap Car Max Stores ***");
Console.WriteLine("*** END to scrap Car Max Stores ***");
}
private List <CarMaxModel> LoadModels(long makeId)
{
var list = new List <CarMaxModel>();
var url = "https://api.carmax.com/v1/api/vehicles/models/4294963045?platform=carmax.com&apikey=adfb3ba2-b212-411e-89e1-35adab91b600";
var content = WebHandler.DownloadContent(url);
var xmlDocument = WebHandler.DownloadDocument(content);
var modelNodes = xmlDocument.SelectNodes("//RefinementModel");
//var request = (HttpWebRequest)WebRequest.Create("http://www.carmax.com/data/LookupOASCapable.aspx/GetFilteredModelOptions");
//request.Method = "POST";
//request.Accept = "application/atom+xml,application/xml";
//request.UserAgent = UserAgent;
//var postData = CreateDataToLoadModel(new CarMaxModelPost()
//{
// makeId = makeId,
// typeId = 0,
// usedNew = "u"
//});
//request.ContentLength = postData.Length;
//request.ContentType = "application/json; charset=utf-8";
//request.Referer = "http://www.carmax.com/data/LookupOASCapable.aspx/GetFilteredModelOptions";
//// Post to the login form.
//var streamWriter = new StreamWriter(request.GetRequestStream());
//streamWriter.Write(postData);
//streamWriter.Close();
//// Get the response.
//var response = (HttpWebResponse)request.GetResponse();
//// Read the response
//var streamReader = new StreamReader(response.GetResponseStream());
//var result = streamReader.ReadToEnd();
//streamReader.Close();
//var jsonObj = (JObject)JsonConvert.DeserializeObject(result);
//var xmlDocument = WebHandler.DownloadDocument(Convert.ToString(((JValue)(jsonObj["d"])).Value));
//var list = new List<CarMaxModel>();
//var modelNodes = xmlDocument.SelectNodes("//*[.]");
//if (modelNodes != null)
//{
// foreach (XmlNode node in modelNodes)
// {
// var modelName = node.InnerText.Trim();
// if (node.Name.Equals("html") || new string[] { "All Models" }.Contains(modelName)) continue;
// var model = new CarMaxModel();
// model.Value = Convert.ToInt64(node.Attributes["value"].Value);
// model.Name = node.InnerText.Trim();
// list.Add(model);
// }
//}
return(list);
}
private CommercialTruckDealerViewModel ParseStandardDealer(XmlNode node)
{
var id = 0;
var nameNode = node.SelectSingleNode("./*[@class='lfloat bscDlrInfo']/span");
var name = nameNode != null?nameNode.InnerText.Trim() : string.Empty;
if (String.IsNullOrEmpty(name))
{
return(null);
}
var addressNode = node.SelectSingleNode("./*[@class='lfloat bscDlrInfo']");
var addressContent = addressNode != null ? addressNode.InnerXml : string.Empty;
var addresscitystatezipcodeRegex = new Regex(@"php\?(.+)\&");
Match addresscitystatezipcodeMatch = addresscitystatezipcodeRegex.Match(addressContent);
var addresscitystatezipcode = addresscitystatezipcodeMatch.Groups.Count > 0 ? HttpUtility.HtmlDecode(addresscitystatezipcodeMatch.Groups[1].Value) : string.Empty;
var temp = addresscitystatezipcode.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
var address = temp[0].Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1];
var city = temp[1].Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1];
var state = temp[2].Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1];
var zipcode = Convert.ToInt32(temp[3].Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1]);
if (id.Equals(0))
{
var idRegex = new Regex(@"dealerId\=(.+)\',\s");
Match idMatch = idRegex.Match(addressContent);
id = idMatch.Groups.Count > 0 ? Convert.ToInt32(idMatch.Groups[1].Value) : 0;
}
var mapUrl = String.Format("http://www.commercialtrucktrader.com/getmap.php?address={0}&city={1}&stateabbr={2}&zip={3}&dealerId={4}", address, city, state, zipcode, id.ToString());
var mapContent = WebHandler.DownloadContent(mapUrl);
var latlongRegex = new Regex(@"VELatLong\((.+)\);");
Match latlongMatch = latlongRegex.Match(mapContent);
var latlong = latlongMatch.Groups.Count > 0 ? latlongMatch.Groups[1].Value : string.Empty;
var lat = 0.0;
double.TryParse(latlong.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[0], out lat);
var log = 0.0;
double.TryParse(latlong.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1], out log);
return(new CommercialTruckDealerViewModel()
{
TruckDealerId = id,
Name = name,
Address = address,
City = city,
State = state,
ZipCode = zipcode,
Latitude = lat,
Longitude = log,
DateStamp = DateTime.Now
});
}
private CommercialTruckDealerViewModel ParseFeaturedDealer(XmlNode node)
{
var id = node.Attributes != null && node.Attributes["id"] != null?Convert.ToInt32(node.Attributes["id"].Value) : 0;
var nameNode = node.SelectSingleNode("./*[@class='lfloat prmDlrLft']/*[@class='prmDlrLogo']/*[@class='prmDlrLogoFx']");
var name = nameNode != null?nameNode.InnerText.Trim() : string.Empty;
if (String.IsNullOrEmpty(name))
{
return(null);
}
var addressNode = node.SelectSingleNode("./*[@class='lfloat prmDlrLft']/*[@class='prmDlrAddy']");
var addressContent = addressNode != null ? addressNode.InnerXml : string.Empty;
var addressRegex = new Regex(@"(.*)<br\s/>");
Match addressMatch = addressRegex.Match(addressContent);
var address = addressMatch.Groups.Count > 0 ? addressMatch.Groups[1].Value : string.Empty;
var citystatezipcodeRegex = new Regex(@"<br\s/>(.+)\s\[");
Match citystatezipcodeMatch = citystatezipcodeRegex.Match(addressContent);
var citystatezipcode = citystatezipcodeMatch.Groups.Count > 0 ? citystatezipcodeMatch.Groups[1].Value : string.Empty;
var temp = citystatezipcode.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
var city = temp[0];
var state = temp[1].Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[0];
var zipcode = Convert.ToInt32(temp[1].Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1]);
var mapUrl = String.Format("http://www.commercialtrucktrader.com/getmap.php?address={0}&city={1}&stateabbr={2}&zip={3}&dealerId={4}", address, city, state, zipcode, id.ToString());
var mapContent = WebHandler.DownloadContent(mapUrl);
var latlongRegex = new Regex(@"VELatLong\((.+)\);");
Match latlongMatch = latlongRegex.Match(mapContent);
var latlong = latlongMatch.Groups.Count > 0 ? latlongMatch.Groups[1].Value : string.Empty;
var lat = 0.0;
double.TryParse(latlong.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[0], out lat);
var log = 0.0;
double.TryParse(latlong.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries).ToArray()[1], out log);
var phoneNode = node.SelectSingleNode("./*[@class='lfloat prmDlrLft']/*[@class='prmDlrPhone']/ul/li");
var phone = phoneNode != null?phoneNode.InnerText.Replace("Toll Free: ", "") : string.Empty;
return(new CommercialTruckDealerViewModel()
{
TruckDealerId = id,
Name = name,
Address = address,
City = city,
State = state,
ZipCode = zipcode,
Phone = phone,
Latitude = lat,
Longitude = log,
DateStamp = DateTime.Now
});
}
public List <CommercialTruckDealerViewModel> GetDealerInfo(List <string> urls)
{
var dealers = new List <CommercialTruckDealerViewModel>();
foreach (var url in urls)
{
var nextPageUrl = url;
while (!string.IsNullOrEmpty(nextPageUrl))
{
var content = WebHandler.DownloadContent(nextPageUrl.Contains(Host) ? nextPageUrl : String.Format("{0}{1}", Host, nextPageUrl));
Console.WriteLine(nextPageUrl);
var xmlDocument = WebHandler.DownloadDocument(content);
if (xmlDocument == null)
{
break;
}
var dealerNodes = xmlDocument.SelectNodes("//*[@class='gdBx2 prmDlr' or @class='gdBx2 bscDlr']");
if (dealerNodes == null)
{
break;
}
foreach (XmlNode node in dealerNodes)
{
try
{
var id = node.Attributes != null && node.Attributes["id"] != null?Convert.ToInt32(node.Attributes["id"].Value) : 0;
if (id.Equals(0))
{
dealers.Add(ParseStandardDealer(node));
}
else
{
dealers.Add(ParseFeaturedDealer(node));
}
}
catch (Exception ex)
{
Console.WriteLine("ERROR when parsing dealer info {0}", ex.Message + " " + ex.StackTrace);
}
}
var nextPageNode = xmlDocument.SelectSingleNode("//*[@id='sortnpaginate']/*[@class='lfloat paginate']/a[@title='Next Page']/@href");
nextPageUrl = nextPageNode != null ? nextPageNode.Value : string.Empty;
InsertTruckDealerIntoDatabase(dealers);
dealers.Clear();
}
}
return(dealers);
}
public List <string> GetAllCategories()
{
var list = new List <string>();
var content = WebHandler.DownloadContent("http://www.commercialtrucktrader.com");
var xmlDocument = WebHandler.DownloadDocument(content);
//var nsManager = WebHandler.GetXmlNamespaceManager(xmlDocument);
var stateNodes = xmlDocument.SelectNodes("//*[@id='typeAny']/option");
if (stateNodes == null)
{
return(list);
}
list.AddRange(from XmlNode node in stateNodes select node.Value);
return(list);
}
private void GetTrucks(string url, string category = "Other Truck")
{
var list = new List <CommercialTruckViewModel>();
var nextPageUrl = url;
while (!string.IsNullOrEmpty(nextPageUrl))
{
Console.WriteLine("PAGE: {0}", nextPageUrl);
var content = WebHandler.DownloadContent(nextPageUrl, 2);
if (String.IsNullOrEmpty(content))
{
break;
}
var xmlDocument = WebHandler.DownloadDocument(content);
if (xmlDocument == null)
{
break;
}
var truckNodes = xmlDocument.SelectNodes("//*[@class='listing']");
if (truckNodes == null)
{
break;
}
foreach (XmlNode node in truckNodes)
{
try
{
var newTruck = GetDetailTruck(node);
newTruck.BodyStyle = category;
list.Add(newTruck);
Console.WriteLine("{0} -- {1} {2} {3}", list.Count, newTruck.Year, newTruck.Make, newTruck.Model);
}
catch (Exception ex)
{
Console.WriteLine("ERROR: {0}", ex.StackTrace);
}
}
var nextPageNode = xmlDocument.SelectSingleNode("//a[@class='listings-pag-selected']").NextSibling;
nextPageUrl = nextPageNode != null && !nextPageNode.InnerText.Equals("Next") ? (nextPageNode.InnerText.Equals(". . .") ? Host + xmlDocument.SelectSingleNode("//a[@class='listings-pag-next-selectexpand']").Attributes["href"].Value : Host + nextPageNode.Attributes["href"].Value) : string.Empty;
InsertTruckIntoDatabase(list);
list.Clear();
}
}
public List <string> GetStateSearchUrls()
{
const string findUrl = "http://www.commercialtrucktrader.com/find";
var list = new List <string>();
var content = WebHandler.DownloadContent("http://www.commercialtrucktrader.com/find/dealer-search/");
var xmlDocument = WebHandler.DownloadDocument(content);
//var nsManager = WebHandler.GetXmlNamespaceManager(xmlDocument);
var stateNodes = xmlDocument.SelectNodes("//*[@id='stCntr']/*[@class='stLst']/a/@href");
if (stateNodes == null)
{
return(list);
}
list.AddRange(from XmlNode node in stateNodes select node.Value.Replace("..", findUrl));
return(list);
}
private void GetTrucks(string type, int startPage, int endPage)
{
var list = new List <CommercialTruckViewModel>();
for (var i = startPage; i <= endPage; i++)
{
var nextPageUrl = String.Format("http://www.commercialtrucktrader.com/search-results?condition={0}&page={1}&schemecode=AD&&", type, i);
Console.WriteLine("PAGE: {0}", nextPageUrl);
var content = WebHandler.DownloadContent(nextPageUrl, 2);
if (String.IsNullOrEmpty(content))
{
break;
}
var xmlDocument = WebHandler.DownloadDocument(content);
if (xmlDocument == null)
{
break;
}
var truckNodes = xmlDocument.SelectNodes("//*[@class='result noborder highlight-green' or @class='result noborder']");
if (truckNodes == null)
{
break;
}
foreach (XmlNode node in truckNodes)
{
try
{
var newTruck = GetDetailTruck(node);
list.Add(newTruck);
Console.WriteLine("{0} -- {1} {2} {3}", list.Count, newTruck.Year, newTruck.Make, newTruck.Model);
}
catch (Exception ex)
{
Console.WriteLine("ERROR: {0}", ex.StackTrace);
}
}
//var nextPageNode = xmlDocument.SelectSingleNode("//*[@class='pageNumber']/a[text()='Next ›']/@href");
//nextPageUrl = nextPageNode != null ? nextPageNode.Value : string.Empty;
InsertTruckIntoDatabase(list);
list.Clear();
}
}
private List <CarMaxState> LoadStates()
{
var list = new List <CarMaxState>();
var content = WebHandler.DownloadContent("https://www.carmax.com/stores/states");
var xmlDocument = WebHandler.DownloadDocument(content);
var stateNodes = xmlDocument.SelectNodes("//*[@class='stores-states--list']/*[.]");
if (stateNodes != null)
{
foreach (XmlNode node in stateNodes)
{
var stateName = node.FirstChild.InnerText.Trim();
var state = new CarMaxState();
state.Value = (Path.GetFileName(node.FirstChild.Attributes["href"].Value));
state.Name = stateName;
list.Add(state);
}
}
return(list);
}
public CarMaxVehicle LoadDetailCarNew(CarMaxVehicle newVehicle)
{
//imageUrl = https://img2.carmax.com/api/images/12702986
var content = WebHandler.DownloadContent("https://beta.carmax.com/cars/" + newVehicle.CarMaxVehicleId);
var xmlDocument = WebHandler.DownloadDocument(content);
var vinNode = xmlDocument.SelectSingleNode("//*[@class='card-grid-section--action-bar-item'][2]/span[2]");
if (vinNode == null)
{
return(null);
}
newVehicle.Vin = vinNode.InnerText.Trim();
var infoNodes = xmlDocument.SelectNodes("//div[@class='card--text-block-list--row']");
if (infoNodes != null)
{
foreach (XmlNode item in infoNodes)
{
if (item.ChildNodes[1] == null || item.ChildNodes[3] == null)
{
continue;
}
var text = item.ChildNodes[1].InnerText.Trim();
var value = item.ChildNodes[3].InnerText.Trim();
switch (text)
{
case "Drive":
newVehicle.DriveTrain = value;
break;
case "Transmission":
newVehicle.Transmission = value;
break;
case "Exterior":
newVehicle.ExteriorColor = value;
break;
case "Interior":
newVehicle.InteriorColor = value;
break;
}
}
}
var featureNodes = xmlDocument.SelectNodes("//ul/li[@class='card-grid-section--item-list-element']");
if (featureNodes != null)
{
var list = new List <string>();
foreach (XmlNode item in featureNodes)
{
list.Add(item.InnerText);
}
newVehicle.Features = string.Join(",", list);
}
var storeNode = xmlDocument.SelectSingleNode("//*[@id='qualifiers-global']");
if (storeNode != null)
{
var jsonObj = (JObject)JsonConvert.DeserializeObject(storeNode.Attributes["data-qualifiers"].Value);
var carmaxStoreId = Convert.ToInt32(((JValue)jsonObj["StoreId"]).Value);
newVehicle.StoreId = (new VINMarketSqlHelper()).GetId("SELECT StoreId FROM CarMaxStore WHERE CarMaxStoreId = " + carmaxStoreId);
}
var stockNode = xmlDocument.SelectSingleNode("//*[@class='card-grid-section--action-bar-item'][1]/span[2]");
if (stockNode != null)
{
newVehicle.Stock = stockNode.InnerText.Trim();
}
var mpgCityNode = xmlDocument.SelectSingleNode("//*[@class='mpg']/*[@class='mpg--value']");
if (mpgCityNode != null)
{
newVehicle.MPGCity = Convert.ToInt32(mpgCityNode.InnerText.Trim());
}
var mpgHighway = xmlDocument.SelectSingleNode("//*[@class='mpg'][2]/*[@class='mpg--value']");
if (mpgHighway != null)
{
newVehicle.MPGHighway = Convert.ToInt32(mpgHighway.InnerText.Trim());
}
var priceNode = xmlDocument.SelectSingleNode("//*[@class='price-mileage--price-container']/*[@class='price-mileage--value']");
if (priceNode != null)
{
newVehicle.Price = CommonHelper.RemoveSpecialCharactersAndReturnNumber(priceNode.InnerText.Trim());
}
var milesNode = xmlDocument.SelectSingleNode("//*[@class='price-mileage--mileage-container']/*[@class='price-mileage--value']");
if (milesNode != null)
{
newVehicle.Miles = Convert.ToInt32(milesNode.InnerText.Trim().ToLower().Replace("k", "000").Replace("<", "").Replace(">", ""));
}
newVehicle.Used = newVehicle.Miles > 1000;
newVehicle.UpdatedDate = DataCommonHelper.GetChicagoDateTime(DateTime.Now);
newVehicle.CreatedDate = DataCommonHelper.GetChicagoDateTime(DateTime.Now);
return(newVehicle);
}
public CarMaxVehicle LoadDetailCar(int year, string make, string model, string trim, long carId, string url)
{
var newVehicle = new CarMaxVehicle()
{
CarMaxVehicleId = carId, Year = year, Make = make, Model = model, Trim = trim, Url = url
};
try
{
var content = WebHandler.DownloadContent(url);
var xmlDocument = WebHandler.DownloadDocument(content);
var vinNode = xmlDocument.SelectSingleNode("//*[@class='vin']");
if (vinNode == null)
{
return(LoadDetailCarNew(newVehicle));
}
newVehicle.Vin = vinNode.ChildNodes[1].Value.Trim();
newVehicle.Price = CommonHelper.RemoveSpecialCharactersAndReturnNumber(xmlDocument.SelectSingleNode("//*[@class='sansser']/*").ChildNodes[0].Value.Trim());
//var certifiedNode = xmlDocument.SelectSingleNode("//*[@class='info-container cqc_tombstone_pos']/*[last()]/*/*/*/*/*[last()-1]/*[last()]");
//newVehicle.Certified = certifiedNode != null && certifiedNode.InnerText.Contains("CarMax Quality Certified");
var infoNodes = xmlDocument.SelectNodes("//*[@class='info-container cqc_tombstone_pos']/*[last()]/*/*/*/*[last()]/*");
if (infoNodes != null)
{
foreach (XmlNode item in infoNodes)
{
if (!(item.Name.Equals("li")) || item.ChildNodes[1] == null)
{
continue;
}
var value = item.ChildNodes[1].Value;
switch (item.FirstChild.InnerText)
{
case "Miles":
newVehicle.Miles = value.Equals("New") ? 0 : Convert.ToInt32(value.ToLower().Replace("k", "000").Replace("<", "").Replace(">", ""));
newVehicle.Used = !value.Equals("New");
break;
case "Drive":
newVehicle.DriveTrain = value;
break;
case "Transmission":
newVehicle.Transmission = value;
break;
case "Exterior":
newVehicle.ExteriorColor = value;
break;
case "Interior":
newVehicle.InteriorColor = value;
break;
case "EPA Mileage":
var tmp = value.Replace("MPG", "").Split('/');
newVehicle.MPGCity = value.Equals("Not Available") ? 0 : Convert.ToInt32(tmp[0]);
newVehicle.MPGHighway = value.Equals("Not Available") ? 0 : Convert.ToInt32(tmp[1]);
break;
case "Stock #":
newVehicle.Stock = value;
break;
case "Rating":
newVehicle.Rating = Convert.ToDecimal(item.ChildNodes[1].InnerText.Replace("(", "").Replace(")", ""));
break;
}
}
}
var storeNode = xmlDocument.SelectSingleNode("//*[@id='qualifiers-global']");
if (storeNode != null)
{
var jsonObj = (JObject)JsonConvert.DeserializeObject(storeNode.Attributes["data-qualifiers"].Value);
var carmaxStoreId = Convert.ToInt32(((JValue)jsonObj["StoreId"]).Value);
newVehicle.StoreId = (new VINMarketSqlHelper()).GetId("SELECT StoreId FROM CarMaxStore WHERE CarMaxStoreId = " + carmaxStoreId);
}
var features = xmlDocument.SelectNodes("//*[@class='features']/*[2]/*");
if (features != null)
{
var temp = (from XmlNode item in features select item.InnerText.Trim()).ToList();
newVehicle.Features = temp.Any() ? temp.Aggregate((a, b) => a + ',' + b) : string.Empty;
}
//var thumbnailImages = xmlDocument.SelectNodes("//*[@id='thumbs']/*");
//var fullImages = xmlDocument.SelectNodes("//*[@id='photos']/*");
//if (fullImages != null)
//{
// var temp = (from XmlNode item in fullImages select item.Attributes["src"].Value).ToList();
// newVehicle.FullPhotos = temp.Any() ? temp.Aggregate((a, b) => a + ',' + b) : string.Empty;
// newVehicle.ThumbnailPhotos = newVehicle.FullPhotos;
//}
newVehicle.UpdatedDate = DataCommonHelper.GetChicagoDateTime(DateTime.Now);
newVehicle.CreatedDate = DataCommonHelper.GetChicagoDateTime(DateTime.Now);
}
catch (Exception ex)
{
_logging.Error("CARMAX - LoadDetailVehicle", ex);
}
return(newVehicle);
}
private void LoadCars(long makeId, string make, long modelId, string model, int distance, int zipcode)
{
var vinmarketSqlHelper = new VINMarketSqlHelper();
var originalUrl = string.Format("http://www.carmax.com/search?ANa={0}&D={1}&zip={2}&N={3}&Ep=search:results:results%20page", modelId, distance, zipcode, makeId);
var searchUrl = originalUrl;
while (!String.IsNullOrEmpty(searchUrl))
{
var content = WebHandler.DownloadContent(searchUrl);
var xmlDocument = WebHandler.DownloadDocument(content);
var aNodes = xmlDocument.SelectNodes("//div[@id='resultsList']/div[@class='car']");
if (aNodes != null && aNodes.Count > 0)
{
var insertQueries = new List <string>();
var updateQueries = new List <string>();
try
{
foreach (XmlNode node in aNodes)
{
var carUrl = "http://www.carmax.com" + node.Attributes["data-details-url"].Value;
var carId = Convert.ToInt64(node.Attributes["sn"].Value);
var vehicleNameNode = node.SelectSingleNode(".//a[@class='vehicleName']//h3");
var year = Convert.ToInt32(vehicleNameNode.InnerText.Substring(0, 4));
var trim = vehicleNameNode.InnerText.Replace(year.ToString(), "").Replace(make, "").Replace(model, "").Trim();
var newVehicle = LoadDetailCar(year, make, model, trim, carId, carUrl);
if (newVehicle == null)
{
continue;
}
var imgNode = node.SelectSingleNode(".//div[@class='photo']//img");
newVehicle.FullPhotos = imgNode != null ? imgNode.Attributes["src"].Value : string.Empty;
newVehicle.ThumbnailPhotos = newVehicle.FullPhotos;
try
{
var id = vinmarketSqlHelper.GetId(string.Format("SELECT VehicleId FROM CarMaxVehicle WHERE VehicleID = @carId"), new { carId });
if (id == 0)
{
var insert = CarMaxQuery.InsertQuery(newVehicle);
insertQueries.Add(insert);
}
else
{
newVehicle.VehicleId = id;
var update = CarMaxQuery.UpdateQuery(newVehicle);
updateQueries.Add(update);
}
//_commonManagementForm.AddNewCarMaxVehicle(newVehicle);
Console.WriteLine("{0} {1} - {2}", make, model, carUrl);
}
catch (Exception ex)
{
_logging.Error("CARMAX - InsertVehicleToDatabase", ex);
Console.WriteLine("CARMAX - InsertVehicleToDatabase {0}", ex.Message);
}
}
}
catch (Exception ex)
{
throw ex;
}
if (insertQueries.Any())
{
vinmarketSqlHelper.QueryText(string.Join(" ", insertQueries));
}
if (updateQueries.Any())
{
vinmarketSqlHelper.QueryText(string.Join(" ", updateQueries));
}
}
var nextNode = xmlDocument.SelectSingleNode("//li[@id='next']");
searchUrl = nextNode != null && nextNode.Attributes["data-url"] != null ? originalUrl + "&" + nextNode.Attributes["data-url"].Value : string.Empty;
}
}
private void LoadCars(string apiKey)
{
var page = 1;
var url = string.Format("https://api.carmax.com/v1/api/vehicles/?SortKey=0&Distance=all&PerPage=50&Zip=92627&Page={1}&platform=carmax.com&apikey={0}", apiKey, page);
var content = WebHandler.DownloadContent(url);
var xmlDocument = WebHandler.DownloadDocument(content);
var totalNode = xmlDocument.SelectSingleNode("//*[local-name()='resultcount']");
if (totalNode == null)
{
return;
}
var totalOfRecords = Convert.ToInt32(totalNode.InnerText);
var numberOfPages = totalOfRecords % 50 == 0 ? totalOfRecords / 50 : (totalOfRecords / 50) + 1;
do
{
if (page > 1)
{
//url = string.Format("https://api.carmax.com/v1/api/vehicles/?SortKey=0&Distance=all&PerPage=50&Zip=92627&Page={1}&platform=carmax.com&apikey={0}", apiKey, page);
xmlDocument = WebHandler.DownloadDocument(WebHandler.DownloadContent(url));
}
var resultNodes = xmlDocument.SelectNodes("//*[local-name()='results']/*[local-name()='resultsrecordmodel']");
if (resultNodes != null)
{
var vinmarketSqlHelper = new VINMarketSqlHelper();
var insertQueries = new List <string>();
var updateQueries = new List <string>();
foreach (XmlNode item in resultNodes)
{
try
{
var newVehicle = LoadDetailCar(item);
var id = vinmarketSqlHelper.GetId(string.Format("SELECT CarMaxVehicleId FROM CarMaxVehicle WHERE CarMaxVehicleId = @carId"), new { carId = newVehicle.CarMaxVehicleId });
if (id == 0)
{
var insert = CarMaxQuery.InsertQuery(newVehicle);
insertQueries.Add(insert);
}
else
{
var update = CarMaxQuery.UpdateQuery(newVehicle);
updateQueries.Add(update);
}
Console.WriteLine("{0} {1} - {2}", newVehicle.Make, newVehicle.Model, newVehicle.Url);
}
catch (Exception ex)
{
Console.WriteLine("CARMAX - InsertVehicleToDatabase {0}", ex.Message);
}
}
if (insertQueries.Any())
{
vinmarketSqlHelper.QueryText(string.Join(" ", insertQueries));
}
if (updateQueries.Any())
{
vinmarketSqlHelper.QueryText(string.Join(" ", updateQueries));
}
}
var urlNode = xmlDocument.SelectSingleNode("//*[local-name()='links']/*[local-name()='resourcelink']/*[local-name()='href']");
if (urlNode != null)
{
url = urlNode.InnerText;
}
else
{
url = null;
}
page++;
Console.WriteLine("Processing next page {0}", page);
} while (page <= numberOfPages && url != null);
}
private CommercialTruckViewModel GetDetailTruck(XmlNode node)
{
if (node == null || node.Attributes == null)
{
return(null);
}
var newTruck = new CommercialTruckViewModel()
{
Vin = String.Empty
};
//var idRegex = new Regex(@"listing.id\:(.*)\.dealer_type");
//Match idMatch = idRegex.Match(node.Attributes["id"].Value);
//newTruck.CommercialTruckId = idMatch.Groups.Count > 0 ? Convert.ToInt32(idMatch.Groups[1].Value) : 0;
var urlDetailNode = node.SelectSingleNode("./div[@class='listing-info lfloat']/div/a");
if (urlDetailNode == null)
{
return(null);
}
newTruck.CommercialTruckId = Convert.ToInt32(urlDetailNode.Attributes["adid"].Value);
if (newTruck.CommercialTruckId.Equals(0))
{
return(null);
}
//var singleNode = node.SelectSingleNode("./div/h3/a/@href");
//if (singleNode != null)
{
var detailUrl = String.Format("{0}{1}", Host, urlDetailNode.Attributes["href"].Value);
var subcontent = WebHandler.DownloadContent(detailUrl, 2);
if (String.IsNullOrEmpty(subcontent))
{
return(null);
}
var subxmlDocument = WebHandler.DownloadDocument(subcontent);
var selectSingleNode = subxmlDocument.SelectSingleNode("//div[@class='details-right']/strong");
if (selectSingleNode != null)
{
var dealer = _commonManagementForm.GetByCommercialTruckDealerName(selectSingleNode.InnerText);
newTruck.DealerId = dealer != null ? dealer.DealerId : 0;
}
newTruck.Url = detailUrl;
//var selectSingleNode1 = subxmlDocument.SelectSingleNode("//span[@itemprop='releaseDate']");
//if (selectSingleNode1 != null) newTruck.Year = Convert.ToInt32(selectSingleNode1.InnerText);
//var singleNode1 = subxmlDocument.SelectSingleNode("//span[@itemprop='manufacturer']");
//if (singleNode1 != null) newTruck.Make = singleNode1.InnerText;
//var xmlNode = subxmlDocument.SelectSingleNode("//span[@itemprop='model']");
//if (xmlNode != null) newTruck.Model = xmlNode.InnerText;
//var categoryNode = subxmlDocument.SelectSingleNode("//span[@itemprop='category']");
//newTruck.Category = categoryNode != null ? categoryNode.InnerText : string.Empty;
var itemNodes = subxmlDocument.SelectNodes("//div[@class='details-right']/ul/li");
if (itemNodes != null)
{
foreach (XmlNode item in itemNodes)
{
var name = item.SelectSingleNode("./span").InnerText;
var value = item.LastChild.Value;
switch (name)
{
case "Year:": newTruck.Year = Convert.ToInt32(value); break;
case "Make:": newTruck.Make = value; break;
case "Model:": newTruck.Model = value; break;
case "Trim:": newTruck.Trim = value; break;
case "Class:": newTruck.Class = value; break;
case "Mileage:": newTruck.Mileage = Convert.ToInt32(value); break;
case "New/Used:": newTruck.IsNew = !value.Equals("U"); break;
case "Color:": newTruck.ExteriorColor = value; break;
case "Fuel Tank Size:": newTruck.Litter = value; break;
case "Engine Model:": newTruck.Engine = value; break;
case "Fuel Type:": newTruck.Fuel = value; break;
case "Body Style:": newTruck.Fuel = value; break;
case "Transmission Speed:": newTruck.Transmission = value; break;
case "Drivetrain:": newTruck.Drivetrain = value; break;
case "VIN:": newTruck.Vin = value; break;
case "Price:":
newTruck.Price = value.Trim().Equals("Call for Price") ? 0 : Convert.ToInt32(value.Trim().Replace("$", "").Replace(",", ""));
break;
}
}
}
var hiddenItemNodes = subxmlDocument.SelectNodes("//div[@class='details-right']/ul[@class='details-specs-more hide']/li");
if (hiddenItemNodes != null)
{
foreach (XmlNode item in hiddenItemNodes)
{
var name = item.SelectSingleNode("./span").InnerText;
var value = item.LastChild.Value;
switch (name)
{
case "Year:": newTruck.Year = Convert.ToInt32(value); break;
case "Make:": newTruck.Make = value; break;
case "Model:": newTruck.Model = value; break;
case "Trim:": newTruck.Trim = value; break;
case "Class:": newTruck.Class = value; break;
case "Mileage:": newTruck.Mileage = Convert.ToInt32(value); break;
case "New/Used:": newTruck.IsNew = !value.Equals("U"); break;
case "Color:": newTruck.ExteriorColor = value; break;
case "Fuel Tank Size:": newTruck.Litter = value; break;
case "Engine Model:": newTruck.Engine = value; break;
case "Fuel Type:": newTruck.Fuel = value; break;
case "Body Style:": newTruck.Fuel = value; break;
case "Transmission Speed:": newTruck.Transmission = value; break;
case "Drivetrain:": newTruck.Drivetrain = value; break;
case "VIN:": newTruck.Vin = value; break;
case "Price:":
newTruck.Price = value.Trim().Equals("Call for Price") ? 0 : Convert.ToInt32(value.Trim().Replace("$", "").Replace(",", ""));
break;
}
}
}
var imageNode = subxmlDocument.SelectSingleNode("//div[@class='details-left']/span/img");
newTruck.Images = imageNode != null ? imageNode.Attributes["src"].Value : string.Empty;
var descriptionNodes = subxmlDocument.SelectNodes("//div[@class='details-left']/h3[@class='bold font1-1']");
foreach (XmlNode item in descriptionNodes)
{
var name = item.InnerText;
var value = item.NextSibling != null && item.NextSibling.NextSibling != null ? item.NextSibling.InnerText : string.Empty;
switch (name)
{
case "Features": newTruck.Package = value; break;
case "Description": newTruck.Description = value; break;
}
}
}
newTruck.DateStamp = DateTime.Now;
newTruck.Updated = DateTime.Now;
return(newTruck);
}
