private static void ProcessVehicleListOnPage(HtmlNodeCollection vehicleListItems, string DealerName, ref List <VEHICLE> PreviouslyFoundVehicles, string Market)
{
if (vehicleListItems == null)
{
return;
}
char[] killcomma = { ',' };
foreach (HtmlNode vehicle in vehicleListItems)
{
HtmlNode VehicleNode = vehicle.SelectSingleNode("div[@data-vin]");
string VehicleVIN = VehicleNode.Attributes["data-vin"].Value.ToString();
string VehicleBodyStyle = VehicleNode.Attributes["data-bodyStyle"].Value.ToString();
string VehicleMake = VehicleNode.Attributes["data-make"].Value.ToString();
string VehicleYear = VehicleNode.Attributes["data-year"].Value.ToString();
string VehicleModel = VehicleNode.Attributes["data-model"].Value.ToString();
string VehicleTrim = VehicleNode.Attributes["data-trim"].Value.ToString();
string VehicleExtColor = VehicleNode.Attributes["data-exteriorcolor"].Value.ToString();
VEHICLE ExistingVehicle = VehicleDBManager.GetVehicleByVIN(VehicleVIN);
List <VehiclePriceHistory> PriceHistory = null;
bool Existing = false;
// Check this vehicle against the list of previously found vehicles.
// If it is found, remove it from the list, which ultimately removes it
// from the "sold vehicle" list.
if (ExistingVehicle != null && PreviouslyFoundVehicles != null)
{
Existing = true;
PriceHistory = RetrieveHistory(ExistingVehicle.VEHICLE_HISTORY);
var FoundThisOne = (from v in PreviouslyFoundVehicles
where v.VIN == VehicleVIN
select v).SingleOrDefault <VEHICLE>();
if (!ReferenceEquals(null, FoundThisOne))
{
PreviouslyFoundVehicles.Remove((VEHICLE)FoundThisOne);
}
}
else
{
PriceHistory = new List <VehiclePriceHistory>();
}
HtmlNodeCollection VehicleNodeChildren = vehicle.ChildNodes;
HtmlNode LinkNode = VehicleNodeChildren[1].SelectSingleNode("div/div[@class='media']/a[@href]");
string VehicleLink = BaseURI + LinkNode.Attributes["href"].Value.ToString();
HtmlNode DescriptionNode = VehicleNodeChildren[1].SelectSingleNode("div/div[@class='description']");
HtmlNodeCollection DescriptionLists = VehicleNodeChildren[1].SelectNodes("div/div[@class='description']/dl");
string VehicleEngine = string.Empty;
string VehicleTrans = string.Empty;
string IntColor = string.Empty;
string Mileage = string.Empty;
string StockNumber = string.Empty;
string ModelCode = string.Empty;
string VehicleDriveType = string.Empty;
foreach (HtmlNode DList in DescriptionLists)
{
string ItemType = string.Empty;
foreach (HtmlNode Item in DList.ChildNodes)
{
string Data = string.Empty;
if (Item.Name == "dt")
{
ItemType = Item.InnerText;
continue;
}
else if (Item.Name == "dd")
{
Data = Item.InnerText.Trim(killcomma);
}
else
{
continue;
}
switch (ItemType)
{
case "Engine:":
VehicleEngine = Data;
break;
case "Transmission:":
if (VehicleTrans.Length <= Data.Length)
{
VehicleTrans = Data;
}
break;
case "Exterior Color:":
if (Data != VehicleBodyStyle)
{
VehicleExtColor = Data;
}
break;
case "Interior Color:":
IntColor = Data;
break;
case "Stock #:":
StockNumber = Data;
break;
case "Mileage:":
Mileage = Data.Trim();
Mileage = Mileage.Replace(",", "");
Mileage = Mileage.Replace("miles", "");
if (Mileage.Length == 0 || Mileage == null || Mileage == " ")
{
Mileage = "0";
}
break;
case "Model Code:":
ModelCode = Data;
break;
case "Drive Line:":
VehicleDriveType = Data;
break;
}
ItemType = string.Empty;
}
}
// Stuff we might have to go to vehicle detail pages to retrieve: Mileage, Drive type, price
// Mileage not on search page? (Snethkamp)
if (Mileage == string.Empty || Mileage == "UNKNOWN" || Mileage == "0")
{
Mileage = GetVehicleDetail(VehicleLink, "ddc-span6", "Mileage", "UNKNOWN");
Mileage = Mileage.Trim();
Mileage = Mileage.Replace(",", "");
Mileage = Mileage.Replace("miles", "");
Mileage = Mileage.Replace(@"\n", "");
}
// Vehicle Drive Type not on search page?
if (VehicleDriveType == string.Empty || VehicleDriveType == "UNKNOWN")
{
VehicleDriveType = GetVehicleDetail(VehicleLink, "item powertrain", "Drive type", "UNKNOWN");
}
if (VehicleDriveType == string.Empty || VehicleDriveType == "UNKNOWN")
{
VehicleDriveType = GetVehicleDetail(VehicleLink, "powertrain", "Drive type", "UNKNOWN");
}
// Wheel Size
string WheelSizeInches = GetVehicleDetail(VehicleLink, "item suspension-handling", "Wheel size", "UNKNOWN");
if (WheelSizeInches == "UNKNOWN")
{
WheelSizeInches = GetVehicleDetail(VehicleLink, "suspension-handling", "Wheel size", "UNKNOWN");
}
if (WheelSizeInches == "UNKNOWN")
{
WheelSizeInches = GetVehicleDetail(VehicleLink, "standard-features", "Wheel Diameter", "UNKNOWN");
}
// Vehicle Price
string Price = string.Empty;
HtmlNode PriceNode = VehicleNodeChildren[1].SelectSingleNode("div/div[@class='pricing-area has-buttons']");
HtmlNode ValueNode = PriceNode.SelectSingleNode("ul/li/span/span[@class='value']");
if (ValueNode != null)
{
Price = PriceNode.SelectSingleNode("ul/li/span/span[@class='value']").InnerText;
Price = Price.Replace("$", "");
Price = Price.Replace(",", "");
if (Price.Contains("/mo") || Price.Contains("month"))
{
Price = "0.00";
}
}
VehiclePriceHistory vph = new VehiclePriceHistory();
vph.VIN = VehicleVIN;
vph.Date_Recorded = DateTime.Now.ToLocalTime();
vph.Price = Price;
PriceHistory.Add(vph);
// Carfax
string CarfaxURL = string.Empty;
HtmlNode CarfaxNode = DescriptionNode.SelectSingleNode("div[@class='calloutDetails']/ul/li[@class='carfax']");
if (CarfaxNode != null)
{
CarfaxURL = CarfaxNode.SelectSingleNode("a").Attributes["href"].Value.ToString();
}
// Images
// List<int> ImageIds = GetandStoreVehicleImages(VehicleLink);
//var strImageIdListcsv = string.Join(", ", ImageIds);
// Console.WriteLine(strImageIdListcsv);
if (ExistingVehicle != null)
{
ExistingVehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
ExistingVehicle.BODY_STYLE = VehicleBodyStyle;
ExistingVehicle.CARFAX_URL = CarfaxURL;
ExistingVehicle.COLOR_EXTERIOR = VehicleExtColor;
ExistingVehicle.COLOR_INTERIOR = IntColor;
ExistingVehicle.CURRENT_PRICE = Price;
ExistingVehicle.DATE_LAST_SEEN = DateTime.Now;
ExistingVehicle.DEALERSHIP_NAME = DealerName;
ExistingVehicle.DEALER_DETAIL_URL = VehicleDetailURL;
ExistingVehicle.DRIVE_TRAIN = VehicleDriveType;
ExistingVehicle.ENGINE = VehicleEngine;
ExistingVehicle.MAKE = VehicleMake;
ExistingVehicle.MARKET = Market;
//ExistingVehicle.IMAGEIDCSV = strImageIdListcsv;
ExistingVehicle.MILEAGE = Mileage;
ExistingVehicle.MODEL = VehicleModel;
ExistingVehicle.MODEL_CODE = ModelCode;
ExistingVehicle.STOCK_NUMBER = StockNumber;
ExistingVehicle.TRANSMISSION = VehicleTrans;
ExistingVehicle.TRIM = VehicleTrim;
ExistingVehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
ExistingVehicle.VIN = VehicleVIN;
ExistingVehicle.WHEEL_SIZE = WheelSizeInches;
ExistingVehicle.YEAR = VehicleYear;
VehicleDBManager.UpdateVehicleRecord(ExistingVehicle);
}
else
{
VEHICLE foundvehicle = new VEHICLE();
foundvehicle.BODY_STYLE = VehicleBodyStyle;
foundvehicle.CARFAX_URL = CarfaxURL;
foundvehicle.COLOR_EXTERIOR = VehicleExtColor;
foundvehicle.COLOR_INTERIOR = IntColor;
foundvehicle.CURRENT_PRICE = Price;
foundvehicle.DATE_LAST_SEEN = DateTime.Now;
foundvehicle.DEALERSHIP_NAME = DealerName;
foundvehicle.DEALER_DETAIL_URL = VehicleDetailURL;
foundvehicle.DRIVE_TRAIN = VehicleDriveType;
foundvehicle.ENGINE = VehicleEngine;
foundvehicle.MAKE = VehicleMake;
foundvehicle.MARKET = Market;
//foundvehicle.IMAGEIDCSV = strImageIdListcsv;
foundvehicle.MILEAGE = Mileage;
foundvehicle.MODEL = VehicleModel;
foundvehicle.MODEL_CODE = ModelCode;
foundvehicle.STOCK_NUMBER = StockNumber;
foundvehicle.TRANSMISSION = VehicleTrans;
foundvehicle.TRIM = VehicleTrim;
foundvehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
foundvehicle.VIN = VehicleVIN;
foundvehicle.WHEEL_SIZE = WheelSizeInches;
foundvehicle.YEAR = VehicleYear;
VehicleDBManager.InsertVehicle(foundvehicle);
}
Console.WriteLine($@"
VIN : {VehicleVIN} Found Previously? {Existing}
{VehicleYear} {VehicleMake} {VehicleModel} {VehicleTrim} ({VehicleBodyStyle})
Drivetrain: Engine: {VehicleEngine} | Transmission: {VehicleTrans}
Drive type: {VehicleDriveType} | Wheel size: {WheelSizeInches}
Color: Exterior: {VehicleExtColor} | Interior: {IntColor}
Mileage: {Mileage}
Price: {Price}
Stock #: {StockNumber} | Model Code: {ModelCode}
Direct URL: {VehicleLink}
Carfax URL: {CarfaxURL}");
VehicleVIN = StockNumber = VehicleYear = VehicleMake = VehicleModel = ModelCode =
VehicleTrim = VehicleEngine = VehicleTrans = VehicleDriveType = WheelSizeInches =
VehicleBodyStyle = VehicleExtColor = Mileage = IntColor = Price = CarfaxURL = VehicleLink = string.Empty;
}
}
public static void ProcessAllDealers()
{
WebClient client = new System.Net.WebClient();
HtmlWeb webGet = new HtmlWeb();
List <DEALERSHIP> dealers = VehicleDBManager.GetDealerships("Dealer.Com");
if (dealers == null)
{
return;
}
foreach (DEALERSHIP dealer in dealers)
{
string DealerName = dealer.DEALER_NAME;
string MainUrl = dealer.DEALER_URL;
Console.WriteLine($"************\r\n*** Processing Dealer: {DealerName}\r\n************");
HtmlDocument mainpage = webGet.Load(MainUrl);
bool KeepProcessing = (mainpage != null);
int PageCount = 0;
BaseURI = new Uri(MainUrl).GetLeftPart(UriPartial.Authority);
Console.WriteLine($"Base URL for dealership: {BaseURI}");
List <VEHICLE> PreviouslyFoundAtDealer = VehicleDBManager.GetAllUnsoldVehiclesForDealer(DealerName);
while (KeepProcessing)
{
Console.WriteLine($"\r\n\r\bRetrieving page #{++PageCount} of vehicles for dealer {DealerName} in the {dealer.MARKET_AREA_NAME} Market Area\r\n");
// Note: if the items are "shared", then they are listings from affiliated dealerships.
HtmlNodeCollection VehicleListItems = mainpage.DocumentNode.SelectNodes("//li[substring(@class, 1, 14)='item notshared']");
ProcessVehicleListOnPage(VehicleListItems, DealerName, ref PreviouslyFoundAtDealer, dealer.MARKET_AREA_NAME);
HtmlNode NextLink = mainpage.DocumentNode.SelectSingleNode("//a[@class='ddc-btn ddc-btn-link ddc-btn-xsmall'][@rel='next']");
string Next = string.Empty;
if (NextLink != null)
{
Next = NextLink.Attributes["href"].Value.ToString();
string NextURL = MainUrl + Next;
mainpage = webGet.Load(NextURL);
KeepProcessing = (mainpage != null);
}
else
{
KeepProcessing = false;
}
}
if (!KeepProcessing && PreviouslyFoundAtDealer != null)
{
// All the vehicles currently at this dealer have been screen-scraped.
// Now go through the complete list of "STILL_FOR_SALE" vehicles from the previous run
// and update any that have disappeared from their website as being sold or auctioned.
Console.WriteLine($"Found {PreviouslyFoundAtDealer.Count} vehicle(s) removed from dealer inventory since last run.");
foreach (VEHICLE SoldOrAuctionedVehicle in PreviouslyFoundAtDealer)
{
SoldOrAuctionedVehicle.STILL_FOR_SALE = "NO";
List <VehiclePriceHistory> PriceHistory = RetrieveHistory(SoldOrAuctionedVehicle.VEHICLE_HISTORY);
var MinDate = (from pricehistories in PriceHistory select pricehistories.Date_Recorded).Min();
var MaxDate = (from pricehistories in PriceHistory select pricehistories.Date_Recorded).Max();
var DaysOnMarket = MaxDate - MinDate;
var FinalPriceHistory = (from pricehistories in PriceHistory
where pricehistories.Date_Recorded == MaxDate
select pricehistories).SingleOrDefault <VehiclePriceHistory>();
FinalPriceHistory.WasFinalPrice = "YES";
SoldOrAuctionedVehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
Console.WriteLine($@"Updating VIN {SoldOrAuctionedVehicle.VIN} : {SoldOrAuctionedVehicle.YEAR} {SoldOrAuctionedVehicle.MAKE} {SoldOrAuctionedVehicle.MODEL} (Stock number {SoldOrAuctionedVehicle.STOCK_NUMBER}) as sold. Final price was {FinalPriceHistory.Price}, Mileage was {SoldOrAuctionedVehicle.MILEAGE}, Days on market was {DaysOnMarket.Days}");
VehicleDBManager.UpdateVehicleRecord(SoldOrAuctionedVehicle);
}
}
}
}