public void UpdateLocalStorageForOfflineUse(ConcurrentDictionary <string, string> localFileChanges, HtmlDocument vehicleWikiPage, string vehicleName, LocalWikiFileTypeEnum fileType, IVehicle vehicle = null)
{
try
{
if (fileType == LocalWikiFileTypeEnum.Undefined)
{
throw new ArgumentException("The 'fileType' parameter for the 'UpdateLocalStorageForOfflineUse' is required but was not provided.");
}
// Build vars that will be used for the local file
string fileName = _stringHelper.RemoveInvalidCharacters(vehicleName.Replace(' ', '_').Replace('/', '-'));
string folderPath = fileType == LocalWikiFileTypeEnum.Html ? ConfigurationManager.AppSettings["LocalWikiHtmlPath"] : ConfigurationManager.AppSettings["LocalWikiJsonPath"];
string filePath = $@"{folderPath}{fileName}.{fileType.ToString().ToLower()}";
if (!Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
// Handle HTML files
if (fileType == LocalWikiFileTypeEnum.Html)
{
_htmlLogger.CreateHtmlFile(localFileChanges, vehicleWikiPage, vehicleName, fileName, filePath);
}
// Handle JSON files
else if (fileType == LocalWikiFileTypeEnum.Json)
{
_jsonLogger.CreateJsonFile(localFileChanges, vehicleName, vehicle, fileName, filePath);
}
}
catch (Exception ex)
{
_consoleManager.WriteException(ex.Message);
}
}
public void ProcessGroundForcesWikiHtmlFiles(ConcurrentDictionary <string, HtmlDocument> vehicleWikiPagesContent, ConcurrentDictionary <string, string> localFileChanges, Dictionary <string, GroundVehicle> vehicleDetails, List <HtmlNode> vehicleWikiEntryLinks, List <string> errorsList, int indexPosition, int expectedNumberOfLinks, bool createJsonFiles, bool createHtmlFiles, bool createExcelFile)
{
try
{
_consoleManager.WriteLineInColour(ConsoleColor.Yellow, "Press ENTER to begin extracting data from the vehicle pages.");
_consoleManager.WaitUntilKeyIsPressed(ConsoleKey.Enter);
foreach (string vehicleWikiPageLinkTitle in vehicleWikiPagesContent.Keys)
{
// Page to traverse
HtmlDocument vehicleWikiPage = vehicleWikiPagesContent.Single(x => x.Key == vehicleWikiPageLinkTitle).Value;
// Get the header that holds the page title | document.getElementsByClassName('firstHeading')[0].firstChild.innerText
HtmlNode pageTitle = vehicleWikiPage.DocumentNode.Descendants().Single(d => d.Id == "firstHeading").FirstChild;
// Get the div that holds all of the content under the title section | document.getElementById('bodyContent')
HtmlNode wikiBody = vehicleWikiPage.DocumentNode.Descendants().Single(d => d.Id == "bodyContent");
// Get the div that holds the content on the RHS of the page where the information table is | document.getElementById('bodyContent').getElementsByClassName('right-area')
HtmlNode rightHandContent = wikiBody.Descendants("div").SingleOrDefault(d => d.Attributes["class"] != null && d.Attributes["class"].Value.Contains("right-area"));
// Get the able that holds all of the vehicle information | document.getElementsByClassName('flight-parameters')[0]
HtmlNode infoBox = rightHandContent?.Descendants("table").SingleOrDefault(d => d.Attributes["class"].Value.Contains("flight-parameters"));
// Name
string vehicleName = _stringHelper.RemoveInvalidCharacters(WebUtility.HtmlDecode(vehicleWikiPageLinkTitle));
// Link
HtmlNode urlNode = vehicleWikiEntryLinks.SingleOrDefault(v => v.InnerText.Equals(vehicleName));
string relativeUrl = urlNode?.Attributes["href"].Value ?? "";
string vehicleWikiEntryFullUrl = new Uri(new Uri(ConfigurationManager.AppSettings["BaseWikiUrl"]), relativeUrl).ToString();
// Fail fast and create error if there is no info box
if (infoBox == null)
{
_consoleManager.WriteLineInColour(ConsoleColor.Red, $"Error processing item {indexPosition} of {expectedNumberOfLinks}", false);
_consoleManager.WriteBlankLine();
errorsList.Add($"No Information found for '{vehicleName}' - {vehicleWikiEntryFullUrl}");
_consoleManager.ResetConsoleTextColour();
indexPosition++;
continue;
}
else
{
// Setup local vars
Dictionary <string, string> vehicleAttributes = new Dictionary <string, string>();
HtmlNodeCollection rows = infoBox.SelectNodes("tr");
_consoleManager.WriteTextLine($"The following values were found for {vehicleName}");
_webCrawler.GetAttributesFromInfoBox(vehicleAttributes, rows);
_consoleManager.ResetConsoleTextColour();
// Country
string countryRawValue = vehicleAttributes.Single(k => k.Key == "Country").Value;
CountryEnum vehicleCountry = _vehicleCountryHelper.GetVehicleCountryFromName(countryRawValue).CountryEnum;
// Weight
string weightRawValue = vehicleAttributes.Single(k => k.Key == "Weight").Value;
int weightWithoutUnits = Int32.Parse(Regex.Match(weightRawValue, @"\d+").Value);
string weightUnitsAbbreviation = (Regex.Matches(weightRawValue, @"\D+").Cast <Match>()).Last().Value.Trim();
VehicleWeightUnitHelper vehicleWeightUnit = _vehicleWeightUnitHelper.GetWeightUnitFromAbbreviation(weightUnitsAbbreviation);
// Vehicle class
string typeRawValue = vehicleAttributes.Single(k => k.Key == "Type").Value;
GroundVehicleTypeHelper vehicleType = _vehicleTypeHelper.GetGroundVehicleTypeFromName(typeRawValue);
// Rank
int rankRawValue = Int32.Parse(vehicleAttributes.Single(k => k.Key == "Rank").Value);
int vehicleRank = rankRawValue;
// Battle rating
double ratingRawValue = Double.Parse(vehicleAttributes.Single(k => k.Key == "Rating").Value);
double vehicleBattleRating = ratingRawValue;
// Engine power
string enginePowerRawValue = vehicleAttributes.Single(k => k.Key == "Engine power").Value;
int enginePowerWithoutUnits = Int32.Parse(Regex.Match(enginePowerRawValue, @"\d+").Value);
string enginePowerUnitsAbbreviation = (Regex.Matches(enginePowerRawValue, @"\D+").Cast <Match>()).Last().Value.Trim();
VehicleEnginePowerUnitHelper vehicleEngineUnit = _vehicleEnginePowerUnitHelper.GetEngineUnitFromAbbreviation(enginePowerUnitsAbbreviation);
// Max speed
string maxSpeedRawValue = vehicleAttributes.Single(k => k.Key == "Max speed").Value;
double maxSpeedWithoutUnits = Double.Parse(Regex.Match(maxSpeedRawValue, @"\d+\.*\d*").Value);
string maxSpeedUnits = (Regex.Matches(maxSpeedRawValue, @"\D+").Cast <Match>()).Last().Value.Trim();
VehicleSpeedUnitHelper vehicleSpeedUnit = _vehicleSpeedUnitHelper.GetSpeedUnitFromAbbreviation(maxSpeedUnits);
// Hull armour
string hullArmourRawValue = vehicleAttributes.Single(k => k.Key == "Hull armour thickness").Value;
string vehicleHullArmourThickness = hullArmourRawValue;
// Superstructure armour
string superstructureArmourRawValue = vehicleAttributes.Single(k => k.Key == "Superstructure armour thickness").Value;
string vehicleSuperstructureArmourThickness = superstructureArmourRawValue;
// Repair time
string freeRepairTimeRawValue = vehicleAttributes.Single(k => k.Key == "Time for free repair").Value;
List <Match> freeRepairTimeList = (Regex.Matches(freeRepairTimeRawValue, @"\d+").Cast <Match>()).ToList();
int freeRepairTimeHours = Int32.Parse(freeRepairTimeList.First().Value);
int freeRepairTimeMinutes = Int32.Parse(freeRepairTimeList.Last().Value);
TimeSpan vehicleFreeRepairTime = new TimeSpan(freeRepairTimeHours, freeRepairTimeMinutes, 0);
// Max repair cost
string maxRepairCostRawValue = vehicleAttributes.Single(k => k.Key == "Max repair cost*").Value;
string maxRepairCostWithoutUnits = Regex.Match(maxRepairCostRawValue, @"\d+").Value;
string maxRepairCostUnits = (Regex.Matches(maxRepairCostRawValue, @"\D+").Cast <Match>()).Last().Value.Trim();
long vehicleMaxRepairCost = Int64.Parse(maxRepairCostWithoutUnits);
VehicleCostUnitHelper vehicleRepairCostUnit = _vehicleCostUnitHelper.GetCostUnitFromAbbreviation(maxRepairCostUnits);
// Purchase cost
string purchaseCostRawValue = vehicleAttributes.Single(k => k.Key == "Cost*").Value;
string purchaseCostWithoutUnits = Regex.Match(purchaseCostRawValue, @"\d+").Value;
string purchaseCostUnits = (Regex.Matches(purchaseCostRawValue, @"\D+").Cast <Match>()).Last().Value.Trim();
long vehiclePurchaseCost = Int64.Parse(purchaseCostWithoutUnits);
VehicleCostUnitHelper vehiclePurchaseCostUnit = _vehicleCostUnitHelper.GetCostUnitFromAbbreviation(purchaseCostUnits);
// Last modified
HtmlNode lastModifiedSection = vehicleWikiPage.DocumentNode.Descendants().SingleOrDefault(x => x.Id == ConfigurationManager.AppSettings["LastModifiedSectionId"]);
string lastModified = lastModifiedSection?.InnerHtml;
// Populate objects
GroundVehicle groundVehicle = new GroundVehicle
{
Name = vehicleName,
Country = vehicleCountry,
Weight = weightWithoutUnits,
VehicleType = (VehicleTypeEnum)vehicleType.Id,
Rank = vehicleRank,
BattleRating = vehicleBattleRating,
EnginePower = enginePowerWithoutUnits,
MaxSpeed = maxSpeedWithoutUnits,
HullArmourThickness = vehicleHullArmourThickness,
SuperstructureArmourThickness = vehicleSuperstructureArmourThickness,
TimeForFreeRepair = vehicleFreeRepairTime,
MaxRepairCost = vehicleMaxRepairCost,
PurchaseCost = vehiclePurchaseCost,
PurchaseCostUnit = vehiclePurchaseCostUnit,
MaxRepairCostUnit = vehicleRepairCostUnit,
MaxSpeedUnit = vehicleSpeedUnit,
WeightUnit = vehicleWeightUnit,
EnginePowerUnit = vehicleEngineUnit,
LastModified = lastModified
};
// Update the local storage if requested
if (createJsonFiles)
{
_logger.UpdateLocalStorageForOfflineUse(localFileChanges, vehicleWikiPage, vehicleName, LocalWikiFileTypeEnum.Json, groundVehicle);
}
if (createHtmlFiles)
{
_logger.UpdateLocalStorageForOfflineUse(localFileChanges, vehicleWikiPage, vehicleName, LocalWikiFileTypeEnum.Html);
}
//WikiEntry entry = new WikiEntry(vehicleName, vehicleWikiEntryFullUrl, VehicleTypeEnum.Ground, vehicleInfo);
// Add the found information to the master list
vehicleDetails.Add(vehicleName, groundVehicle);
_consoleManager.WriteLineInColour(ConsoleColor.Green, $"Processed item {indexPosition} of {expectedNumberOfLinks} successfully");
_consoleManager.WriteBlankLine();
}
indexPosition++;
}
if (createExcelFile)
{
_excelLogger.CreateExcelFile(vehicleDetails);
}
}
catch (Exception ex)
{
_consoleManager.WriteException(ex.Message);
}
}
