public void UpdateLocalStorageForOfflineUse(ConcurrentDictionary <string, string> localFileChanges, HtmlDocument vehicleWikiPage, string vehicleName, LocalWikiFileTypeEnum fileType, IVehicle vehicle = null) { try { if (fileType == LocalWikiFileTypeEnum.Undefined) { throw new ArgumentException("The 'fileType' parameter for the 'UpdateLocalStorageForOfflineUse' is required but was not provided."); } // Build vars that will be used for the local file string fileName = _stringHelper.RemoveInvalidCharacters(vehicleName.Replace(' ', '_').Replace('/', '-')); string folderPath = fileType == LocalWikiFileTypeEnum.Html ? ConfigurationManager.AppSettings["LocalWikiHtmlPath"] : ConfigurationManager.AppSettings["LocalWikiJsonPath"]; string filePath = $@"{folderPath}{fileName}.{fileType.ToString().ToLower()}"; if (!Directory.Exists(folderPath)) { Directory.CreateDirectory(folderPath); } // Handle HTML files if (fileType == LocalWikiFileTypeEnum.Html) { _htmlLogger.CreateHtmlFile(localFileChanges, vehicleWikiPage, vehicleName, fileName, filePath); } // Handle JSON files else if (fileType == LocalWikiFileTypeEnum.Json) { _jsonLogger.CreateJsonFile(localFileChanges, vehicleName, vehicle, fileName, filePath); } } catch (Exception ex) { _consoleManager.WriteException(ex.Message); } }
public void ProcessGroundForcesWikiHtmlFiles(ConcurrentDictionary <string, HtmlDocument> vehicleWikiPagesContent, ConcurrentDictionary <string, string> localFileChanges, Dictionary <string, GroundVehicle> vehicleDetails, List <HtmlNode> vehicleWikiEntryLinks, List <string> errorsList, int indexPosition, int expectedNumberOfLinks, bool createJsonFiles, bool createHtmlFiles, bool createExcelFile) { try { _consoleManager.WriteLineInColour(ConsoleColor.Yellow, "Press ENTER to begin extracting data from the vehicle pages."); _consoleManager.WaitUntilKeyIsPressed(ConsoleKey.Enter); foreach (string vehicleWikiPageLinkTitle in vehicleWikiPagesContent.Keys) { // Page to traverse HtmlDocument vehicleWikiPage = vehicleWikiPagesContent.Single(x => x.Key == vehicleWikiPageLinkTitle).Value; // Get the header that holds the page title | document.getElementsByClassName('firstHeading')[0].firstChild.innerText HtmlNode pageTitle = vehicleWikiPage.DocumentNode.Descendants().Single(d => d.Id == "firstHeading").FirstChild; // Get the div that holds all of the content under the title section | document.getElementById('bodyContent') HtmlNode wikiBody = vehicleWikiPage.DocumentNode.Descendants().Single(d => d.Id == "bodyContent"); // Get the div that holds the content on the RHS of the page where the information table is | document.getElementById('bodyContent').getElementsByClassName('right-area') HtmlNode rightHandContent = wikiBody.Descendants("div").SingleOrDefault(d => d.Attributes["class"] != null && d.Attributes["class"].Value.Contains("right-area")); // Get the able that holds all of the vehicle information | document.getElementsByClassName('flight-parameters')[0] HtmlNode infoBox = rightHandContent?.Descendants("table").SingleOrDefault(d => d.Attributes["class"].Value.Contains("flight-parameters")); // Name string vehicleName = _stringHelper.RemoveInvalidCharacters(WebUtility.HtmlDecode(vehicleWikiPageLinkTitle)); // Link HtmlNode urlNode = vehicleWikiEntryLinks.SingleOrDefault(v => v.InnerText.Equals(vehicleName)); string relativeUrl = urlNode?.Attributes["href"].Value ?? ""; string vehicleWikiEntryFullUrl = new Uri(new Uri(ConfigurationManager.AppSettings["BaseWikiUrl"]), relativeUrl).ToString(); // Fail fast and create error if there is no info box if (infoBox == null) { _consoleManager.WriteLineInColour(ConsoleColor.Red, $"Error processing item {indexPosition} of {expectedNumberOfLinks}", false); _consoleManager.WriteBlankLine(); errorsList.Add($"No Information found for '{vehicleName}' - {vehicleWikiEntryFullUrl}"); _consoleManager.ResetConsoleTextColour(); indexPosition++; continue; } else { // Setup local vars Dictionary <string, string> vehicleAttributes = new Dictionary <string, string>(); HtmlNodeCollection rows = infoBox.SelectNodes("tr"); _consoleManager.WriteTextLine($"The following values were found for {vehicleName}"); _webCrawler.GetAttributesFromInfoBox(vehicleAttributes, rows); _consoleManager.ResetConsoleTextColour(); // Country string countryRawValue = vehicleAttributes.Single(k => k.Key == "Country").Value; CountryEnum vehicleCountry = _vehicleCountryHelper.GetVehicleCountryFromName(countryRawValue).CountryEnum; // Weight string weightRawValue = vehicleAttributes.Single(k => k.Key == "Weight").Value; int weightWithoutUnits = Int32.Parse(Regex.Match(weightRawValue, @"\d+").Value); string weightUnitsAbbreviation = (Regex.Matches(weightRawValue, @"\D+").Cast <Match>()).Last().Value.Trim(); VehicleWeightUnitHelper vehicleWeightUnit = _vehicleWeightUnitHelper.GetWeightUnitFromAbbreviation(weightUnitsAbbreviation); // Vehicle class string typeRawValue = vehicleAttributes.Single(k => k.Key == "Type").Value; GroundVehicleTypeHelper vehicleType = _vehicleTypeHelper.GetGroundVehicleTypeFromName(typeRawValue); // Rank int rankRawValue = Int32.Parse(vehicleAttributes.Single(k => k.Key == "Rank").Value); int vehicleRank = rankRawValue; // Battle rating double ratingRawValue = Double.Parse(vehicleAttributes.Single(k => k.Key == "Rating").Value); double vehicleBattleRating = ratingRawValue; // Engine power string enginePowerRawValue = vehicleAttributes.Single(k => k.Key == "Engine power").Value; int enginePowerWithoutUnits = Int32.Parse(Regex.Match(enginePowerRawValue, @"\d+").Value); string enginePowerUnitsAbbreviation = (Regex.Matches(enginePowerRawValue, @"\D+").Cast <Match>()).Last().Value.Trim(); VehicleEnginePowerUnitHelper vehicleEngineUnit = _vehicleEnginePowerUnitHelper.GetEngineUnitFromAbbreviation(enginePowerUnitsAbbreviation); // Max speed string maxSpeedRawValue = vehicleAttributes.Single(k => k.Key == "Max speed").Value; double maxSpeedWithoutUnits = Double.Parse(Regex.Match(maxSpeedRawValue, @"\d+\.*\d*").Value); string maxSpeedUnits = (Regex.Matches(maxSpeedRawValue, @"\D+").Cast <Match>()).Last().Value.Trim(); VehicleSpeedUnitHelper vehicleSpeedUnit = _vehicleSpeedUnitHelper.GetSpeedUnitFromAbbreviation(maxSpeedUnits); // Hull armour string hullArmourRawValue = vehicleAttributes.Single(k => k.Key == "Hull armour thickness").Value; string vehicleHullArmourThickness = hullArmourRawValue; // Superstructure armour string superstructureArmourRawValue = vehicleAttributes.Single(k => k.Key == "Superstructure armour thickness").Value; string vehicleSuperstructureArmourThickness = superstructureArmourRawValue; // Repair time string freeRepairTimeRawValue = vehicleAttributes.Single(k => k.Key == "Time for free repair").Value; List <Match> freeRepairTimeList = (Regex.Matches(freeRepairTimeRawValue, @"\d+").Cast <Match>()).ToList(); int freeRepairTimeHours = Int32.Parse(freeRepairTimeList.First().Value); int freeRepairTimeMinutes = Int32.Parse(freeRepairTimeList.Last().Value); TimeSpan vehicleFreeRepairTime = new TimeSpan(freeRepairTimeHours, freeRepairTimeMinutes, 0); // Max repair cost string maxRepairCostRawValue = vehicleAttributes.Single(k => k.Key == "Max repair cost*").Value; string maxRepairCostWithoutUnits = Regex.Match(maxRepairCostRawValue, @"\d+").Value; string maxRepairCostUnits = (Regex.Matches(maxRepairCostRawValue, @"\D+").Cast <Match>()).Last().Value.Trim(); long vehicleMaxRepairCost = Int64.Parse(maxRepairCostWithoutUnits); VehicleCostUnitHelper vehicleRepairCostUnit = _vehicleCostUnitHelper.GetCostUnitFromAbbreviation(maxRepairCostUnits); // Purchase cost string purchaseCostRawValue = vehicleAttributes.Single(k => k.Key == "Cost*").Value; string purchaseCostWithoutUnits = Regex.Match(purchaseCostRawValue, @"\d+").Value; string purchaseCostUnits = (Regex.Matches(purchaseCostRawValue, @"\D+").Cast <Match>()).Last().Value.Trim(); long vehiclePurchaseCost = Int64.Parse(purchaseCostWithoutUnits); VehicleCostUnitHelper vehiclePurchaseCostUnit = _vehicleCostUnitHelper.GetCostUnitFromAbbreviation(purchaseCostUnits); // Last modified HtmlNode lastModifiedSection = vehicleWikiPage.DocumentNode.Descendants().SingleOrDefault(x => x.Id == ConfigurationManager.AppSettings["LastModifiedSectionId"]); string lastModified = lastModifiedSection?.InnerHtml; // Populate objects GroundVehicle groundVehicle = new GroundVehicle { Name = vehicleName, Country = vehicleCountry, Weight = weightWithoutUnits, VehicleType = (VehicleTypeEnum)vehicleType.Id, Rank = vehicleRank, BattleRating = vehicleBattleRating, EnginePower = enginePowerWithoutUnits, MaxSpeed = maxSpeedWithoutUnits, HullArmourThickness = vehicleHullArmourThickness, SuperstructureArmourThickness = vehicleSuperstructureArmourThickness, TimeForFreeRepair = vehicleFreeRepairTime, MaxRepairCost = vehicleMaxRepairCost, PurchaseCost = vehiclePurchaseCost, PurchaseCostUnit = vehiclePurchaseCostUnit, MaxRepairCostUnit = vehicleRepairCostUnit, MaxSpeedUnit = vehicleSpeedUnit, WeightUnit = vehicleWeightUnit, EnginePowerUnit = vehicleEngineUnit, LastModified = lastModified }; // Update the local storage if requested if (createJsonFiles) { _logger.UpdateLocalStorageForOfflineUse(localFileChanges, vehicleWikiPage, vehicleName, LocalWikiFileTypeEnum.Json, groundVehicle); } if (createHtmlFiles) { _logger.UpdateLocalStorageForOfflineUse(localFileChanges, vehicleWikiPage, vehicleName, LocalWikiFileTypeEnum.Html); } //WikiEntry entry = new WikiEntry(vehicleName, vehicleWikiEntryFullUrl, VehicleTypeEnum.Ground, vehicleInfo); // Add the found information to the master list vehicleDetails.Add(vehicleName, groundVehicle); _consoleManager.WriteLineInColour(ConsoleColor.Green, $"Processed item {indexPosition} of {expectedNumberOfLinks} successfully"); _consoleManager.WriteBlankLine(); } indexPosition++; } if (createExcelFile) { _excelLogger.CreateExcelFile(vehicleDetails); } } catch (Exception ex) { _consoleManager.WriteException(ex.Message); } }