protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character) { var move = default(IMove); if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1) { string name = GetStatName(cells[0]); string faf = cells[2].InnerText; string angle = cells[4].InnerText; string bkbwbkb = cells[5].InnerText; string kbg = cells[6].InnerText; string landingLag = cells[7].InnerText; var autocancel = new CommonAutocancelParameterResolver().Resolve(cells[8]); move = ScrapingServices.CreateMove(); move = new CommonMoveParameterResolver(character.Game).Resolve(cells, move); move.Name = name; move.Angle = angle; move.BaseKnockBackSetKnockback = bkbwbkb; move.FirstActionableFrame = faf; move.KnockbackGrowth = kbg; move.LandingLag = landingLag; move.AutoCancel = autocancel; move.MoveType = MoveType.Aerial.GetEnumDescription(); move.Owner = character.Name; move.OwnerId = character.OwnerId; } return(move); }
protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character) { var move = default(IMove); if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1) { string name = GetStatName(cells[0]); //string hitboxActive = cells[1].InnerText; string faf = cells[2].InnerText; //string basedmg = cells[3].InnerText; string angle = cells[4].InnerText; string bkbwbkb = cells[5].InnerText; string kbg = cells[6].InnerText; move = ScrapingServices.CreateMove(); move = new CommonMoveParameterResolver(character.Game).Resolve(cells, move); move.Name = name; move.Angle = angle; //move.BaseDamage = basedmg; move.BaseKnockBackSetKnockback = bkbwbkb; move.FirstActionableFrame = faf; //move.HitboxActive = hitboxActive; move.KnockbackGrowth = kbg; move.MoveType = MoveType.Special.GetEnumDescription(); move.Owner = character.Name; move.OwnerId = character.OwnerId; //move.Game = character.SourceUrl.Contains("Ultimate") ? Games.Ultimate : Games.Smash4; } return(move); }
protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character) { if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1) { string name = GetStatName(cells[0]); IMove move = ScrapingServices.CreateMove(); move.Name = name; move.Owner = character.Name; move.OwnerId = character.OwnerId; move.MoveType = MoveType.Throw.GetEnumDescription(); if (name.IndexOf(ScrapingConstants.CommonMoveNames.Throw, StringComparison.OrdinalIgnoreCase) >= 0) { if (character.SourceUrl.Contains("Smash4")) //this value isn't present in Ultimate values on kh site yet. { bool isWeightDependent = TranslateYesNoToBool(cells[1].InnerText); move.IsWeightDependent = isWeightDependent; string baseDamage = new Smash4BaseDamageResolver().GetRawValue(cells[2]); move.BaseDamage = baseDamage; string angle = cells[3].InnerText; string baseKnockbackSetKnockback = cells[4].InnerText; string knockbackGrowth = cells[5].InnerText; move.Angle = angle; move.BaseKnockBackSetKnockback = baseKnockbackSetKnockback; move.KnockbackGrowth = knockbackGrowth; } else if (character.SourceUrl.Contains("Ultimate")) { string baseDamage = new UltimateBaseDamageResolver().GetRawValue(cells[1]); move.BaseDamage = baseDamage; string angle = cells[2].InnerText; string baseKnockbackSetKnockback = cells[3].InnerText; string knockbackGrowth = cells[4].InnerText; move.Angle = angle; move.BaseKnockBackSetKnockback = baseKnockbackSetKnockback; move.KnockbackGrowth = knockbackGrowth; move.Game = character.Game; } return(move); } else { return(default(IMove)); } } else { return(default(IMove)); } }
static void Main(string[] args) { NewUrlsCount = 0; NewUrlsCountThreadSafe = 0; UrlsScraped = 0; UsedCarsUrlsBag = new ConcurrentDictionary <string, UsedCarModel>(); ScrapingServices scrapingService = new ScrapingServices(); Log.Logger = new LoggerConfiguration() .MinimumLevel.Debug() .WriteTo.File("UsedCars-Scraper-logs.txt", outputTemplate: "{Timestamp} [{Level}] {Message}{NewLine}{Exception}").CreateLogger(); Stopwatch scrapingStopWatch = new Stopwatch(); Stopwatch globalStopWatch = new Stopwatch(); try { Console.WriteLine("STARTING THE PROGRAM *"); scrapingStopWatch.Start(); globalStopWatch.Start(); // 1) Scrape the adverts urls from the websites first scrapingService.ScrapeUrlsInParallel(Constants.Counties); scrapingStopWatch.Stop(); string urlsScrapingDuration = string.Format("{0:D2}:{1:D2}:{2:D2}", scrapingStopWatch.Elapsed.Hours, scrapingStopWatch.Elapsed.Minutes, scrapingStopWatch.Elapsed.Seconds); // 2) Scrape the content for each advert url scrapingStopWatch.Restart(); scrapingService.ScrapeAdverts(null); scrapingStopWatch.Stop(); globalStopWatch.Stop(); //Persist the report ScrapingServices.PersistScrapingLog( NewUrlsCountThreadSafe, urlsScrapingDuration, UrlsScraped, string.Format("{0:D2}:{1:D2}:{2:D2}", scrapingStopWatch.Elapsed.Hours, scrapingStopWatch.Elapsed.Minutes, scrapingStopWatch.Elapsed.Seconds), string.Format("{0:D2}:{1:D2}:{2:D2}", globalStopWatch.Elapsed.Hours, globalStopWatch.Elapsed.Minutes, globalStopWatch.Elapsed.Seconds), true); ScrapingServices.DeleteInvalidLinks(); Console.WriteLine("Done!"); } catch (Exception e) { Log.Information("Exception in the main."); Log.Debug(e, "{Timestamp} [{Level}] {Message}{NewLine}{Exception}"); } }
protected virtual IEnumerable <HtmlNode> GetTableRows(string sourceUrl, string xpath) { var htmlParser = ScrapingServices.CreateParserFromSourceUrl(sourceUrl); //get moves table html var movesTableHtml = htmlParser.GetSingle(xpath); var moveTableRows = HtmlNode.CreateNode(movesTableHtml)?.SelectNodes(ScrapingConstants.XPathTableRows); if (moveTableRows == null) { throw new Exception($"Error getting move table data after attempting to scrape full table using xpath: '{ScrapingConstants.XPathTableRows};"); } return(moveTableRows); }
protected override IEnumerable <HtmlNode> GetTableRows(string sourceUrl, string xpath) { const string exceptionMessageBase = "Error getting move table data after attempting to scrape full table using xpath: "; var htmlParser = ScrapingServices.CreateParserFromSourceUrl(sourceUrl); //account for extra info tables having the same id if (htmlParser.GetCollection(ScrapingConstants.XPathTableNodeGroundStats + "//thead/tr/*").Count() > 4) { string movesTableHtml = htmlParser.GetSingle(xpath); var movesTableHtmlNode = HtmlNode.CreateNode(movesTableHtml); //get row data var tableRows = movesTableHtmlNode.SelectNodes(ScrapingConstants.XPathTableRows); if (tableRows == null) { throw new Exception( $"{exceptionMessageBase}'{ScrapingConstants.XPathTableRows};"); } return(FilterHeadersFromFoundTableRows(tableRows)); } else { string movesTableHtml = htmlParser.GetSingle(ScrapingConstants.XPathTableNodeGroundStatsAdjusted); var movesTableHtmlNode = HtmlNode.CreateNode(movesTableHtml); var tableRows = movesTableHtmlNode.SelectNodes(ScrapingConstants.XPathTableRows); if (tableRows == null) { throw new Exception( $"{exceptionMessageBase}'{ScrapingConstants.XPathTableRows};"); } return(FilterHeadersFromFoundTableRows(tableRows)); } }
protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character) { var move = default(IMove); string name = GetStatName(cells[0]); //a throw move is not a ground move if (name.IndexOf(ScrapingConstants.CommonMoveNames.Throw, StringComparison.OrdinalIgnoreCase) >= 0) { return(default(IMove)); } if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1) { move = ScrapingServices.CreateMove(); move = new CommonMoveParameterResolver(character.Game).Resolve(cells, move); string faf = cells[2].InnerText; string angle = cells[4].InnerText; string bkbwbkb = cells[5].InnerText; string kbg = cells[6].InnerText; move.Name = name; move.Angle = angle; move.BaseKnockBackSetKnockback = bkbwbkb; move.FirstActionableFrame = faf; move.KnockbackGrowth = kbg; move.MoveType = MoveType.Ground.GetEnumDescription(); move.Owner = character.Name; move.OwnerId = character.OwnerId; } return(move); }