protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character)
{
var move = default(IMove);
if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1)
{
string name = GetStatName(cells[0]);
string faf = cells[2].InnerText;
string angle = cells[4].InnerText;
string bkbwbkb = cells[5].InnerText;
string kbg = cells[6].InnerText;
string landingLag = cells[7].InnerText;
var autocancel = new CommonAutocancelParameterResolver().Resolve(cells[8]);
move = ScrapingServices.CreateMove();
move = new CommonMoveParameterResolver(character.Game).Resolve(cells, move);
move.Name = name;
move.Angle = angle;
move.BaseKnockBackSetKnockback = bkbwbkb;
move.FirstActionableFrame = faf;
move.KnockbackGrowth = kbg;
move.LandingLag = landingLag;
move.AutoCancel = autocancel;
move.MoveType = MoveType.Aerial.GetEnumDescription();
move.Owner = character.Name;
move.OwnerId = character.OwnerId;
}
return(move);
}
protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character)
{
var move = default(IMove);
if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1)
{
string name = GetStatName(cells[0]);
//string hitboxActive = cells[1].InnerText;
string faf = cells[2].InnerText;
//string basedmg = cells[3].InnerText;
string angle = cells[4].InnerText;
string bkbwbkb = cells[5].InnerText;
string kbg = cells[6].InnerText;
move = ScrapingServices.CreateMove();
move = new CommonMoveParameterResolver(character.Game).Resolve(cells, move);
move.Name = name;
move.Angle = angle;
//move.BaseDamage = basedmg;
move.BaseKnockBackSetKnockback = bkbwbkb;
move.FirstActionableFrame = faf;
//move.HitboxActive = hitboxActive;
move.KnockbackGrowth = kbg;
move.MoveType = MoveType.Special.GetEnumDescription();
move.Owner = character.Name;
move.OwnerId = character.OwnerId;
//move.Game = character.SourceUrl.Contains("Ultimate") ? Games.Ultimate : Games.Smash4;
}
return(move);
}
protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character)
{
if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1)
{
string name = GetStatName(cells[0]);
IMove move = ScrapingServices.CreateMove();
move.Name = name;
move.Owner = character.Name;
move.OwnerId = character.OwnerId;
move.MoveType = MoveType.Throw.GetEnumDescription();
if (name.IndexOf(ScrapingConstants.CommonMoveNames.Throw, StringComparison.OrdinalIgnoreCase) >= 0)
{
if (character.SourceUrl.Contains("Smash4")) //this value isn't present in Ultimate values on kh site yet.
{
bool isWeightDependent = TranslateYesNoToBool(cells[1].InnerText);
move.IsWeightDependent = isWeightDependent;
string baseDamage = new Smash4BaseDamageResolver().GetRawValue(cells[2]);
move.BaseDamage = baseDamage;
string angle = cells[3].InnerText;
string baseKnockbackSetKnockback = cells[4].InnerText;
string knockbackGrowth = cells[5].InnerText;
move.Angle = angle;
move.BaseKnockBackSetKnockback = baseKnockbackSetKnockback;
move.KnockbackGrowth = knockbackGrowth;
}
else if (character.SourceUrl.Contains("Ultimate"))
{
string baseDamage = new UltimateBaseDamageResolver().GetRawValue(cells[1]);
move.BaseDamage = baseDamage;
string angle = cells[2].InnerText;
string baseKnockbackSetKnockback = cells[3].InnerText;
string knockbackGrowth = cells[4].InnerText;
move.Angle = angle;
move.BaseKnockBackSetKnockback = baseKnockbackSetKnockback;
move.KnockbackGrowth = knockbackGrowth;
move.Game = character.Game;
}
return(move);
}
else
{
return(default(IMove));
}
}
else
{
return(default(IMove));
}
}
static void Main(string[] args)
{
NewUrlsCount = 0;
NewUrlsCountThreadSafe = 0;
UrlsScraped = 0;
UsedCarsUrlsBag = new ConcurrentDictionary <string, UsedCarModel>();
ScrapingServices scrapingService = new ScrapingServices();
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.File("UsedCars-Scraper-logs.txt", outputTemplate: "{Timestamp} [{Level}] {Message}{NewLine}{Exception}").CreateLogger();
Stopwatch scrapingStopWatch = new Stopwatch();
Stopwatch globalStopWatch = new Stopwatch();
try
{
Console.WriteLine("STARTING THE PROGRAM *");
scrapingStopWatch.Start();
globalStopWatch.Start();
// 1) Scrape the adverts urls from the websites first
scrapingService.ScrapeUrlsInParallel(Constants.Counties);
scrapingStopWatch.Stop();
string urlsScrapingDuration = string.Format("{0:D2}:{1:D2}:{2:D2}", scrapingStopWatch.Elapsed.Hours, scrapingStopWatch.Elapsed.Minutes, scrapingStopWatch.Elapsed.Seconds);
// 2) Scrape the content for each advert url
scrapingStopWatch.Restart();
scrapingService.ScrapeAdverts(null);
scrapingStopWatch.Stop();
globalStopWatch.Stop();
//Persist the report
ScrapingServices.PersistScrapingLog(
NewUrlsCountThreadSafe,
urlsScrapingDuration,
UrlsScraped,
string.Format("{0:D2}:{1:D2}:{2:D2}", scrapingStopWatch.Elapsed.Hours, scrapingStopWatch.Elapsed.Minutes, scrapingStopWatch.Elapsed.Seconds),
string.Format("{0:D2}:{1:D2}:{2:D2}", globalStopWatch.Elapsed.Hours, globalStopWatch.Elapsed.Minutes, globalStopWatch.Elapsed.Seconds),
true);
ScrapingServices.DeleteInvalidLinks();
Console.WriteLine("Done!");
}
catch (Exception e)
{
Log.Information("Exception in the main.");
Log.Debug(e, "{Timestamp} [{Level}] {Message}{NewLine}{Exception}");
}
}
protected virtual IEnumerable <HtmlNode> GetTableRows(string sourceUrl, string xpath)
{
var htmlParser = ScrapingServices.CreateParserFromSourceUrl(sourceUrl);
//get moves table html
var movesTableHtml = htmlParser.GetSingle(xpath);
var moveTableRows = HtmlNode.CreateNode(movesTableHtml)?.SelectNodes(ScrapingConstants.XPathTableRows);
if (moveTableRows == null)
{
throw new Exception($"Error getting move table data after attempting to scrape full table using xpath: '{ScrapingConstants.XPathTableRows};");
}
return(moveTableRows);
}
protected override IEnumerable <HtmlNode> GetTableRows(string sourceUrl, string xpath)
{
const string exceptionMessageBase = "Error getting move table data after attempting to scrape full table using xpath: ";
var htmlParser = ScrapingServices.CreateParserFromSourceUrl(sourceUrl);
//account for extra info tables having the same id
if (htmlParser.GetCollection(ScrapingConstants.XPathTableNodeGroundStats + "//thead/tr/*").Count() > 4)
{
string movesTableHtml = htmlParser.GetSingle(xpath);
var movesTableHtmlNode = HtmlNode.CreateNode(movesTableHtml);
//get row data
var tableRows = movesTableHtmlNode.SelectNodes(ScrapingConstants.XPathTableRows);
if (tableRows == null)
{
throw new Exception(
$"{exceptionMessageBase}'{ScrapingConstants.XPathTableRows};");
}
return(FilterHeadersFromFoundTableRows(tableRows));
}
else
{
string movesTableHtml = htmlParser.GetSingle(ScrapingConstants.XPathTableNodeGroundStatsAdjusted);
var movesTableHtmlNode = HtmlNode.CreateNode(movesTableHtml);
var tableRows = movesTableHtmlNode.SelectNodes(ScrapingConstants.XPathTableRows);
if (tableRows == null)
{
throw new Exception(
$"{exceptionMessageBase}'{ScrapingConstants.XPathTableRows};");
}
return(FilterHeadersFromFoundTableRows(tableRows));
}
}
protected override IMove GetMove(HtmlNodeCollection cells, WebCharacter character)
{
var move = default(IMove);
string name = GetStatName(cells[0]);
//a throw move is not a ground move
if (name.IndexOf(ScrapingConstants.CommonMoveNames.Throw, StringComparison.OrdinalIgnoreCase) >= 0)
{
return(default(IMove));
}
if (!string.IsNullOrEmpty(cells[0].InnerText) && cells.Count > 1)
{
move = ScrapingServices.CreateMove();
move = new CommonMoveParameterResolver(character.Game).Resolve(cells, move);
string faf = cells[2].InnerText;
string angle = cells[4].InnerText;
string bkbwbkb = cells[5].InnerText;
string kbg = cells[6].InnerText;
move.Name = name;
move.Angle = angle;
move.BaseKnockBackSetKnockback = bkbwbkb;
move.FirstActionableFrame = faf;
move.KnockbackGrowth = kbg;
move.MoveType = MoveType.Ground.GetEnumDescription();
move.Owner = character.Name;
move.OwnerId = character.OwnerId;
}
return(move);
}
