public static IEnumerable <AttributeScraper> AllWithScrapingServices(IAttributeScrapingServices scrapingServices, string baseUrl = "")
{
Guard.VerifyObjectNotNull(scrapingServices, nameof(scrapingServices));
return(new List <AttributeScraper>
{
new AttributeScraper(baseUrl, scrapingServices, AerialJump),
new AttributeScraper(baseUrl, scrapingServices, AirAcceleration),
new AttributeScraper(baseUrl, scrapingServices, AirDodge),
new AttributeScraper(baseUrl, scrapingServices, AirFriction),
new AttributeScraper(baseUrl, scrapingServices, AirSpeed),
new AttributeScraper(baseUrl, scrapingServices, Counters),
new AttributeScraper(baseUrl, scrapingServices, FallSpeed),
new AttributeScraper(baseUrl, scrapingServices, FullHop),
new AttributeScraper(baseUrl, scrapingServices, Gravity),
new AttributeScraper(baseUrl, scrapingServices, JumpSquat),
new AttributeScraper(baseUrl, scrapingServices, LedgeHop),
new AttributeScraper(baseUrl, scrapingServices, Reflector),
new AttributeScraper(baseUrl, scrapingServices, Rolls),
new AttributeScraper(baseUrl, scrapingServices, RunSpeed),
new AttributeScraper(baseUrl, scrapingServices, ShieldSize),
new AttributeScraper(baseUrl, scrapingServices, ShortHop),
new AttributeScraper(baseUrl, scrapingServices, Trip, HardTrip, ScrapingConstants.XPathTableNodeAttributesFirstTable),
new AttributeScraper(baseUrl, scrapingServices, Trip, SoftTrip, ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, Spotdodge),
new AttributeScraper(baseUrl, scrapingServices, Traction, xpathToTable: ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, GetUpStand, GetUpStandOnBack, ScrapingConstants.XPathTableNodeAttributesFirstTable),
new AttributeScraper(baseUrl, scrapingServices, GetUpStand, GetUpStandOnFront, ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, GetUpBackRoll, GetUpBackRollBack, ScrapingConstants.XPathTableNodeAttributesFirstTable),
new AttributeScraper(baseUrl, scrapingServices, GetUpBackRoll, GetUpBackRollFront, ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, GetUpForwardRoll, GetUpForwardRollBack, ScrapingConstants.XPathTableNodeAttributesFirstTable),
new AttributeScraper(baseUrl, scrapingServices, GetUpForwardRoll, GetUpForwardRollFront, ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, Tech, TechFrameData, ScrapingConstants.XPathTableNodeAttributesFirstTable),
new AttributeScraper(baseUrl, scrapingServices, Tech, TechRollForward, ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, Tech, TechRollBackward, ScrapingConstants.XPathTableNodeAttributesThirdTable),
new AttributeScraper(baseUrl, scrapingServices, LedgeAttack),
new AttributeScraper(baseUrl, scrapingServices, LedgeGetUp),
new AttributeScraper(baseUrl, scrapingServices, LedgeRoll),
new AttributeScraper(baseUrl, scrapingServices, JabLock, JabLockFront, ScrapingConstants.XPathTableNodeAttributesFirstTable),
new AttributeScraper(baseUrl, scrapingServices, JabLock, JabLockBack, ScrapingConstants.XPathTableNodeAttributesSecondTable),
new AttributeScraper(baseUrl, scrapingServices, LedgeJump),
new AttributeScraper(baseUrl, scrapingServices, WalkSpeed),
new AttributeScraper(baseUrl, scrapingServices, Weight)
});
}
public void SetUp()
{
var instanceIdGenerator = new InstanceIdGenerator();
_htmlParserProvider = new DefaultHtmlParserProvider();
_movementProvider = new DefaultMovementProvider(instanceIdGenerator);
_moveProvider = new DefaultMoveProvider(instanceIdGenerator);
_pageDownloader = new DefaultPageDownloader();
_webClientProvider = new DefaultWebClientProvider();
_attributeProvider = new DefaultAttributeProvider(instanceIdGenerator);
_imageScrapingProvider = new DefaultImageScrapingProvider();
_imageScrapingService = new DefaultImageScrapingService(_imageScrapingProvider);
_uniqueDataProvider = new DefaultUniqueDataProvider(instanceIdGenerator);
_webServices = new DefaultWebServices(_htmlParserProvider, _webClientProvider, _pageDownloader);
_attributeScrapingServices = new DefaultAttributeScrapingServices(_attributeProvider, _webServices);
_moveScrapingServices = new DefaultMoveScrapingServices(_moveProvider, _webServices);
_movementScrapingServices = new DefaultMovementScrapingServices(_movementProvider, _webServices);
_uniqueDataScrapingServices = new DefaultUniqueDataScrapingServices(_uniqueDataProvider, _webServices);
_groundMoveScraper = new GroundMoveScraper(_moveScrapingServices);
_aerialMoveScraper = new AerialMoveScraper(_moveScrapingServices);
_specialMoveScraper = new SpecialMoveScraper(_moveScrapingServices);
_throwMovesScraper = new ThrowMoveScraper(_moveScrapingServices);
_characterMoveScraper = new DefaultCharacterMoveScraper(new List <IMoveScraper>
{
_groundMoveScraper, _aerialMoveScraper, _specialMoveScraper, _throwMovesScraper
});
var attributeScrapers = new List <IAttributeScraper>
{
new AirSpeedScraper(_attributeScrapingServices),
new AirDodgeScraper(_attributeScrapingServices)
};
_movementScraper = new DefaultMovementScraper(_movementScrapingServices);
_characterDataScrapingServices = new DefaultCharacterDataScrapingServices(_imageScrapingService, _movementScraper,
attributeScrapers, _characterMoveScraper, _uniqueDataScrapingServices, _webServices, instanceIdGenerator);
_characterDataScraper = new DefaultCharacterDataScraper(_characterDataScrapingServices);
}
public DefaultCharacterDataScraper MakeCharacterDataScraper()
{
var instanceIdGenerator = new InstanceIdGenerator();
_htmlParserProvider = new DefaultHtmlParserProvider();
_movementProvider = new DefaultMovementProvider(instanceIdGenerator);
_moveProvider = new DefaultMoveProvider(instanceIdGenerator);
_pageDownloader = new DefaultPageDownloader();
_webClientProvider = new DefaultWebClientProvider();
_attributeProvider = new DefaultAttributeProvider(instanceIdGenerator);
_imageScrapingProvider = new DefaultImageScrapingProvider();
_imageScrapingService = new DefaultColorScrapingService(_characterCss); //_imageScrapingProvider);
_uniqueDataProvider = new DefaultUniqueDataProvider(instanceIdGenerator);
_webServices = new DefaultWebServices(_htmlParserProvider, _webClientProvider, _pageDownloader);
_attributeScrapingServices = new DefaultAttributeScrapingServices(_attributeProvider, _webServices);
_moveScrapingServices = new DefaultMoveScrapingServices(_moveProvider, _webServices);
_movementScrapingServices = new DefaultMovementScrapingServices(_movementProvider, _webServices);
_uniqueDataScrapingServices = new DefaultUniqueDataScrapingServices(_uniqueDataProvider, _webServices);
_groundMoveScraper = new GroundMoveScraper(_moveScrapingServices);
_aerialMoveScraper = new AerialMoveScraper(_moveScrapingServices);
_specialMoveScraper = new SpecialMoveScraper(_moveScrapingServices);
_throwMoveScraper = new ThrowMoveScraper(_moveScrapingServices);
_characterMoveScraper = new DefaultCharacterMoveScraper(new List <IMoveScraper> {
_groundMoveScraper, _aerialMoveScraper, _specialMoveScraper, _throwMoveScraper
});
var attributeScrapers = AttributeScrapers.AllWithScrapingServices(_attributeScrapingServices, _urlUnderTest);
_movementScraper = new DefaultMovementScraper(_movementScrapingServices);
_characterDataScrapingServices = new DefaultCharacterDataScrapingServices(_imageScrapingService, _movementScraper,
attributeScrapers, _characterMoveScraper, _uniqueDataScrapingServices, _webServices, instanceIdGenerator);
return(new DefaultCharacterDataScraper(_characterDataScrapingServices));
}
public AttributeScraper(string baseUrl, IAttributeScrapingServices scrapingServices, string attributeName,
string attributeDisplayName = "", string xpathToTable = "")
{
Guard.VerifyObjectNotNull(scrapingServices, nameof(scrapingServices));
ScrapingServices = scrapingServices;
//these are only switched for smash4 data. Ultimate has both dashspeed and runspeed...
if (attributeName == "RunSpeed" && baseUrl.Contains("Smash4"))
{
AttributeName = "DashSpeed"; //grrr..
}
else
{
AttributeName = attributeName;
}
if (string.IsNullOrEmpty(attributeDisplayName))
{
AttributeDisplayName = AttributeName;
}
else
{
AttributeDisplayName = attributeDisplayName;
}
SourceUrl = $"{baseUrl}{AttributeName}";
Scrape = character =>
{
Guard.VerifyObjectNotNull(character, nameof(character));
Guard.VerifyStringIsNotNullOrEmpty(character.Name, nameof(character.Name));
var attributeValueRows = new List <ICharacterAttributeRow>();
var htmlParser = ScrapingServices.CreateParserFromSourceUrl(SourceUrl);
string attributeTableHtml = string.Empty;
if (string.IsNullOrEmpty(xpathToTable))
{
attributeTableHtml =
htmlParser.GetSingle(ScrapingConstants.XPathTableNodeAttributesSecondTable) ??
htmlParser.GetSingle(ScrapingConstants.XPathTableNodeAttributesFirstTable);
}
else
{
attributeTableHtml = htmlParser.GetSingle(xpathToTable);
}
string xpath = ScrapingConstants.XPathEveryoneElseTableRow.Replace(ScrapingConstants.EveryoneKey, $"contains(., \"{character.DisplayName}\")");
var tableHtmlNode = HtmlNode.CreateNode(attributeTableHtml);
//scrape using default character name
var attributeTableRows = tableHtmlNode?.SelectNodes(xpath);
var rows = new List <HtmlNode>();
if (attributeTableRows == null)
{
//try all configured potential names for the character
foreach (string name in character.PotentialScrapingNames)
{
string altCharacterNameXPath =
ScrapingConstants.XPathEveryoneElseTableRow.Replace(
ScrapingConstants.EveryoneKey, $"contains(text(), \"{name}\")");
attributeTableRows = tableHtmlNode?.SelectNodes(altCharacterNameXPath);
if (attributeTableRows != null)
{
rows.AddRange(attributeTableRows.ToList());
}
}
//assume it's lumped in with everyone else value
if (attributeTableRows == null)
{
attributeTableRows =
tableHtmlNode?
.SelectNodes(ScrapingConstants.XPathEveryoneElseTableRow);
if (attributeTableRows != null)
{
rows.AddRange(attributeTableRows.ToList());
}
}
}
else
{
rows.AddRange(attributeTableRows.ToList());
}
var headers = GetHeaders(htmlParser);
foreach (var row in rows)
{
var attributeValues = new List <IAttribute>();
var cells = row.SelectNodes(ScrapingConstants.XPathTableCells);
for (int i = 0; i < cells.Count; i++)
{
string headerValue = headers[i];
if (headerValue.Equals("character", StringComparison.OrdinalIgnoreCase) ||
headerValue.Equals("rank", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var attributeValue = ScrapingServices.CreateAttribute();
attributeValue.Name = headerValue;
attributeValue.Value = cells[i].InnerText.Replace("×", "x").Replace("%", "%");
attributeValue.Owner = character.Name;
attributeValue.OwnerId = character.OwnerId;
attributeValues.Add(attributeValue);
}
var characterAttributeRow = ScrapingServices.CreateCharacterAttributeRow();
characterAttributeRow.Values = attributeValues;
characterAttributeRow.Name = string.IsNullOrEmpty(attributeDisplayName) ? AttributeName : attributeDisplayName;
characterAttributeRow.Owner = character.Name;
characterAttributeRow.OwnerId = character.OwnerId;
characterAttributeRow.Game = SourceUrl.Contains("Ultimate") ? Games.Ultimate : Games.Smash4;
attributeValueRows.Add(characterAttributeRow);
}
return(attributeValueRows);
};
}
