public void UpdateDatabaseFromTheGatherer()
{
// Get all search-able sets
var baseDoc = HtmlUtils.GetHTML(sTheGathereURL);
string sSetsMenuID = "ctl00_ctl00_MainContent_Content_SearchControls_setAddText";
var scripts = baseDoc.DocumentNode.SelectNodes("//script[@type='text/javascript']");
// Extract menu identifier (safety)
foreach (var script in scripts)
{
if (script.InnerText.Contains("ClientIDs"))
{
foreach (var line in script.InnerText.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
if (line.Contains("ClientIDs.setAddText"))
{
int iFirst = line.IndexOf("'") + 1;
int iLast = line.LastIndexOf("'");
sSetsMenuID = line.Substring(iFirst, iLast - iFirst);
}
}
}
}
// Get combobox options
var menuCombobox = baseDoc.DocumentNode.SelectSingleNode("//select[@id='" + sSetsMenuID + "']");
if (menuCombobox != null)
{
var lstSets = menuCombobox.SelectNodes(".//option");
foreach (var sSetID in lstSets)
{
if (sSetID.Attributes["value"].Value.Length == 0)
{
continue;
}
m_Sets.Add(new Set(sSetID.Attributes["value"].Value));
}
}
int i = 0;
foreach (var set in m_Sets)
{
++i;
Console.WriteLine("Importing Set[" + i.ToString() + "/" + m_Sets.Count.ToString() + "]: " + set.sSetName);
set.ImportFromTheGatherer();
set.Export();
}
Export();
}
public void ImportFromTheGatherer()
{
List <CardRequest> lstRequest = new List <CardRequest>();
uint iPage = 0;
int iMaxCardNumber = 0;
bool bFinished = false;
string sSetSearch = "&set=[\"" + sSetName.Replace(" ", "+") + "\"]";
while (!bFinished)
{
// Generate Search link
string sSetSearchURL = sSearchPage + iPage.ToString() + sShortType + sOutputType + sSetSearch;
var oPageSet = HtmlUtils.GetHTML(sSetSearchURL);
var lstCards = oPageSet.DocumentNode.SelectNodes("//tr[@class='cardItem']");
if (lstCards == null)
{
break;
}
foreach (var card in lstCards)
{
var numberNode = card.SelectSingleNode(".//td[@class='number']");
int iCardNumber = int.Parse(numberNode.InnerHtml);
if (iCardNumber > iMaxCardNumber)
{
iMaxCardNumber = iCardNumber;
}
else
{
bFinished = true; break;
}
var linkNode = card.SelectSingleNode(".//a[@class='nameLink']");
string sCardName = HtmlUtils.CleanInnerText(linkNode.InnerText);
string sUrl = sCardPage + linkNode.Attributes["href"].Value.Replace("../Card/Details.aspx?", "");
lstRequest.Add(new CardRequest(sCardName, sUrl));
}
++iPage;
}
int i = 0;
foreach (var oRequest in lstRequest)
{
++i;
Card oNewCard = new Card(oRequest.m_sCardName, sSetName);
Console.WriteLine("Importing Card[" + i.ToString() + "/" + lstRequest.Count + "]:" + oRequest.m_sCardName);
oNewCard.ImportFromTheGatherer(oRequest.m_sUrl);
m_lstCards.Add(oNewCard);
}
}
public void ImportFromTheGatherer(string sUrl)
{
sCardImportURL = sUrl;
if (System.IO.File.Exists(GetCardFile()))
{
ImportFromCard(XmlUtils.LoadXMLFile(GetCardFile()));
return;
}
var doc = HtmlUtils.GetHTML(sUrl).DocumentNode;
var scripts = doc.SelectNodes("//script[@type='text/javascript']");
string nameRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl02_nameRow";
string manaRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_manaRow";
string cmcRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_cmcRow";
string typeRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_typeRow";
string textRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_textRow";
string ptRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ptRow";
string setRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_setRow";
string rarityRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_rarityRow";
string numberRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_numberRow";
string artistRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_artistRow";
string imageRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl02_cardImage";
// Update row IDs
foreach (var script in scripts)
{
if (script.InnerText.Contains("ClientIDs"))
{
foreach (var line in script.InnerText.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
if (line.Contains("ClientIDs.nameRow"))
{
nameRow = GetID(line);
}
else if (line.Contains("ClientIDs.manaRow"))
{
manaRow = GetID(line);
}
else if (line.Contains("ClientIDs.cmcRow"))
{
cmcRow = GetID(line);
}
else if (line.Contains("ClientIDs.typeRow"))
{
typeRow = GetID(line);
}
else if (line.Contains("ClientIDs.textRow"))
{
textRow = GetID(line);
}
else if (line.Contains("ClientIDs.ptRow"))
{
ptRow = GetID(line);
}
else if (line.Contains("ClientIDs.setRow"))
{
setRow = GetID(line);
}
else if (line.Contains("ClientIDs.rarityRow"))
{
rarityRow = GetID(line);
}
else if (line.Contains("ClientIDs.numberRow"))
{
numberRow = GetID(line);
}
else if (line.Contains("ClientIDs.artistRow"))
{
artistRow = GetID(line);
}
else if (line.Contains("ClientIDs.cardImage"))
{
imageRow = GetID(line);
}
}
break;
}
}
// Get Name
sCardName = sType = GetInnerValue(GetRow(doc, nameRow)).Replace("/", "-").Replace("\\", "-");
// Get Set
var setNode = GetRow(doc, setRow);
if (setNode != null)
{
var textNodes = GetValueNode(setNode).SelectNodes(".//a");
foreach (var node in textNodes)
{
var expImg = node.SelectSingleNode(".//img");
if (expImg != null)
{
sSetIcon = expImg.Attributes["alt"].Value;
SaveIcon(sSetIcon, GetTheGathererImageURL(expImg.Attributes["src"].Value));
}
else
{
sSet = CleanInnerText(node.InnerText);
}
}
}
// Get Mana Cost
var manaNode = GetRow(doc, manaRow);
if (manaNode != null)
{
foreach (var node in GetValueNode(manaNode).SelectNodes(".//img"))
{
string manaType = node.Attributes["alt"].Value;
sManaCost.Add(manaType);
SaveIcon(manaType, GetTheGathererImageURL(node.Attributes["src"].Value));
}
}
// Get Converted Mana Cost
int.TryParse(GetInnerValue(GetRow(doc, cmcRow)), out iCMC);
// Get Type
sType = GetInnerValue(GetRow(doc, typeRow));
// Get Text
var textNode = GetRow(doc, textRow);
if (textNode != null)
{
foreach (var node in GetValueNode(textNode).SelectNodes(".//div[@class='cardtextbox']"))
{
sText.Add(node.InnerText);
}
}
// Get Power/Resistance
var ptNode = GetRow(doc, ptRow);
if (ptNode != null)
{
string[] iValues = GetInnerValue(ptNode).Split('/');
for (int i = 0; i < iValues.Length; ++i)
{
if (i == 0)
{
int.TryParse(iValues[0], out iPower);
}
if (i == 1)
{
int.TryParse(iValues[1], out iToughness);
}
}
}
// Get Rarity
eRarity = GetInnerValue(GetRow(doc, rarityRow));
// Get Setnumber
int.TryParse(GetInnerValue(GetRow(doc, numberRow)), out iSetNumber);
// Get Artist
sArtist = GetInnerValue(GetRow(doc, artistRow));
// Get Proxy
var proxyNode = doc.SelectSingleNode("//img[@id='" + imageRow + "']");
sCardProxyURL = GetTheGathererImageURL(proxyNode.Attributes["src"].Value);
}
