public void UpdateDatabaseFromTheGatherer() { // Get all search-able sets var baseDoc = HtmlUtils.GetHTML(sTheGathereURL); string sSetsMenuID = "ctl00_ctl00_MainContent_Content_SearchControls_setAddText"; var scripts = baseDoc.DocumentNode.SelectNodes("//script[@type='text/javascript']"); // Extract menu identifier (safety) foreach (var script in scripts) { if (script.InnerText.Contains("ClientIDs")) { foreach (var line in script.InnerText.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { if (line.Contains("ClientIDs.setAddText")) { int iFirst = line.IndexOf("'") + 1; int iLast = line.LastIndexOf("'"); sSetsMenuID = line.Substring(iFirst, iLast - iFirst); } } } } // Get combobox options var menuCombobox = baseDoc.DocumentNode.SelectSingleNode("//select[@id='" + sSetsMenuID + "']"); if (menuCombobox != null) { var lstSets = menuCombobox.SelectNodes(".//option"); foreach (var sSetID in lstSets) { if (sSetID.Attributes["value"].Value.Length == 0) { continue; } m_Sets.Add(new Set(sSetID.Attributes["value"].Value)); } } int i = 0; foreach (var set in m_Sets) { ++i; Console.WriteLine("Importing Set[" + i.ToString() + "/" + m_Sets.Count.ToString() + "]: " + set.sSetName); set.ImportFromTheGatherer(); set.Export(); } Export(); }
public void ImportFromTheGatherer() { List <CardRequest> lstRequest = new List <CardRequest>(); uint iPage = 0; int iMaxCardNumber = 0; bool bFinished = false; string sSetSearch = "&set=[\"" + sSetName.Replace(" ", "+") + "\"]"; while (!bFinished) { // Generate Search link string sSetSearchURL = sSearchPage + iPage.ToString() + sShortType + sOutputType + sSetSearch; var oPageSet = HtmlUtils.GetHTML(sSetSearchURL); var lstCards = oPageSet.DocumentNode.SelectNodes("//tr[@class='cardItem']"); if (lstCards == null) { break; } foreach (var card in lstCards) { var numberNode = card.SelectSingleNode(".//td[@class='number']"); int iCardNumber = int.Parse(numberNode.InnerHtml); if (iCardNumber > iMaxCardNumber) { iMaxCardNumber = iCardNumber; } else { bFinished = true; break; } var linkNode = card.SelectSingleNode(".//a[@class='nameLink']"); string sCardName = HtmlUtils.CleanInnerText(linkNode.InnerText); string sUrl = sCardPage + linkNode.Attributes["href"].Value.Replace("../Card/Details.aspx?", ""); lstRequest.Add(new CardRequest(sCardName, sUrl)); } ++iPage; } int i = 0; foreach (var oRequest in lstRequest) { ++i; Card oNewCard = new Card(oRequest.m_sCardName, sSetName); Console.WriteLine("Importing Card[" + i.ToString() + "/" + lstRequest.Count + "]:" + oRequest.m_sCardName); oNewCard.ImportFromTheGatherer(oRequest.m_sUrl); m_lstCards.Add(oNewCard); } }
public void ImportFromTheGatherer(string sUrl) { sCardImportURL = sUrl; if (System.IO.File.Exists(GetCardFile())) { ImportFromCard(XmlUtils.LoadXMLFile(GetCardFile())); return; } var doc = HtmlUtils.GetHTML(sUrl).DocumentNode; var scripts = doc.SelectNodes("//script[@type='text/javascript']"); string nameRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl02_nameRow"; string manaRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_manaRow"; string cmcRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_cmcRow"; string typeRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_typeRow"; string textRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_textRow"; string ptRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ptRow"; string setRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_setRow"; string rarityRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_rarityRow"; string numberRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_numberRow"; string artistRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_artistRow"; string imageRow = "ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl02_cardImage"; // Update row IDs foreach (var script in scripts) { if (script.InnerText.Contains("ClientIDs")) { foreach (var line in script.InnerText.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { if (line.Contains("ClientIDs.nameRow")) { nameRow = GetID(line); } else if (line.Contains("ClientIDs.manaRow")) { manaRow = GetID(line); } else if (line.Contains("ClientIDs.cmcRow")) { cmcRow = GetID(line); } else if (line.Contains("ClientIDs.typeRow")) { typeRow = GetID(line); } else if (line.Contains("ClientIDs.textRow")) { textRow = GetID(line); } else if (line.Contains("ClientIDs.ptRow")) { ptRow = GetID(line); } else if (line.Contains("ClientIDs.setRow")) { setRow = GetID(line); } else if (line.Contains("ClientIDs.rarityRow")) { rarityRow = GetID(line); } else if (line.Contains("ClientIDs.numberRow")) { numberRow = GetID(line); } else if (line.Contains("ClientIDs.artistRow")) { artistRow = GetID(line); } else if (line.Contains("ClientIDs.cardImage")) { imageRow = GetID(line); } } break; } } // Get Name sCardName = sType = GetInnerValue(GetRow(doc, nameRow)).Replace("/", "-").Replace("\\", "-"); // Get Set var setNode = GetRow(doc, setRow); if (setNode != null) { var textNodes = GetValueNode(setNode).SelectNodes(".//a"); foreach (var node in textNodes) { var expImg = node.SelectSingleNode(".//img"); if (expImg != null) { sSetIcon = expImg.Attributes["alt"].Value; SaveIcon(sSetIcon, GetTheGathererImageURL(expImg.Attributes["src"].Value)); } else { sSet = CleanInnerText(node.InnerText); } } } // Get Mana Cost var manaNode = GetRow(doc, manaRow); if (manaNode != null) { foreach (var node in GetValueNode(manaNode).SelectNodes(".//img")) { string manaType = node.Attributes["alt"].Value; sManaCost.Add(manaType); SaveIcon(manaType, GetTheGathererImageURL(node.Attributes["src"].Value)); } } // Get Converted Mana Cost int.TryParse(GetInnerValue(GetRow(doc, cmcRow)), out iCMC); // Get Type sType = GetInnerValue(GetRow(doc, typeRow)); // Get Text var textNode = GetRow(doc, textRow); if (textNode != null) { foreach (var node in GetValueNode(textNode).SelectNodes(".//div[@class='cardtextbox']")) { sText.Add(node.InnerText); } } // Get Power/Resistance var ptNode = GetRow(doc, ptRow); if (ptNode != null) { string[] iValues = GetInnerValue(ptNode).Split('/'); for (int i = 0; i < iValues.Length; ++i) { if (i == 0) { int.TryParse(iValues[0], out iPower); } if (i == 1) { int.TryParse(iValues[1], out iToughness); } } } // Get Rarity eRarity = GetInnerValue(GetRow(doc, rarityRow)); // Get Setnumber int.TryParse(GetInnerValue(GetRow(doc, numberRow)), out iSetNumber); // Get Artist sArtist = GetInnerValue(GetRow(doc, artistRow)); // Get Proxy var proxyNode = doc.SelectSingleNode("//img[@id='" + imageRow + "']"); sCardProxyURL = GetTheGathererImageURL(proxyNode.Attributes["src"].Value); }