private void DownloadFromURL(string iURL, string iItemName = "", CookieCollection iCookies = null) { VPThreading.SetText(_progressLabel, "Downloading grid from given URL..."); if (null == iCookies) { iCookies = Miqobot.LogInForum(); } List <string> listGridRawContent = Miqobot.GetAllGridsRawContentFromForum(iURL, iCookies); List <MiqobotGrid> listGrids = new List <MiqobotGrid>(); MiqoGridFinderOptions options = new MiqoGridFinderOptions(); options.Load(VPL.Application.Data.OptionLocation.GlobalOption); List <FFXIVSearchItem> listAllGatheredItems = options.ListAllGatheredItems; if (listAllGatheredItems.Count <= 0) { VPThreading.SetText(_progressLabel, "Downloading all item names..."); listAllGatheredItems = GarlandTool.Search("", null, FFXIVItem.TypeItem.Gathered); } options.ListAllGatheredItems = listAllGatheredItems; options.Save(); VPThreading.SetText(_progressLabel, "Reading grids..."); foreach (string rawContent in listGridRawContent) { listGrids.AddRange(Miqobot.GetAllGridsFromContent(rawContent)); } DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath()); DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid")); DirectoryInfo analyzeDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "DownloadedGrids")); if (!analyzeDirectory.Exists) { analyzeDirectory.Create(); } if (!cacheDirectory.Exists) { VPThreading.SetText(_progressLabel, "Failed to compute database status, CacheGrid directory does not exist."); return; } VPThreading.SetText(_progressLabel, "Matching grid to item list..."); int gridIndex = 1; foreach (MiqobotGrid grid in listGrids) { List <string> listCorrespondingItemNames = new List <string>(); foreach (FFXIVSearchItem item in listAllGatheredItems) { if (null == grid.Description) { continue; } if (grid.Description.ToLower().Contains(item.Name.ToLower())) { listCorrespondingItemNames.Add(item.Name); } if (grid.Header.ToLower().Contains(item.Name.ToLower())) { listCorrespondingItemNames.Add(item.Name); } } if (listCorrespondingItemNames.Count <= 0 && iItemName != "") { listCorrespondingItemNames.Add(iItemName); } List <string> listFilteredCorrespondingItemNames = new List <string>(); foreach (string itemName in listCorrespondingItemNames) { bool hasBigger = false; foreach (string itemName2 in listCorrespondingItemNames) { if (itemName2 != itemName && itemName2.Contains(itemName)) { hasBigger = true; break; } } if (!hasBigger) { listFilteredCorrespondingItemNames.Add(itemName); } } foreach (string itemName in listFilteredCorrespondingItemNames) { //Check if grid exist string gridItemName = itemName + " Grid"; //Looking into cache directory FileInfo cacheGridFile = new FileInfo(Path.Combine(cacheDirectory.FullName, gridItemName + ".txt")); if (cacheGridFile.Exists) { continue; } //Saving grid string pathGrid = Path.Combine(analyzeDirectory.FullName, gridItemName + "---" + gridIndex + ".txt"); if (File.Exists(pathGrid)) { File.Delete(pathGrid); } File.WriteAllText(pathGrid, "grid." + grid.Header + Environment.NewLine + grid.Content); gridIndex++; } } }
private void UpdateGridBDDThread() { List <string> _gatherableItemNames = new List <string>(); List <string> _gridOKItemNames = new List <string>(); Service_Misc.LogText(_logTextBox, "Updating database..."); MiqoCraftCore.MiqoCraftCore.DownloadGrids(); try { List <MiqoItemPage> result = new List <MiqoItemPage>(); CookieCollection logMiqobotCookies = Miqobot.LogInForum(); CookieCollection oCookies = new CookieCollection(); HttpStatusCode oCode = HttpStatusCode.NotFound; HtmlAgilityPack.HtmlDocument answer = Service_Misc.GetWebPageFromRequest("GET https://miqobot.com/forum/forums/topic/index-gathering-grids/ HTTP/1.1|Host: miqobot.com|Connection: keep-alive|Cache-Control: max-age=0|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36|Sec-Fetch-Mode: navigate|Sec-Fetch-User: ?1|Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3|Sec-Fetch-Site: same-origin|Referer: https://miqobot.com/forum/forums/forum/grids-and-presets/|Accept-Encoding: gzip, deflate, br|Accept-Language: fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7|Cookie: wordpress_test_cookie=WP+Cookie+check; _ga=GA1.2.1771485810.1566089776||", logMiqobotCookies, ref oCookies, ref oCode); if (null == answer) { Service_Misc.LogText(_logTextBox, "Failed to update database. No answer from miqobot forum."); return; } HtmlAgilityPack.HtmlNode firstAnswerNode = Service_Misc.GetFirstChildNode(answer.DocumentNode, "div", "topic-tag-gathering"); List <HtmlAgilityPack.HtmlNode> listItemNodes = firstAnswerNode.Descendants("li").ToList(); DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath()); DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid")); if (!cacheDirectory.Exists) { Service_Misc.LogText(_logTextBox, "Failed to compute database status, CacheGrid directory does not exist."); return; } foreach (HtmlAgilityPack.HtmlNode node in listItemNodes) { if (null == node) { continue; } string nodeInnerTextLower = node.InnerText.ToLower(); if (nodeInnerTextLower.Contains("Mining Your Own Business".ToLower())) { break; } if (nodeInnerTextLower.Contains("lv.")) { //Found node ! string level = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[0].Trim(); string itemName = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[1].Trim().Replace("(hidden)", "").Trim(); _gatherableItemNames.Add(itemName); if (File.Exists(Path.Combine(cacheDirectory.FullName, itemName + " Grid.txt"))) { _gridOKItemNames.Add(itemName); } } } lock (_locker) { MiqoCraftOptions options = new MiqoCraftOptions(); options.Load(OptionLocation.GlobalOption); options.ListGatherableItems = _gatherableItemNames; options.ListGridOKItems = _gridOKItemNames; options.Save(); } UpdateBDDStatusFromOptions(); { Service_Misc.LogText(_logTextBox, "Database updated !"); return; } } catch { Service_Misc.LogText(_logTextBox, "Failed to compute database status, CacheGrid directory does not exist."); return; } }
private void DownloadMissingItemGrids() { List <MiqoItemPage> result = new List <MiqoItemPage>(); CookieCollection logMiqobotCookies = Miqobot.LogInForum(); CookieCollection oCookies = new CookieCollection(); HttpStatusCode oCode = HttpStatusCode.NotFound; HtmlAgilityPack.HtmlDocument answer = Service_Misc.GetWebPageFromRequest("GET https://miqobot.com/forum/forums/topic/index-gathering-grids/ HTTP/1.1|Host: miqobot.com|Connection: keep-alive|Cache-Control: max-age=0|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36|Sec-Fetch-Mode: navigate|Sec-Fetch-User: ?1|Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3|Sec-Fetch-Site: same-origin|Referer: https://miqobot.com/forum/forums/forum/grids-and-presets/|Accept-Encoding: gzip, deflate, br|Accept-Language: fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7|Cookie: wordpress_test_cookie=WP+Cookie+check; _ga=GA1.2.1771485810.1566089776||", logMiqobotCookies, ref oCookies, ref oCode); if (null == answer) { VPThreading.SetText(_progressLabel, "Failed to update database. No answer from miqobot forum."); return; } HtmlAgilityPack.HtmlNode firstAnswerNode = Service_Misc.GetFirstChildNode(answer.DocumentNode, "div", "topic-tag-gathering"); List <HtmlAgilityPack.HtmlNode> listItemNodes = firstAnswerNode.Descendants("li").ToList(); DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath()); DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid")); DirectoryInfo analyzeDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "DownloadedGrids")); if (!analyzeDirectory.Exists) { analyzeDirectory.Create(); } if (!cacheDirectory.Exists) { VPThreading.SetText(_progressLabel, "Failed to compute database status, CacheGrid directory does not exist."); return; } int indexProg = 0; foreach (HtmlAgilityPack.HtmlNode node in listItemNodes) { indexProg++; VPThreading.SetProgress(_progressBar, indexProg * 100 / listItemNodes.Count); if (null == node) { continue; } string nodeInnerTextLower = node.InnerText.ToLower(); if (nodeInnerTextLower.Contains("Mining Your Own Business".ToLower())) { break; } if (nodeInnerTextLower.Contains("lv.")) { //Found node ! string level = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[0].Trim(); string itemName = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[1].Trim().Replace("(hidden)", "").Trim(); //Check if grid exist string gridItemName = itemName + " Grid"; //Looking into cache directory FileInfo cacheGridFile = new FileInfo(Path.Combine(cacheDirectory.FullName, gridItemName + ".txt")); if (cacheGridFile.Exists) { VPThreading.SetText(_progressLabel, "Item already has a grid."); continue; } VPThreading.SetText(_progressLabel, "Trying to find a new grid : " + itemName); //Looking for all links List <MiqoItemPage> listLinks = Miqobot.GetURLItem(itemName, logMiqobotCookies, answer); foreach (MiqoItemPage page in listLinks) { //List<string> ListGrids = Miqobot.GetAllGridsFromForum(itemName, logMiqobotCookies, page); //if (null == ListGrids) continue; //foreach(string grid in ListGrids) //{ // try // { // string pathGrid = Path.Combine(analyzeDirectory.FullName, gridItemName + "---" + gridIndex + ".txt"); // if (File.Exists(pathGrid)) File.Delete(pathGrid); // File.WriteAllText(pathGrid, grid); // gridIndex++; // } // catch // { // } //} DownloadFromURL(page.URL, itemName, logMiqobotCookies); } } } VPThreading.SetProgress(_progressBar, 100); VPThreading.SetText(_progressLabel, "Done!"); }