private void DownloadFromURL(string iURL, string iItemName = "", CookieCollection iCookies = null)
{
VPThreading.SetText(_progressLabel, "Downloading grid from given URL...");
if (null == iCookies)
{
iCookies = Miqobot.LogInForum();
}
List <string> listGridRawContent = Miqobot.GetAllGridsRawContentFromForum(iURL, iCookies);
List <MiqobotGrid> listGrids = new List <MiqobotGrid>();
MiqoGridFinderOptions options = new MiqoGridFinderOptions();
options.Load(VPL.Application.Data.OptionLocation.GlobalOption);
List <FFXIVSearchItem> listAllGatheredItems = options.ListAllGatheredItems;
if (listAllGatheredItems.Count <= 0)
{
VPThreading.SetText(_progressLabel, "Downloading all item names...");
listAllGatheredItems = GarlandTool.Search("", null, FFXIVItem.TypeItem.Gathered);
}
options.ListAllGatheredItems = listAllGatheredItems;
options.Save();
VPThreading.SetText(_progressLabel, "Reading grids...");
foreach (string rawContent in listGridRawContent)
{
listGrids.AddRange(Miqobot.GetAllGridsFromContent(rawContent));
}
DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath());
DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid"));
DirectoryInfo analyzeDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "DownloadedGrids"));
if (!analyzeDirectory.Exists)
{
analyzeDirectory.Create();
}
if (!cacheDirectory.Exists)
{
VPThreading.SetText(_progressLabel, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
VPThreading.SetText(_progressLabel, "Matching grid to item list...");
int gridIndex = 1;
foreach (MiqobotGrid grid in listGrids)
{
List <string> listCorrespondingItemNames = new List <string>();
foreach (FFXIVSearchItem item in listAllGatheredItems)
{
if (null == grid.Description)
{
continue;
}
if (grid.Description.ToLower().Contains(item.Name.ToLower()))
{
listCorrespondingItemNames.Add(item.Name);
}
if (grid.Header.ToLower().Contains(item.Name.ToLower()))
{
listCorrespondingItemNames.Add(item.Name);
}
}
if (listCorrespondingItemNames.Count <= 0 && iItemName != "")
{
listCorrespondingItemNames.Add(iItemName);
}
List <string> listFilteredCorrespondingItemNames = new List <string>();
foreach (string itemName in listCorrespondingItemNames)
{
bool hasBigger = false;
foreach (string itemName2 in listCorrespondingItemNames)
{
if (itemName2 != itemName && itemName2.Contains(itemName))
{
hasBigger = true;
break;
}
}
if (!hasBigger)
{
listFilteredCorrespondingItemNames.Add(itemName);
}
}
foreach (string itemName in listFilteredCorrespondingItemNames)
{
//Check if grid exist
string gridItemName = itemName + " Grid";
//Looking into cache directory
FileInfo cacheGridFile = new FileInfo(Path.Combine(cacheDirectory.FullName, gridItemName + ".txt"));
if (cacheGridFile.Exists)
{
continue;
}
//Saving grid
string pathGrid = Path.Combine(analyzeDirectory.FullName, gridItemName + "---" + gridIndex + ".txt");
if (File.Exists(pathGrid))
{
File.Delete(pathGrid);
}
File.WriteAllText(pathGrid, "grid." + grid.Header + Environment.NewLine + grid.Content);
gridIndex++;
}
}
}
private void UpdateGridBDDThread()
{
List <string> _gatherableItemNames = new List <string>();
List <string> _gridOKItemNames = new List <string>();
Service_Misc.LogText(_logTextBox, "Updating database...");
MiqoCraftCore.MiqoCraftCore.DownloadGrids();
try
{
List <MiqoItemPage> result = new List <MiqoItemPage>();
CookieCollection logMiqobotCookies = Miqobot.LogInForum();
CookieCollection oCookies = new CookieCollection();
HttpStatusCode oCode = HttpStatusCode.NotFound;
HtmlAgilityPack.HtmlDocument answer = Service_Misc.GetWebPageFromRequest("GET https://miqobot.com/forum/forums/topic/index-gathering-grids/ HTTP/1.1|Host: miqobot.com|Connection: keep-alive|Cache-Control: max-age=0|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36|Sec-Fetch-Mode: navigate|Sec-Fetch-User: ?1|Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3|Sec-Fetch-Site: same-origin|Referer: https://miqobot.com/forum/forums/forum/grids-and-presets/|Accept-Encoding: gzip, deflate, br|Accept-Language: fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7|Cookie: wordpress_test_cookie=WP+Cookie+check; _ga=GA1.2.1771485810.1566089776||",
logMiqobotCookies, ref oCookies, ref oCode);
if (null == answer)
{
Service_Misc.LogText(_logTextBox, "Failed to update database. No answer from miqobot forum.");
return;
}
HtmlAgilityPack.HtmlNode firstAnswerNode = Service_Misc.GetFirstChildNode(answer.DocumentNode, "div", "topic-tag-gathering");
List <HtmlAgilityPack.HtmlNode> listItemNodes = firstAnswerNode.Descendants("li").ToList();
DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath());
DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid"));
if (!cacheDirectory.Exists)
{
Service_Misc.LogText(_logTextBox, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
foreach (HtmlAgilityPack.HtmlNode node in listItemNodes)
{
if (null == node)
{
continue;
}
string nodeInnerTextLower = node.InnerText.ToLower();
if (nodeInnerTextLower.Contains("Mining Your Own Business".ToLower()))
{
break;
}
if (nodeInnerTextLower.Contains("lv."))
{
//Found node !
string level = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[0].Trim();
string itemName = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[1].Trim().Replace("(hidden)", "").Trim();
_gatherableItemNames.Add(itemName);
if (File.Exists(Path.Combine(cacheDirectory.FullName, itemName + " Grid.txt")))
{
_gridOKItemNames.Add(itemName);
}
}
}
lock (_locker)
{
MiqoCraftOptions options = new MiqoCraftOptions();
options.Load(OptionLocation.GlobalOption);
options.ListGatherableItems = _gatherableItemNames;
options.ListGridOKItems = _gridOKItemNames;
options.Save();
}
UpdateBDDStatusFromOptions();
{
Service_Misc.LogText(_logTextBox, "Database updated !");
return;
}
}
catch
{
Service_Misc.LogText(_logTextBox, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
}
private void DownloadMissingItemGrids()
{
List <MiqoItemPage> result = new List <MiqoItemPage>();
CookieCollection logMiqobotCookies = Miqobot.LogInForum();
CookieCollection oCookies = new CookieCollection();
HttpStatusCode oCode = HttpStatusCode.NotFound;
HtmlAgilityPack.HtmlDocument answer = Service_Misc.GetWebPageFromRequest("GET https://miqobot.com/forum/forums/topic/index-gathering-grids/ HTTP/1.1|Host: miqobot.com|Connection: keep-alive|Cache-Control: max-age=0|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36|Sec-Fetch-Mode: navigate|Sec-Fetch-User: ?1|Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3|Sec-Fetch-Site: same-origin|Referer: https://miqobot.com/forum/forums/forum/grids-and-presets/|Accept-Encoding: gzip, deflate, br|Accept-Language: fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7|Cookie: wordpress_test_cookie=WP+Cookie+check; _ga=GA1.2.1771485810.1566089776||",
logMiqobotCookies, ref oCookies, ref oCode);
if (null == answer)
{
VPThreading.SetText(_progressLabel, "Failed to update database. No answer from miqobot forum.");
return;
}
HtmlAgilityPack.HtmlNode firstAnswerNode = Service_Misc.GetFirstChildNode(answer.DocumentNode, "div", "topic-tag-gathering");
List <HtmlAgilityPack.HtmlNode> listItemNodes = firstAnswerNode.Descendants("li").ToList();
DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath());
DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid"));
DirectoryInfo analyzeDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "DownloadedGrids"));
if (!analyzeDirectory.Exists)
{
analyzeDirectory.Create();
}
if (!cacheDirectory.Exists)
{
VPThreading.SetText(_progressLabel, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
int indexProg = 0;
foreach (HtmlAgilityPack.HtmlNode node in listItemNodes)
{
indexProg++;
VPThreading.SetProgress(_progressBar, indexProg * 100 / listItemNodes.Count);
if (null == node)
{
continue;
}
string nodeInnerTextLower = node.InnerText.ToLower();
if (nodeInnerTextLower.Contains("Mining Your Own Business".ToLower()))
{
break;
}
if (nodeInnerTextLower.Contains("lv."))
{
//Found node !
string level = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[0].Trim();
string itemName = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[1].Trim().Replace("(hidden)", "").Trim();
//Check if grid exist
string gridItemName = itemName + " Grid";
//Looking into cache directory
FileInfo cacheGridFile = new FileInfo(Path.Combine(cacheDirectory.FullName, gridItemName + ".txt"));
if (cacheGridFile.Exists)
{
VPThreading.SetText(_progressLabel, "Item already has a grid.");
continue;
}
VPThreading.SetText(_progressLabel, "Trying to find a new grid : " + itemName);
//Looking for all links
List <MiqoItemPage> listLinks = Miqobot.GetURLItem(itemName, logMiqobotCookies, answer);
foreach (MiqoItemPage page in listLinks)
{
//List<string> ListGrids = Miqobot.GetAllGridsFromForum(itemName, logMiqobotCookies, page);
//if (null == ListGrids) continue;
//foreach(string grid in ListGrids)
//{
// try
// {
// string pathGrid = Path.Combine(analyzeDirectory.FullName, gridItemName + "---" + gridIndex + ".txt");
// if (File.Exists(pathGrid)) File.Delete(pathGrid);
// File.WriteAllText(pathGrid, grid);
// gridIndex++;
// }
// catch
// {
// }
//}
DownloadFromURL(page.URL, itemName, logMiqobotCookies);
}
}
}
VPThreading.SetProgress(_progressBar, 100);
VPThreading.SetText(_progressLabel, "Done!");
}