private static IEnumerable <CsvItem> GetChineseItems(Cyalume lumina, HttpClient http)
{
var rawData = http.GetStreamAsync(new Uri("https://raw.githubusercontent.com/thewakingsands/ffxiv-datamining-cn/master/Item.csv")).GetAwaiter().GetResult();
using var sr = new StreamReader(rawData);
using var csv = new CsvReader(sr, CultureInfo.InvariantCulture);
var iscSheet = lumina.GetExcelSheet <ItemSearchCategory>();
var items = new List <CsvItem>();
for (var i = 0; i < 3; i++)
{
csv.Read();
}
while (csv.Read())
{
var itemSearchCategory = csv.GetField <int>(17);
items.Add(new CsvItem
{
Key = csv.GetField <int>(0),
Name = csv.GetField <string>(10),
LevelItem = csv.GetField <int>(12),
Rarity = csv.GetField <int>(13),
ItemSearchCategory = iscSheet.First(isc => isc.RowId == itemSearchCategory),
});
}
return(items);
}
static void Main(string[] args)
{
var mode = args[1];
var outputPath = Path.Combine("..", "..", "..", "..", "..", "public", "i", "icon2x");
Directory.CreateDirectory(outputPath);
var luminaJp = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.Japanese
});
switch (mode)
{
case "alllist":
MakeListAll(luminaJp, outputPath);
return;
case "allicon":
DownloadIconAll(outputPath);
return;
}
}
private static void MakeListAll(Cyalume lumina, string outputPath)
{
Console.WriteLine("Starting game data export...");
var itemsJp = lumina.GetExcelSheet <Item>();
foreach (var category in lumina.GetExcelSheet <ItemSearchCategory>())
{
// We don't need those, not for sale
if (category.RowId == 0)
{
continue;
}
foreach (var item in itemsJp.Where(item => item.ItemSearchCategory.Value.RowId == category.RowId))
{
marketableDict.Add(item.Name, (int)item.RowId);
}
}
Console.WriteLine($"{marketableDict.Count} marketable items.");
var pages = GetPageCount();
Parallel.For(1, pages, i =>
{
// Can't access i in there cause it gets modified
var thisPage = i;
var searchPage = Retry.Do(() => Get(GetSearchUrl(thisPage)), TimeSpan.FromSeconds(20), 100);
var tableNode =
searchPage.DocumentNode.SelectSingleNode(
"/html/body/div[3]/div[2]/div[1]/div/div[2]/div[2]/div[5]/div/table/tbody");
var tableEntries = tableNode.SelectNodes("tr");
Console.WriteLine($"=> Page {i}");
foreach (var tableEntry in tableEntries)
{
var itemRow = tableEntry.ChildNodes[1];
var itemDivs = itemRow.ChildNodes.Where(x => x.Name == "div");
var item1 = itemDivs.ElementAt(1);
var item2 = item1.ChildNodes[3];
var itemUrl = item2.GetAttributeValue("href", string.Empty);
var itemName = item2.InnerHtml;
Console.WriteLine($" => {itemName}: {itemUrl}");
if (marketableDict.TryGetValue(itemName, out var key))
{
lock (eorzeaDbDict)
eorzeaDbDict.Add(key, itemUrl);
Console.WriteLine($" => MARKETABLE");
}
else
{
Console.WriteLine($" => NOT MARKETABLE");
}
}
lock (eorzeaDbDict)
{
File.WriteAllText(Path.Combine(outputPath, "dbMapping.json"),
JsonConvert.SerializeObject(eorzeaDbDict, Formatting.Indented));
}
});
}
public static void Main(string[] args)
{
var outputPath = Path.Combine("..", "..", "..", "..", "..", "DataExports");
var categoryJsOutputPath = Path.Combine("..", "..", "..", "..", "..", "public", "data");
var http = new HttpClient();
if (Directory.Exists(outputPath))
{
Directory.Delete(outputPath, true);
}
Directory.CreateDirectory(outputPath);
var lumina = new Cyalume(args[0], new LuminaOptions {
PanicOnSheetChecksumMismatch = false
});
var luminaDe = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.German, PanicOnSheetChecksumMismatch = false
});
var luminaFr = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.French, PanicOnSheetChecksumMismatch = false
});
var luminaJp = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.Japanese, PanicOnSheetChecksumMismatch = false
});
var items = lumina.GetExcelSheet <Item>().ToList();
var itemsDe = luminaDe.GetExcelSheet <Item>().ToList();
var itemsFr = luminaFr.GetExcelSheet <Item>().ToList();
var itemsJp = luminaJp.GetExcelSheet <Item>().ToList();
var itemsChs = GetChineseItems(lumina, http).ToList();
var categories = lumina.GetExcelSheet <ItemSearchCategory>().ToList();
var categoriesDe = luminaDe.GetExcelSheet <ItemSearchCategory>().ToList();
var categoriesFr = luminaFr.GetExcelSheet <ItemSearchCategory>().ToList();
var categoriesJa = luminaJp.GetExcelSheet <ItemSearchCategory>().ToList();
Console.WriteLine($"Global items: {items.Count}");
Console.WriteLine($"CN items: {itemsChs.Count}");
Console.WriteLine("Starting game data export...");
categoryjs:
Console.WriteLine("== Category JS Export ==");
CategoryJsExports.Generate(lumina, luminaDe, luminaFr, luminaJp, categoryJsOutputPath);
CategoryJsExports.GenerateChinese(lumina, itemsChs, categoryJsOutputPath);
items:
Console.WriteLine("== Item Export ==");
ItemExports.GenerateItemJSON(items, itemsDe, itemsFr, itemsJp, itemsChs, categories, outputPath);
marketableitems:
Console.WriteLine("== Marketable Item JSON Export ==");
ItemExports.GenerateMarketableItemJSON(items, categories, outputPath);
ItemExports.GenerateIdNameMappingJSON(items, outputPath);
ItemExports.GenerateNameIdMappingJSON(items, outputPath);
itemsearchcategories:
Console.WriteLine("== Item Search Category Export ==");
Console.Write("...");
ItemSearchCategoryExports.GenerateJSON(categories, categoriesDe, categoriesFr, categoriesJa, outputPath);
Console.WriteLine("Done!");
Console.WriteLine("== Chinese Item Search Category Mappings Export ==");
Console.Write("...");
ItemSearchCategoryExports.GenerateChineseMappingsJSON(http, outputPath);
Console.WriteLine("Done!");
town_export:
Console.WriteLine("== Town Export ==");
Console.Write("...");
TownExports.GenerateJSON(lumina, luminaDe, luminaFr, luminaJp, http, outputPath);
Console.WriteLine("Done!");
world_export:
Console.WriteLine("== World Export ==");
Console.Write("...");
WorldExports.GenerateJSON(lumina, outputPath);
Console.WriteLine("Done!");
end:
Console.WriteLine("All done!");
Console.ReadKey();
}