private static IEnumerable <CsvItem> GetChineseItems(Cyalume lumina, HttpClient http)
{
var rawData = http.GetStreamAsync(new Uri("https://raw.githubusercontent.com/thewakingsands/ffxiv-datamining-cn/master/Item.csv")).GetAwaiter().GetResult();
using var sr = new StreamReader(rawData);
using var csv = new CsvReader(sr, CultureInfo.InvariantCulture);
var iscSheet = lumina.GetExcelSheet <ItemSearchCategory>();
var items = new List <CsvItem>();
for (var i = 0; i < 3; i++)
{
csv.Read();
}
while (csv.Read())
{
var itemSearchCategory = csv.GetField <int>(17);
items.Add(new CsvItem
{
Key = csv.GetField <int>(0),
Name = csv.GetField <string>(10),
LevelItem = csv.GetField <int>(12),
Rarity = csv.GetField <int>(13),
ItemSearchCategory = iscSheet.First(isc => isc.RowId == itemSearchCategory),
});
}
return(items);
}
public static void Main(string[] args)
{
FFXIVNetworkMonitor monitor = new FFXIVNetworkMonitor();
var GameDir = Path.Combine(Process.GetProcessesByName("ffxiv_dx11")[0].MainModule.FileName, "..\\sqpack");
lumina = new Lumina.Lumina(GameDir);
monitor.MessageReceived = (string connection, long epoch, byte[] message) => MessageReceived(connection, epoch, message);
monitor.MessageSent = (string connection, long epoch, byte[] message) => MessageSent(connection, epoch, message);
monitor.Start();
// Run for 12000 seconds
System.Threading.Thread.Sleep(12000000);
monitor.Stop();
}
private static void DownloadIconAll(Cyalume lumina, string outputPath)
{
var dbEntries =
JsonConvert.DeserializeObject <Dictionary <int, string> >(
File.ReadAllText(Path.Combine(outputPath, "dbMapping.json")));
Parallel.For(0, dbEntries.Count, new ParallelOptions
{
MaxDegreeOfParallelism = 4
}, (index) => {
var dbEntry = dbEntries.ElementAt(index);
if (DownloadIcon(dbEntry.Key, dbEntry.Value, outputPath))
{
Console.WriteLine($" => DOWNLOADED: {dbEntry.Key}, {index}/{dbEntries.Count}");
}
});
}
static void Main(string[] args)
{
var mode = args[1];
var outputPath = Path.Combine("..", "..", "..", "..", "public", "i", "icon2x");
Directory.CreateDirectory(outputPath);
var luminaJp = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.Japanese
});
switch (mode)
{
case "alllist":
MakeListAll(luminaJp, outputPath);
return;
case "allicon":
DownloadIconAll(luminaJp, outputPath);
return;
}
}
public static void Main(string[] args)
{
var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
var umbraData = Path.Combine(appDataPath, "Umbra");
var dataDir = new DirectoryInfo(umbraData);
if (!dataDir.Exists)
{
dataDir.Create();
}
var logsDir = dataDir.CreateSubdirectory("logs");
// configure logging first
Log.Logger = new LoggerConfiguration()
.WriteTo.Async(x => x.Console())
.WriteTo.Async(x => x.File(
Path.Combine(logsDir.FullName, "umbra_.log"),
rollingInterval: RollingInterval.Day,
retainedFileCountLimit: 14
))
.MinimumLevel.Information()
.CreateLogger();
Locator.CurrentMutable.UseSerilogFullLogger();
// fix poco yells that rxui spits out, they're annoying and useless
Locator.CurrentMutable.RegisterConstant(new RxUI.CustomObservableForProperty(), typeof(ICreatesObservableForProperty));
var configFile = Path.Combine(dataDir.FullName, "config.json");
var settings = new ConfigurationBuilder <IUmbraSettings>()
.UseJsonFile(configFile)
.Build();
Locator.CurrentMutable.RegisterConstant(settings, typeof(IUmbraSettings));
if (settings.DataPath == null)
{
// attempt to guess a data path before asking someone to locate it
settings.DataPath = GuessClientDataPath();
// if we still haven't found something, make the user do it for us
if (settings.DataPath == null)
{
using (var dialog = new System.Windows.Forms.FolderBrowserDialog())
{
var result = dialog.ShowDialog();
}
}
}
Log.Information("using game datapath: {DataPath}", settings.DataPath);
// init lumina
var luminaOptions = new LuminaOptions
{
CacheFileResources = true,
DefaultExcelLanguage = settings.ExcelLanguage
};
var lumina = new Lumina.Lumina(
args.Length > 0 ? args[0] : settings.DataPath,
luminaOptions
);
foreach (var ver in settings.PreviousVersions)
{
Log.Information("previous version: {@PrevVersion}", ver);
}
Locator.CurrentMutable.RegisterConstant(luminaOptions);
Locator.CurrentMutable.RegisterConstant(lumina);
Locator.CurrentMutable.RegisterViewsForViewModels(Assembly.GetCallingAssembly());
// bootstrap app
var app = new App();
app.InitializeComponent();
app.Run();
}
public static void Main(string[] args)
{
var outputPath = Path.Combine("..", "..", "..", "..", "DataExports");
var categoryJsOutputPath = Path.Combine("..", "..", "..", "..", "public", "data");
var http = new HttpClient();
if (Directory.Exists(outputPath))
{
Directory.Delete(outputPath, true);
}
Directory.CreateDirectory(outputPath);
var lumina = new Cyalume(args[0]);
var luminaDe = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.German
});
var luminaFr = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.French
});
var luminaJp = new Cyalume(args[0], new LuminaOptions {
DefaultExcelLanguage = Language.Japanese
});
var items = lumina.GetExcelSheet <Item>().ToList();
var itemsDe = luminaDe.GetExcelSheet <Item>().ToList();
var itemsFr = luminaFr.GetExcelSheet <Item>().ToList();
var itemsJp = luminaJp.GetExcelSheet <Item>().ToList();
var itemsChs = GetChineseItems(lumina, http).ToList();
var categories = lumina.GetExcelSheet <ItemSearchCategory>().ToList();
Console.WriteLine($"Global items: {items.Count}");
Console.WriteLine($"CN items: {itemsChs.Count}");
Console.WriteLine("Starting game data export...");
categoryjs:
Console.WriteLine("== Category JS Export ==");
CategoryJsExports.Generate(lumina, luminaDe, luminaFr, luminaJp, categoryJsOutputPath);
CategoryJsExports.GenerateChinese(lumina, itemsChs, categoryJsOutputPath);
items:
Console.WriteLine("== Item Export ==");
ItemExports.GenerateItemJSON(items, itemsDe, itemsFr, itemsJp, itemsChs, categories, outputPath);
marketableitems:
Console.WriteLine("== Marketable Item JSON Export ==");
ItemExports.GenerateMarketableItemJSON(items, categories, outputPath);
itemsearchcategories:
Console.WriteLine("== Item Search Category Export ==");
Console.Write("...");
ItemSearchCategoryExports.GenerateJSON(lumina, outputPath);
Console.WriteLine("Done!");
Console.WriteLine("== Chinese Item Search Category Mappings Export ==");
Console.Write("...");
ItemSearchCategoryExports.GenerateChineseMappingsJSON(http, outputPath);
Console.WriteLine("Done!");
town_export:
Console.WriteLine("== Town Export ==");
Console.Write("...");
TownExports.GenerateJSON(lumina, luminaDe, luminaFr, luminaJp, http, outputPath);
Console.WriteLine("Done!");
world_export:
Console.WriteLine("== World Export ==");
Console.Write("...");
WorldExports.GenerateJSON(lumina, outputPath);
Console.WriteLine("Done!");
end:
Console.WriteLine("All done!");
Console.ReadKey();
}
private static void MakeListAll(Cyalume lumina, string outputPath)
{
Console.WriteLine("Starting game data export...");
var itemsJp = lumina.GetExcelSheet <Item>();
foreach (var category in lumina.GetExcelSheet <ItemSearchCategory>())
{
// We don't need those, not for sale
if (category.RowId == 0)
{
continue;
}
foreach (var item in itemsJp.Where(item => item.ItemSearchCategory.Value.RowId == category.RowId))
{
marketableDict.Add(item.Name, (int)item.RowId);
}
}
Console.WriteLine($"{marketableDict.Count} marketable items.");
var pages = GetPageCount();
for (var i = 1; i <= pages; i++)
{
// Can't access i in there cause it gets modified
var thisPage = i;
var searchPage = Retry.Do(() => Get(GetSearchUrl(thisPage)), TimeSpan.FromSeconds(20), 100);
var tableNode =
searchPage.DocumentNode.SelectSingleNode(
"/html/body/div[3]/div[2]/div[1]/div/div[2]/div[2]/div[5]/div/table/tbody");
var tableEntries = tableNode.SelectNodes("tr");
Console.WriteLine($"=> Page {i}");
foreach (var tableEntry in tableEntries)
{
var itemRow = tableEntry.ChildNodes[1];
var itemDivs = itemRow.ChildNodes.Where(x => x.Name == "div");
var item1 = itemDivs.ElementAt(1);
var item2 = item1.ChildNodes[3];
var itemUrl = item2.GetAttributeValue("href", string.Empty);
var itemName = item2.InnerHtml;
Console.WriteLine($" => {itemName}: {itemUrl}");
if (marketableDict.TryGetValue(itemName, out var key))
{
eorzeaDbDict.Add(key, itemUrl);
Console.WriteLine($" => MARKETABLE");
}
else
{
Console.WriteLine($" => NOT MARKETABLE");
}
}
File.WriteAllText(Path.Combine(outputPath, "dbMapping.json"), JsonConvert.SerializeObject(eorzeaDbDict, Formatting.Indented));
}
}
