public void Write()
{
ItemIconDatabase.WriteUpdates();
IconDatabase.WriteUpdates();
foreach (var lang in _languagesCodes)
{
CreatePartials(lang);
WriteCore(lang);
WriteEquipmentCalculators(lang);
WriteItems(lang);
WriteQuests(lang);
WriteLeves(lang);
WriteNpcs(lang);
WriteActions(lang);
WriteFish(lang);
WriteNodes(lang);
WriteAchievements(lang);
WriteInstances(lang);
WriteFates(lang);
WriteMobs(lang);
WriteBrowsers(lang);
PatchDatabase.WritePatchLists(this, _update, lang);
}
PatchDatabase.WriteMasterPatchList();
}
void WriteIcon(UInt16 iconId, string hash)
{
var url = _baseItemIconUrl + hash;
var bytes = RequestBytes(url);
ItemIconDatabase.WriteIcon(iconId, bytes);
}
void WriteIcon(ItemIconData data, string hash)
{
var url = _baseItemIconUrl + hash;
var bytes = RequestBytes(url);
ItemIconDatabase.WriteNewIcon(data, bytes);
}
public void FetchIcons()
{
// Start at a random place in the set.
var start = (new Random()).Next(ItemIconDatabase.ItemsNeedingIcons.Count);
var itemsToFetch = new List <Saint.Item>(ItemIconDatabase.ItemsNeedingIcons.Skip(start));
itemsToFetch.AddRange(ItemIconDatabase.ItemsNeedingIcons.Take(start));
foreach (var sItem in itemsToFetch)
{
// A prior item may share this icon, so always check if it was written.
var iconId = (UInt16)sItem.GetRaw("Icon");
if (ItemIconDatabase.HasIcon(iconId))
{
continue;
}
// Scrape search data from Lodestone.
var itemUrl = SearchItem(iconId, sItem.Name);
string hash = null;
if (itemUrl != null)
{
hash = FetchItem(iconId, sItem.Name, itemUrl);
}
if (hash == null)
{
// Can't find this entry. Move on.
continue;
}
// Fetch the icon and write entries.
WriteIcon(iconId, hash);
}
}
public void FetchIcons()
{
// Start at a random place in the set.
var start = (new Random()).Next(ItemIconDatabase.ItemsNeedingIcons.Count);
var itemsToFetch = new List <Saint.Item>(ItemIconDatabase.ItemsNeedingIcons.Skip(start));
itemsToFetch.AddRange(ItemIconDatabase.ItemsNeedingIcons.Take(start));
var count = 0;
var options = new ParallelOptions()
{
MaxDegreeOfParallelism = 8
};
Parallel.ForEach(itemsToFetch, options, sItem =>
{
var num = Interlocked.Increment(ref count);
// A prior item may share this icon, so always check if it was written.
var iconId = (UInt16)sItem.GetRaw("Icon");
if (ItemIconDatabase.HasIcon(iconId))
{
return;
}
var progress = $"{num}/{itemsToFetch.Count}, {100*num/itemsToFetch.Count}%";
// Scrape search data from Lodestone.
var itemUrl = SearchItem(iconId, sItem.Name, progress);
string hash = null;
if (itemUrl != null)
{
hash = FetchItem(iconId, sItem.Name, itemUrl, progress);
}
if (hash == null)
{
// Can't find this entry. Move on.
return;
}
// Fetch the icon and write entries.
WriteIcon(iconId, hash);
});
}
public void FetchIcons()
{
// Run the process with an initial queue of items.
var items = new Stack <Item>(ItemIconDatabase.ItemsNeedingIcons);
while (items.Count > 0)
{
var item = items.Pop();
if (!ItemIconDatabase.IconDataByItemId.TryGetValue(item.Key, out var data))
{
data = new ItemIconData();
data.ItemId = item.Key;
data.RawIconKey = (UInt16)item.GetRaw("Icon");
data.Name = item.Name.ToString();
ItemIconDatabase.IconDataByItemId[item.Key] = data;
}
// Scrape search data from Lodestone.
var itemUrl = SearchItem(item.Key, item.Name, data);
string hash = null;
if (itemUrl != null)
{
hash = FetchItem(item.Key, item.Name, itemUrl, data);
}
if (hash == null)
{
// This entry failed. Enqueue an alternate to search for.
var alternate = ItemIconDatabase.DequeueAlternate(item.Icon);
if (alternate != null)
{
items.Push(alternate);
}
continue;
}
// Fetch the icon and write entries.
WriteIcon(data, hash);
WriteEntries();
}
}
void BuildItems()
{
var libraIndex = _builder.Libra.Table <Libra.Item>()
.ToArray()
.ToDictionary(i => i.Key);
foreach (var sItem in _builder.ItemsToImport)
{
var item = _builder.CreateItem(sItem.Key);
_builder.Localize.Strings(item, sItem, "Name");
_builder.Localize.HtmlStrings(item, sItem, "Description");
_builder.Db.ItemsByName[(string)item.en.name] = item;
item.patch = PatchDatabase.Get("item", sItem.Key);
item.patchCategory = PatchDatabase.GetPatchCategory(sItem);
item.price = sItem.Ask;
item.ilvl = sItem.ItemLevel.Key;
item.category = sItem.ItemUICategory.Key;
if (sItem.IsUnique)
{
item.unique = 1;
}
if (sItem.IsDyeable)
{
item.dyeable = 1;
}
if (!sItem.IsUntradable)
{
item.tradeable = 1;
}
if (sItem.Bid > 0)
{
item.sell_price = sItem.Bid;
}
var rarity = sItem.As <byte>("Rarity");
if (rarity > 0)
{
item.rarity = rarity;
}
if (sItem.IsConvertable)
{
item.convertable = 1;
}
if (sItem.IsAetherialReducible)
{
item.reducible = 1;
}
if (sItem.ItemSearchCategory.Key == 0)
{
item.unlistable = 1;
}
if (sItem.IsCollectable)
{
item.collectable = 1;
}
// Mark applicable materia for advanced melding.
if (sItem.ItemUICategory.Key == 58 && !sItem.IsAdvancedMeldingPermitted)
{
item.advancedMeldingForbidden = 1;
}
item.stackSize = sItem.StackSize;
if (sItem.RepairClassJob.Key != 0)
{
item.repair = sItem.RepairClassJob.Key;
}
BuildAttributes(item, sItem);
item.icon = ItemIconDatabase.EnsureIcon(sItem);
// Additional data
var additionalData = sItem.AdditionalData;
if (additionalData != null)
{
if (additionalData.Sheet.Name == "GardeningSeed")
{
_gardeningSeeds.Add(sItem);
}
}
#region Libra Supplement
libraIndex.TryGetValue(sItem.Key, out var lItem);
if (lItem != null && lItem.data != null)
{
dynamic extraLibraData = JsonConvert.DeserializeObject(lItem.data);
// Mob drops
if (extraLibraData.bnpc != null && extraLibraData.bnpc.Count > 0)
{
var mobIds = new JArray();
foreach (long mob in extraLibraData.bnpc)
{
mobIds.Add(mob);
if (!_builder.ItemDropsByMobId.TryGetValue(mob, out var itemIds))
{
itemIds = new List <int>();
_builder.ItemDropsByMobId[mob] = itemIds;
}
itemIds.Add(sItem.Key);
}
// References are added by Mobs module.
item.drops = mobIds;
}
// Instances
if (extraLibraData.instance_content != null)
{
foreach (int instanceId in extraLibraData.instance_content)
{
if (!_builder.Db.ItemsByInstanceId.TryGetValue(instanceId, out var instanceItems))
{
_builder.Db.ItemsByInstanceId[instanceId] = instanceItems = new List <dynamic>();
}
instanceItems.Add(item);
if (item.instances == null)
{
item.instances = new JArray();
}
JArray itemInstances = item.instances;
if (!itemInstances.Contains(instanceId))
{
itemInstances.Add(instanceId);
_builder.Db.AddReference(item, "instance", instanceId, true);
}
}
}
}
#endregion
//todo: item.repair_price = ? Not important
// Mark embedded categories
if (sItem.ItemUICategory.Key == 59)
{
// Crystals
_builder.Db.EmbeddedIngredientItems.Add(item);
_builder.Db.EmbeddedPartialItemIds.Add(sItem.Key);
}
if (sItem.ItemUICategory.Key == 60 && sItem.ItemSearchCategory.Key == 59)
{
_builder.Db.EmbeddedPartialItemIds.Add(sItem.Key); // Catalysts with a matching search category.
}
}
// Needs another pass once all items are processed.
BuildAdditionalItemData();
}
public LodestoneIconScraper(ItemIconDatabase itemIconDatabase)
{
_itemIconDatabase = itemIconDatabase;
}