public static void Ripp(Defines.ParserType type, uint entry, uint typeId, uint subTypeId, List<string> content)
{
if (!isInitialized)
{
Initialize();
isInitialized = true;
}
if (Rippers.ContainsKey((int)type))
Rippers[(int)type](entry, typeId, subTypeId, content);
}
static void Main(String[] args)
{
Console.Clear();
Console.Title = "Wowhead Ripper";
DBC.LoadDBCs();
DB2.LoadDB2s();
List <String> files = new List <String>();
foreach (String fileName in args)
{
if (!File.Exists(fileName))
{
Console.WriteLine("File {0} doesnt exist, skipping", fileName);
continue;
}
if (File.ReadAllLines(fileName).Length == 0)
{
Console.WriteLine("File {0} is empty, skipping", fileName);
continue;
}
files.Add(fileName);
}
foreach (String fileName in files)
{
Console.WriteLine("Loading file {0}...", fileName);
var file = File.ReadAllText(fileName);
Regex regex = new Regex(@"([0-9]+) +([0-9]+) +([0-9]+) *[,\r\n]+");
foreach (Match result in regex.Matches(file))
{
UInt32 typeId = uint.Parse(result.Groups[1].ToString());
Int32 subTypeIdFlag = int.Parse(result.Groups[2].ToString());
UInt32 entry = uint.Parse(result.Groups[3].ToString());
if (typeId >= Defines.GetMaxTypeId())
{
Console.WriteLine("Incorrect typeId for command \"{0} {1} {2}\"", typeId, subTypeIdFlag, entry);
continue;
}
if (subTypeIdFlag >= (1 << (int)Defines.GetMaxTypeId()))
{
Console.WriteLine("Incorrect flags for command \"{0} {1} {2}\"", typeId, subTypeIdFlag, entry);
continue;
}
if (!commandList.Keys.Contains(new KeyValuePair <uint, uint>(typeId, entry)))
{
commandList.Add(new KeyValuePair <uint, uint>(typeId, entry), subTypeIdFlag);
}
else
{
commandList[new KeyValuePair <uint, uint>(typeId, entry)] |= subTypeIdFlag;
}
}
}
Console.WriteLine("Got {0} records to parse.", commandList.Count);
Console.WriteLine("Starting Parsing, please Stand by.");
if (configFileName != "")
{
singleFileStream = new StreamWriter(configFileName, true);
count = commandList.Count;
singleFileStream.AutoFlush = true;
}
foreach (KeyValuePair <uint, uint> key in commandList.Keys)
{
new Thread(new ThreadStart(delegate { ParseData(key.Key, commandList[key], key.Value); })).Start();
if (usePreCached)
{
Thread.Sleep(20);
}
else
{
Thread.Sleep(700); // Needs to be done because Wowhead will think that you are a bot
}
}
while (count != dataDone)
{
// Wait till all threads close
}
if (singleFileStream != null)
{
singleFileStream.Flush();
singleFileStream.Close();
}
Console.Beep();
Console.WriteLine("Parsing done!");
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
public static void ParseData(UInt32 typeId, Int32 subTypeIdFlags, UInt32 entry)
{
List <String> content;
List <int> ids = Defines.ExtractFlags(typeId, subTypeIdFlags);
try
{
content = (usePreCached ? ReadFile(Defines.GenerateWowheadFileName(typeId, entry)) : ReadPage(Defines.GenerateWowheadUrl(typeId, entry)));
}
catch (Exception e)
{
dataDone += Defines.GetValidFlags(typeId, subTypeIdFlags);
Console.WriteLine("{0}% - Error while parsing Entry {1} ({2})", Math.Round(dataDone / (float)commandList.Count * 100, 2), entry, e.Message);
return;
}
foreach (uint subTypeId in ids)
{
Ripp(Defines.GetParserType(typeId, subTypeId), entry, typeId, subTypeId, content);
}
return;
}
public static void LootParser(uint entry, uint typeId, uint subTypeId, List <String> content)
{
StringBuilder strBuilder = new StringBuilder();
strBuilder.AppendLine(String.Format("-- Parsing {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry));
strBuilder.AppendLine(String.Format("DELETE FROM `{0}` WHERE entry = {1};", Defines.GetDBName(typeId, subTypeId), entry));
WowheadSerializer serializer = new WowheadSerializer(content, typeId, subTypeId);
foreach (Dictionary <String, Object> objectInto in serializer.Objects)
{
try
{
WowheadObject wowheadObject = new WowheadObject(objectInto);
UInt32 lootId = wowheadObject.GetId();
String name = wowheadObject.GetFixedName();
UInt32 mincount = wowheadObject.GetStackArray()[0];
UInt32 maxcount = wowheadObject.GetStackArray()[1];
Double pct = (wowheadObject.GetCount() / (Double)serializer.TotalCount) * 100.0f;
pct = Math.Round(pct, 3);
String stringPct = pct.ToString().Replace(",", "."); // needs to be changed otherwise SQL errors
Int32 lootmode = 0;
// Fishing lootmode - always 1
if (typeId == 0 && subTypeId == 0)
{
lootmode = 1;
}
// - infront of quest items
Int32 questId = wowheadObject.GetQuestId();
String str = String.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}' , '{7}'); -- {8}",
Defines.GetDBName(typeId, subTypeId), entry, questId, stringPct, 1, lootmode, mincount, maxcount, name);
strBuilder.AppendLine(str);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
// We need to parse aditional data
// Gameobjects
if (typeId == 1)
{
strBuilder.AppendLine(String.Format("UPDATE `gameobject_template` SET data1 = {0} WHERE entry = {0};", entry));
}
// Item Contains
// Disenchanging
foreach (String s in content)
{
if (Defines.StringContains(s, "[tooltip=tooltip_reqenchanting]").Success)
{
if (typeId == 2 && subTypeId == 3)
{
strBuilder.AppendLine(String.Format("UPDATE `item_template` SET DisenchantID = '{0}', RequiredDisenchantSkill = '{1}' WHERE entry = '{0}';", entry, /*Defines.StringContains(line, "[tooltip=tooltip_reqenchanting]").Success ? */ uint.Parse(Defines.GetStringBetweenTwoOthers(s, "[tooltip=tooltip_reqenchanting]", "[/tooltip]")) /* : 1)*/));
}
}
}
strBuilder.AppendLine(String.Format("-- Parsed {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry));
strBuilder.AppendLine("");
WriteSQL(typeId, entry, strBuilder.ToString());
Console.WriteLine("{0}% - Parsed {1} loot for entry {2}", Math.Round(++dataDone / (float)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry);
}
public static void DroppedByParser(uint entry, uint typeId, uint subTypeId, List <String> content)
{
StringBuilder strBuilder = new StringBuilder();
strBuilder.AppendLine(String.Format("-- Parsing {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry));
strBuilder.AppendLine(String.Format("DELETE FROM `{0}` WHERE item = {1};", Defines.GetDBName(typeId, subTypeId), entry));
WowheadSerializer serializer = new WowheadSerializer(content, typeId, subTypeId);
foreach (Dictionary <String, Object> objectInto in serializer.Objects)
{
try
{
WowheadObject wowheadObject = new WowheadObject(objectInto);
UInt32 lootId = wowheadObject.GetId();
String name = wowheadObject.GetName();
Int32 maxcount = 1; // NYI
Int32 mincount = 1; // NYI
Double pct = (wowheadObject.GetCount() / (Double)wowheadObject.GetOutOf()) * 100.0f;
pct = Math.Round(pct, 3);
String stringPct = pct.ToString().Replace(",", "."); // needs to be changed otherwise SQL errors
String str = String.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}' , '{7}'); -- {8}",
Defines.GetDBName(typeId, subTypeId), lootId, entry, stringPct, 1, 0, mincount, maxcount, name);
strBuilder.AppendLine(str);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
strBuilder.AppendLine(String.Format("-- Parsed {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry));
strBuilder.AppendLine("");
WriteSQL(typeId, entry, strBuilder.ToString());
Console.WriteLine("{0}% - Parsed {1} data for entry {2}", Math.Round(++dataDone / (Double)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry);
}
public RipperAttribute(Defines.ParserType type)
{
Type = type;
}
public static void ParseVendor(UInt32 entry, UInt32 typeId, UInt32 subTypeId, List <String> content)
{
StringBuilder strBuilder = new StringBuilder();
strBuilder.AppendLine(String.Format("-- Parsing {0} vendor data for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry));
strBuilder.AppendLine(String.Format("DELETE FROM `{0}` WHERE entry = {1};", Defines.GetDBName(typeId, subTypeId), entry));
WowheadSerializer serializer = new WowheadSerializer(content, typeId, subTypeId);
foreach (Dictionary <String, Object> objectInto in serializer.Objects)
{
try
{
WowheadObject wowheadObject = new WowheadObject(objectInto);
UInt32 id = wowheadObject.GetId();
String name = wowheadObject.GetFixedName();
UInt32 extendedCost = ExtendedCosts.GetExtendedCost(wowheadObject.GetCurrencyCost(), wowheadObject.GetItemCost(), 0);
// Needed (NULL cost ?)
if (extendedCost == 2)
{
extendedCost = 0;
}
String str = String.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}'); -- {8}",
Defines.GetDBName(typeId, subTypeId), entry, 0, id, 0, 0, extendedCost, 1, name);
strBuilder.AppendLine(str);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
strBuilder.AppendLine(String.Format("-- Parsed {0} data for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry));
strBuilder.AppendLine("");
WriteSQL(typeId, entry, strBuilder.ToString());
Console.WriteLine("{0}% - Parsed {1} data for entry {2}", Math.Round(++dataDone / (float)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry);
}
public static void ParseData(UInt32 type, UInt32 entry)
{
UInt32 totalCount = 0;
UInt32 count = 0;
List <string> content;
try
{
content = ReadPage(Defines.GenerateWowheadUrl(type, entry));
}
catch (Exception e)
{
Console.WriteLine("Id {0} Doesn't exist ({1})", entry, e.Message);
datad++;
return;
}
foreach (string line in content)
{
Regex dataRegex = Defines.GetDataRegex(type);
Regex totalCountRegex = Defines.GetTotalCountRegex(type);
Match m = dataRegex.Match(line);
Match m2 = totalCountRegex.Match(line);
if (m2.Success)
{
string str = m2.Groups[0].Captures[0].Value;
string[] numbers = Regex.Split(str, @"\D+");
totalCount = uint.Parse(numbers[2]);
}
if (!m.Success)
{
continue;
}
var json = new JavaScriptSerializer()
{
MaxJsonLength = int.MaxValue
};
string data = m.Groups[1].Captures[0].Value;
data = data.Replace("[,", "[0,"); // otherwise deserializer complains
object[] m_object = (object[])json.DeserializeObject(data);
AddToStream(string.Format("-- Parsing {0} loot for entry {1}", Defines.id_name[0], entry));
AddToStream(string.Format("DELETE FROM `{0}` WHERE entry = {1};", Defines.db_name[type], entry));
AddToStream("");
foreach (dict objectInto in m_object)
{
try
{
count++;
int id = (int)objectInto["id"];
int maxcount = 1;
int mincount = 1;
float pct = 0.0f;
string name = "";
int lootmode = 0;
if (type == 0)
{
lootmode = 1;
}
if (objectInto.ContainsKey("name"))
{
name = (string)objectInto["name"];
}
int m_count = (int)objectInto["count"];
int ArraySize = ((Array)objectInto["stack"]).GetLength(0);
int[] stack = new int[ArraySize];
Array.Copy((Array)objectInto["stack"], stack, ArraySize);
pct = (float)m_count / totalCount * 100.0f;
maxcount = stack[1];
mincount = stack[0];
pct = (float)Math.Round(pct, 3);
string strpct = pct.ToString();
strpct = strpct.Replace(",", "."); // needs to be changed otherwise SQL errors
string str = string.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}' , '{7}'); -- {8}",
Defines.db_name[type], entry, id, strpct, 1, lootmode, mincount, maxcount, name);
AddToStream(str);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
if (count != 0)
{
AddToStream(string.Format("-- Parsed {0} loot for entry {1}", Defines.id_name[0], entry));
AddToStream("");
}
Console.WriteLine("Parsed {0} loot for entry {1}", Defines.id_name[0], entry);
datad++;
return;
}
datad++;
return;
}
public static void ParseOutdatedQuests(uint entry, uint typeId, uint subTypeId, List <string> content)
{
StringBuilder strBuilder = new StringBuilder();
foreach (string line in content)
{
Match match = Defines.GetDataRegex(typeId, subTypeId).Match(line);
if (match.Success)
{
strBuilder.AppendLine(string.Format("UPDATE `{0}` SET minLvl = -1, maxLvl = -1 WHERE entry = {1};", Defines.GetDBName(typeId, subTypeId), entry));
}
}
WriteSQL(typeId, entry, strBuilder.ToString());
Console.WriteLine("{0}% - Parsed {1} data for entry {2}", Math.Round(++dataDone / (float)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry);
}
