public static void Ripp(Defines.ParserType type, uint entry, uint typeId, uint subTypeId, List<string> content) { if (!isInitialized) { Initialize(); isInitialized = true; } if (Rippers.ContainsKey((int)type)) Rippers[(int)type](entry, typeId, subTypeId, content); }
static void Main(String[] args) { Console.Clear(); Console.Title = "Wowhead Ripper"; DBC.LoadDBCs(); DB2.LoadDB2s(); List <String> files = new List <String>(); foreach (String fileName in args) { if (!File.Exists(fileName)) { Console.WriteLine("File {0} doesnt exist, skipping", fileName); continue; } if (File.ReadAllLines(fileName).Length == 0) { Console.WriteLine("File {0} is empty, skipping", fileName); continue; } files.Add(fileName); } foreach (String fileName in files) { Console.WriteLine("Loading file {0}...", fileName); var file = File.ReadAllText(fileName); Regex regex = new Regex(@"([0-9]+) +([0-9]+) +([0-9]+) *[,\r\n]+"); foreach (Match result in regex.Matches(file)) { UInt32 typeId = uint.Parse(result.Groups[1].ToString()); Int32 subTypeIdFlag = int.Parse(result.Groups[2].ToString()); UInt32 entry = uint.Parse(result.Groups[3].ToString()); if (typeId >= Defines.GetMaxTypeId()) { Console.WriteLine("Incorrect typeId for command \"{0} {1} {2}\"", typeId, subTypeIdFlag, entry); continue; } if (subTypeIdFlag >= (1 << (int)Defines.GetMaxTypeId())) { Console.WriteLine("Incorrect flags for command \"{0} {1} {2}\"", typeId, subTypeIdFlag, entry); continue; } if (!commandList.Keys.Contains(new KeyValuePair <uint, uint>(typeId, entry))) { commandList.Add(new KeyValuePair <uint, uint>(typeId, entry), subTypeIdFlag); } else { commandList[new KeyValuePair <uint, uint>(typeId, entry)] |= subTypeIdFlag; } } } Console.WriteLine("Got {0} records to parse.", commandList.Count); Console.WriteLine("Starting Parsing, please Stand by."); if (configFileName != "") { singleFileStream = new StreamWriter(configFileName, true); count = commandList.Count; singleFileStream.AutoFlush = true; } foreach (KeyValuePair <uint, uint> key in commandList.Keys) { new Thread(new ThreadStart(delegate { ParseData(key.Key, commandList[key], key.Value); })).Start(); if (usePreCached) { Thread.Sleep(20); } else { Thread.Sleep(700); // Needs to be done because Wowhead will think that you are a bot } } while (count != dataDone) { // Wait till all threads close } if (singleFileStream != null) { singleFileStream.Flush(); singleFileStream.Close(); } Console.Beep(); Console.WriteLine("Parsing done!"); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); }
public static void ParseData(UInt32 typeId, Int32 subTypeIdFlags, UInt32 entry) { List <String> content; List <int> ids = Defines.ExtractFlags(typeId, subTypeIdFlags); try { content = (usePreCached ? ReadFile(Defines.GenerateWowheadFileName(typeId, entry)) : ReadPage(Defines.GenerateWowheadUrl(typeId, entry))); } catch (Exception e) { dataDone += Defines.GetValidFlags(typeId, subTypeIdFlags); Console.WriteLine("{0}% - Error while parsing Entry {1} ({2})", Math.Round(dataDone / (float)commandList.Count * 100, 2), entry, e.Message); return; } foreach (uint subTypeId in ids) { Ripp(Defines.GetParserType(typeId, subTypeId), entry, typeId, subTypeId, content); } return; }
public static void LootParser(uint entry, uint typeId, uint subTypeId, List <String> content) { StringBuilder strBuilder = new StringBuilder(); strBuilder.AppendLine(String.Format("-- Parsing {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry)); strBuilder.AppendLine(String.Format("DELETE FROM `{0}` WHERE entry = {1};", Defines.GetDBName(typeId, subTypeId), entry)); WowheadSerializer serializer = new WowheadSerializer(content, typeId, subTypeId); foreach (Dictionary <String, Object> objectInto in serializer.Objects) { try { WowheadObject wowheadObject = new WowheadObject(objectInto); UInt32 lootId = wowheadObject.GetId(); String name = wowheadObject.GetFixedName(); UInt32 mincount = wowheadObject.GetStackArray()[0]; UInt32 maxcount = wowheadObject.GetStackArray()[1]; Double pct = (wowheadObject.GetCount() / (Double)serializer.TotalCount) * 100.0f; pct = Math.Round(pct, 3); String stringPct = pct.ToString().Replace(",", "."); // needs to be changed otherwise SQL errors Int32 lootmode = 0; // Fishing lootmode - always 1 if (typeId == 0 && subTypeId == 0) { lootmode = 1; } // - infront of quest items Int32 questId = wowheadObject.GetQuestId(); String str = String.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}' , '{7}'); -- {8}", Defines.GetDBName(typeId, subTypeId), entry, questId, stringPct, 1, lootmode, mincount, maxcount, name); strBuilder.AppendLine(str); } catch (Exception e) { Console.WriteLine(e.Message); } } // We need to parse aditional data // Gameobjects if (typeId == 1) { strBuilder.AppendLine(String.Format("UPDATE `gameobject_template` SET data1 = {0} WHERE entry = {0};", entry)); } // Item Contains // Disenchanging foreach (String s in content) { if (Defines.StringContains(s, "[tooltip=tooltip_reqenchanting]").Success) { if (typeId == 2 && subTypeId == 3) { strBuilder.AppendLine(String.Format("UPDATE `item_template` SET DisenchantID = '{0}', RequiredDisenchantSkill = '{1}' WHERE entry = '{0}';", entry, /*Defines.StringContains(line, "[tooltip=tooltip_reqenchanting]").Success ? */ uint.Parse(Defines.GetStringBetweenTwoOthers(s, "[tooltip=tooltip_reqenchanting]", "[/tooltip]")) /* : 1)*/)); } } } strBuilder.AppendLine(String.Format("-- Parsed {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry)); strBuilder.AppendLine(""); WriteSQL(typeId, entry, strBuilder.ToString()); Console.WriteLine("{0}% - Parsed {1} loot for entry {2}", Math.Round(++dataDone / (float)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry); }
public static void DroppedByParser(uint entry, uint typeId, uint subTypeId, List <String> content) { StringBuilder strBuilder = new StringBuilder(); strBuilder.AppendLine(String.Format("-- Parsing {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry)); strBuilder.AppendLine(String.Format("DELETE FROM `{0}` WHERE item = {1};", Defines.GetDBName(typeId, subTypeId), entry)); WowheadSerializer serializer = new WowheadSerializer(content, typeId, subTypeId); foreach (Dictionary <String, Object> objectInto in serializer.Objects) { try { WowheadObject wowheadObject = new WowheadObject(objectInto); UInt32 lootId = wowheadObject.GetId(); String name = wowheadObject.GetName(); Int32 maxcount = 1; // NYI Int32 mincount = 1; // NYI Double pct = (wowheadObject.GetCount() / (Double)wowheadObject.GetOutOf()) * 100.0f; pct = Math.Round(pct, 3); String stringPct = pct.ToString().Replace(",", "."); // needs to be changed otherwise SQL errors String str = String.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}' , '{7}'); -- {8}", Defines.GetDBName(typeId, subTypeId), lootId, entry, stringPct, 1, 0, mincount, maxcount, name); strBuilder.AppendLine(str); } catch (Exception e) { Console.WriteLine(e.Message); } } strBuilder.AppendLine(String.Format("-- Parsed {0} loot for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry)); strBuilder.AppendLine(""); WriteSQL(typeId, entry, strBuilder.ToString()); Console.WriteLine("{0}% - Parsed {1} data for entry {2}", Math.Round(++dataDone / (Double)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry); }
public RipperAttribute(Defines.ParserType type) { Type = type; }
public static void ParseVendor(UInt32 entry, UInt32 typeId, UInt32 subTypeId, List <String> content) { StringBuilder strBuilder = new StringBuilder(); strBuilder.AppendLine(String.Format("-- Parsing {0} vendor data for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry)); strBuilder.AppendLine(String.Format("DELETE FROM `{0}` WHERE entry = {1};", Defines.GetDBName(typeId, subTypeId), entry)); WowheadSerializer serializer = new WowheadSerializer(content, typeId, subTypeId); foreach (Dictionary <String, Object> objectInto in serializer.Objects) { try { WowheadObject wowheadObject = new WowheadObject(objectInto); UInt32 id = wowheadObject.GetId(); String name = wowheadObject.GetFixedName(); UInt32 extendedCost = ExtendedCosts.GetExtendedCost(wowheadObject.GetCurrencyCost(), wowheadObject.GetItemCost(), 0); // Needed (NULL cost ?) if (extendedCost == 2) { extendedCost = 0; } String str = String.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}'); -- {8}", Defines.GetDBName(typeId, subTypeId), entry, 0, id, 0, 0, extendedCost, 1, name); strBuilder.AppendLine(str); } catch (Exception e) { Console.WriteLine(e.Message); } } strBuilder.AppendLine(String.Format("-- Parsed {0} data for entry {1}", Defines.GetStreamName(typeId, subTypeId), entry)); strBuilder.AppendLine(""); WriteSQL(typeId, entry, strBuilder.ToString()); Console.WriteLine("{0}% - Parsed {1} data for entry {2}", Math.Round(++dataDone / (float)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry); }
public static void ParseData(UInt32 type, UInt32 entry) { UInt32 totalCount = 0; UInt32 count = 0; List <string> content; try { content = ReadPage(Defines.GenerateWowheadUrl(type, entry)); } catch (Exception e) { Console.WriteLine("Id {0} Doesn't exist ({1})", entry, e.Message); datad++; return; } foreach (string line in content) { Regex dataRegex = Defines.GetDataRegex(type); Regex totalCountRegex = Defines.GetTotalCountRegex(type); Match m = dataRegex.Match(line); Match m2 = totalCountRegex.Match(line); if (m2.Success) { string str = m2.Groups[0].Captures[0].Value; string[] numbers = Regex.Split(str, @"\D+"); totalCount = uint.Parse(numbers[2]); } if (!m.Success) { continue; } var json = new JavaScriptSerializer() { MaxJsonLength = int.MaxValue }; string data = m.Groups[1].Captures[0].Value; data = data.Replace("[,", "[0,"); // otherwise deserializer complains object[] m_object = (object[])json.DeserializeObject(data); AddToStream(string.Format("-- Parsing {0} loot for entry {1}", Defines.id_name[0], entry)); AddToStream(string.Format("DELETE FROM `{0}` WHERE entry = {1};", Defines.db_name[type], entry)); AddToStream(""); foreach (dict objectInto in m_object) { try { count++; int id = (int)objectInto["id"]; int maxcount = 1; int mincount = 1; float pct = 0.0f; string name = ""; int lootmode = 0; if (type == 0) { lootmode = 1; } if (objectInto.ContainsKey("name")) { name = (string)objectInto["name"]; } int m_count = (int)objectInto["count"]; int ArraySize = ((Array)objectInto["stack"]).GetLength(0); int[] stack = new int[ArraySize]; Array.Copy((Array)objectInto["stack"], stack, ArraySize); pct = (float)m_count / totalCount * 100.0f; maxcount = stack[1]; mincount = stack[0]; pct = (float)Math.Round(pct, 3); string strpct = pct.ToString(); strpct = strpct.Replace(",", "."); // needs to be changed otherwise SQL errors string str = string.Format("INSERT INTO `{0}` VALUES ( '{1}', '{2}', '{3}', '{4}', '{5}', '{6}' , '{7}'); -- {8}", Defines.db_name[type], entry, id, strpct, 1, lootmode, mincount, maxcount, name); AddToStream(str); } catch (Exception e) { Console.WriteLine(e.Message); } } if (count != 0) { AddToStream(string.Format("-- Parsed {0} loot for entry {1}", Defines.id_name[0], entry)); AddToStream(""); } Console.WriteLine("Parsed {0} loot for entry {1}", Defines.id_name[0], entry); datad++; return; } datad++; return; }
public static void ParseOutdatedQuests(uint entry, uint typeId, uint subTypeId, List <string> content) { StringBuilder strBuilder = new StringBuilder(); foreach (string line in content) { Match match = Defines.GetDataRegex(typeId, subTypeId).Match(line); if (match.Success) { strBuilder.AppendLine(string.Format("UPDATE `{0}` SET minLvl = -1, maxLvl = -1 WHERE entry = {1};", Defines.GetDBName(typeId, subTypeId), entry)); } } WriteSQL(typeId, entry, strBuilder.ToString()); Console.WriteLine("{0}% - Parsed {1} data for entry {2}", Math.Round(++dataDone / (float)commandList.Count * 100, 2), Defines.GetStreamName(typeId, subTypeId), entry); }