public static void Main(string[] args)
{
string projectName;
var process = FindSuitableProcess(out projectName);
if (process == null)
{
return;
}
var outputPath = Path.Combine("schemas", projectName);
Directory.CreateDirectory(outputPath);
using (var memory = new ProcessMemory())
{
if (memory.Open(process) == false)
{
return;
}
ExportExpressionFunctions(memory, outputPath);
var enums = ExportEnums(memory, outputPath);
ExportSchemas(memory, outputPath, enums);
}
}
private static uint HashKeyValueList(ProcessMemory memory, uint baseAddress, uint hash)
{
var entries = new List <KeyValuePair <string, string> >();
var listAddress = memory.ReadU32(baseAddress);
var count = memory.ReadS32(listAddress + 8);
if (count > 0)
{
var entriesAddress = memory.ReadU32(listAddress + 4);
var data = memory.ReadAllBytes(entriesAddress, count * 8);
for (int i = 0; i < count; i++)
{
var name = memory.ReadStringZ(BitConverter.ToUInt32(data, (i * 8) + 0));
var value = memory.ReadStringZ(BitConverter.ToUInt32(data, (i * 8) + 4));
entries.Add(new KeyValuePair <string, string>(name, value));
}
}
var sb = new StringBuilder();
foreach (var entry in entries.OrderBy(e => e.Key.ToLowerInvariant()))
{
sb.Append(entry.Key);
sb.Append(entry.Value);
}
return(Adler32.Hash(sb.ToString(), hash));
}
private static string TranslateArgumentType(
Natives.ExpressionArgument arg,
ProcessMemory memory)
{
switch (arg.Type)
{
case 0x0000:
return("void");
case 0x0002:
return("int");
case 0x0004:
return("flt");
case 0x0005:
return("intarray");
case 0x0006:
return("floatarray");
case 0x0007:
return("Vec3");
case 0x0008:
return("Vec4");
case 0x0009:
return("Mat4");
case 0x000A:
return("Quat");
case 0x000B:
return("str");
case 0x000C:
return("multivalarray");
case 0x0080:
return("entityarray");
case 0x0081:
return(memory.ReadStringZ(arg.TypeNamePointer));
case 0x0082:
return("MultiVal");
case 0x2809:
return("loc");
}
return(string.Format("*INV:{0:X8}*", arg.Type));
}
private static bool Locate(ProcessMemory memory, out uint result, params LocateDelegate[] locators)
{
for (int i = 0, j = locators.Length - 1; i < locators.Length; i++, j--)
{
if (locators[j](memory, out result) == true)
{
return(true);
}
}
result = 0;
return(false);
}
private static void ExportExpressionFunction(
string itemName,
uint pointer,
ProcessMemory memory,
TextWriter writer)
{
var sb = new StringBuilder();
var func = memory.ReadStructure <Natives.ExpressionFunction>(pointer);
sb.AppendFormat("[{0:X8}] ", func.Handler);
var funcName = memory.ReadStringZ(func.NamePointer);
var codeName = memory.ReadStringZ(func.ExprCodeNamePointer);
var tags = new string[12];
for (int i = 0; i < 12; i++)
{
tags[i] = memory.ReadStringZ(func.Tags[i]);
}
var validTags = tags
.Where(t => string.IsNullOrWhiteSpace(t) == false)
.ToArray();
if (validTags.Length > 0)
{
sb.AppendFormat("{0} : ",
string.Join(", ", validTags));
}
sb.AppendFormat("{0} {1}(",
TranslateArgumentType(func.ReturnType, memory),
funcName);
for (int i = 0; i < func.ArgumentCount; i++)
{
if (i > 0)
{
sb.Append(", ");
}
sb.AppendFormat("{0} {1}",
TranslateArgumentType(func.Arguments[i], memory),
memory.ReadStringZ(func.Arguments[i].NamePointer));
}
sb.Append(")");
writer.WriteLine(sb.ToString());
}
private static void ExportExpressionFunctions(ProcessMemory memory, string outputPath)
{
// expression functions
var exprFuncs = new List <KeyValuePair <string, uint> >();
uint stashPointer;
if (Locate(memory,
out stashPointer,
Locators.ExpressionFunctionTableLocator1.Locate,
Locators.ExpressionFunctionTableLocator2.Locate) == false)
{
Console.WriteLine("Warning: failed to locate expression function table.");
return;
}
var stashCount = memory.ReadS32(stashPointer + 0x08);
var stashEntryPointer = memory.ReadU32(stashPointer + 0x14);
var stashEntries = memory.ReadAllBytes(stashEntryPointer, stashCount * 8);
for (int i = 0; i < stashCount; i++)
{
var namePointer = BitConverter.ToUInt32(stashEntries, (i * 8) + 0);
var dataPointer = BitConverter.ToUInt32(stashEntries, (i * 8) + 4);
if (namePointer == 0 &&
dataPointer == 0)
{
continue;
}
var name = memory.ReadStringZ(namePointer);
exprFuncs.Add(new KeyValuePair <string, uint>(name, dataPointer));
}
if (exprFuncs.Count > 0)
{
using (var output = File.Create(Path.Combine(outputPath, "expression functions.txt")))
{
var writer = new StreamWriter(output);
foreach (var kv in exprFuncs.OrderBy(kv => kv.Key))
{
ExportExpressionFunction(kv.Key, kv.Value, memory, writer);
}
writer.Flush();
}
}
}
private static string GetEnumType(ProcessMemory memory, uint baseAddress)
{
switch (memory.ReadU32(baseAddress))
{
case 1:
{
return("int");
}
case 2:
{
return("string");
}
}
throw new NotSupportedException();
}
private static void ExportEnum(
ProcessMemory memory,
uint address,
IEnumerable <KeyValuePair <string, string> > elements,
XmlWriter xml)
{
xml.WriteStartElement("elements");
xml.WriteAttributeString("type", GetEnumType(memory, address));
foreach (var kv in elements)
{
xml.WriteStartElement("element");
xml.WriteAttributeString("name", kv.Key);
xml.WriteValue(kv.Value);
xml.WriteEndElement();
}
xml.WriteEndElement();
}
private static List <KeyValuePair <string, string> > ReadKeyValueList(ProcessMemory memory, uint baseAddress)
{
var entries = new List <KeyValuePair <string, string> >();
var listAddress = memory.ReadU32(baseAddress);
if (listAddress != 0)
{
var count = memory.ReadS32(listAddress + 8);
if (count > 0)
{
var entriesAddress = memory.ReadU32(listAddress + 4);
var data = memory.ReadAllBytes(entriesAddress, count * 8);
for (int i = 0; i < count; i++)
{
var name = memory.ReadStringZ(BitConverter.ToUInt32(data, (i * 8) + 0));
var value = memory.ReadStringZ(BitConverter.ToUInt32(data, (i * 8) + 4));
entries.Add(new KeyValuePair <string, string>(name, value));
}
}
}
return(entries);
}
public static uint HashTable(ProcessMemory memory, uint address, uint hash)
{
var columns = new List <KeyValuePair <Natives.ParserColumn, string> >();
var currentAddress = address;
while (true)
{
var column = memory.ReadStructure <Natives.ParserColumn>(currentAddress);
currentAddress += 40;
var name = memory.ReadStringZ(column.NamePointer);
if (column.Type == 0)
{
if (string.IsNullOrEmpty(name) == true)
{
break;
}
}
columns.Add(new KeyValuePair <Natives.ParserColumn, string>(column, name));
}
foreach (var kv in columns)
{
var column = kv.Key;
if (column.Flags.HasAnyOptions(Parser.ColumnFlags.REDUNDANTNAME |
Parser.ColumnFlags.UNOWNED) == true)
{
continue;
}
var name = kv.Value;
if (string.IsNullOrEmpty(name) == false)
{
hash = Adler32.Hash(name, hash);
}
hash = Adler32.Hash(column.Type, hash);
var token = Parser.GlobalTokens.GetToken(column.Token);
switch (token.GetParameter(column.Flags, 0))
{
case Parser.ColumnParameter.NumberOfElements:
case Parser.ColumnParameter.Default:
case Parser.ColumnParameter.StringLength:
case Parser.ColumnParameter.Size:
{
hash = Adler32.Hash(column.Parameter0, hash);
break;
}
case Parser.ColumnParameter.BitOffset:
{
hash = Adler32.Hash(column.Parameter0 >> 16, hash);
break;
}
case Parser.ColumnParameter.DefaultString:
case Parser.ColumnParameter.CommandString:
{
if (column.Parameter0 != 0)
{
hash = Adler32.Hash(memory.ReadStringZ(column.Parameter0), hash);
}
break;
}
}
var param1 = token.GetParameter(column.Flags, 1);
if (column.Parameter1 != 0 &&
(column.Token == 20 || column.Token == 21) &&
address != column.Parameter1 &&
column.Flags.HasAnyOptions(Parser.ColumnFlags.STRUCT_NORECURSE) == false)
{
hash = HashTable(memory, column.Parameter1, hash);
}
if (column.Parameter1 != 0 &&
param1 == Parser.ColumnParameter.StaticDefineList)
{
hash = HashStaticDefineList(memory, column.Parameter1, hash);
}
if (column.Token == 23)
{
var formatString = memory.ReadStringZ(column.FormatStringPointer);
if (string.IsNullOrEmpty(formatString) == false)
{
hash = Adler32.Hash(formatString, hash);
}
}
}
return(hash);
}
public static uint HashStaticDefineList(ProcessMemory memory, uint baseAddress, uint hash)
{
var valueType = 4;
while (true)
{
var type = memory.ReadU32(baseAddress);
if (type == 0)
{
break;
}
switch (type)
{
case 1:
{
valueType = 1;
baseAddress += 8;
break;
}
case 2:
{
valueType = 2;
baseAddress += 8;
break;
}
case 3:
{
var listAddress = memory.ReadU32(baseAddress + 4);
baseAddress += 8;
if (listAddress != 0)
{
hash = HashKeyValueList(memory, listAddress, hash);
}
break;
}
case 5:
{
var parent = memory.ReadU32(baseAddress + 4);
return(HashStaticDefineList(memory, parent, hash));
}
default:
{
var name = memory.ReadStringZ(type);
hash = Adler32.Hash(name, hash);
switch (valueType)
{
case 1:
{
var value = memory.ReadU32(baseAddress + 4);
hash = Adler32.Hash(value, hash);
baseAddress += 8;
break;
}
case 2:
{
var value = memory.ReadStringZ(baseAddress + 4);
hash = Adler32.Hash(value, hash);
baseAddress += 8;
break;
}
default:
{
throw new NotImplementedException();
}
}
break;
}
}
}
return(hash);
}
private static void ExportSchemas(ProcessMemory memory,
string outputPath,
List <KeyValuePair <string, uint> > enums)
{
uint stashPointer;
if (Locate(memory,
out stashPointer,
Locators.ParserTableLocator1.Locate,
Locators.ParserTableLocator2.Locate) == false)
{
Console.WriteLine("Warning: failed to locate schema table.");
return;
}
var parsers = new List <KeyValuePair <string, uint> >();
var stashCount = memory.ReadS32(stashPointer + 0x08);
var stashEntryPointer = memory.ReadU32(stashPointer + 0x14);
var stashEntries = memory.ReadAllBytes(stashEntryPointer, stashCount * 8);
for (int i = 0; i < stashCount; i++)
{
var namePointer = BitConverter.ToUInt32(stashEntries, (i * 8) + 0);
var dataPointer = BitConverter.ToUInt32(stashEntries, (i * 8) + 4);
if (namePointer == 0 &&
dataPointer == 0)
{
continue;
}
var name = memory.ReadStringZ(namePointer);
parsers.Add(new KeyValuePair <string, uint>(name, dataPointer));
}
if (parsers.Count > 0)
{
Directory.CreateDirectory(Path.Combine(outputPath, "schemas"));
foreach (var kv in parsers.OrderBy(kv => kv.Key))
{
var name = kv.Key;
var pointer = kv.Value;
Console.WriteLine("[schema] {0}", name);
var settings = new XmlWriterSettings()
{
Indent = true,
};
var schemaPath = Path.Combine(outputPath, "schemas", name + ".schema.xml");
using (var xml = XmlWriter.Create(schemaPath, settings))
{
xml.WriteStartDocument();
xml.WriteStartElement("parser");
xml.WriteAttributeString("name", name);
xml.WriteStartElement("table");
ExportParserTable(memory, pointer, xml, parsers, enums);
xml.WriteEndElement();
xml.WriteEndElement();
xml.WriteEndDocument();
}
}
}
}
private static IEnumerable <KeyValuePair <string, string> > ReadEnum(ProcessMemory memory, uint baseAddress)
{
var elements = new List <KeyValuePair <string, string> >();
var valueType = 4;
while (true)
{
var type = memory.ReadU32(baseAddress);
if (type == 0)
{
break;
}
switch (type)
{
case 1:
{
valueType = 1;
baseAddress += 8;
break;
}
case 2:
{
valueType = 2;
baseAddress += 8;
break;
}
// dynamic bullshit
case 3:
{
baseAddress += 8;
break;
}
case 5:
{
var parent = memory.ReadU32(baseAddress + 4);
var nested = ReadEnum(memory, parent);
elements.AddRange(nested);
return(elements);
}
default:
{
var name = memory.ReadStringZ(type);
object value;
switch (valueType)
{
case 1:
{
value = memory.ReadS32(baseAddress + 4);
baseAddress += 8;
break;
}
case 2:
{
value = memory.ReadStringZ(baseAddress + 4);
baseAddress += 8;
break;
}
default:
{
throw new NotImplementedException();
}
}
elements.Add(new KeyValuePair <string, string>(name, value.ToString()));
break;
}
}
}
return(elements);
}
private static void ExportParserTable(
ProcessMemory memory,
uint address,
XmlWriter xml,
List <KeyValuePair <string, uint> > parsers,
List <KeyValuePair <string, uint> > enums)
{
/*xml.WriteComment(string.Format(" {0:X8} ",
* 0x00400000u + (address - (uint)memory.MainModuleAddress.ToInt32())));*/
var currentAddress = address;
var columns = new List <KeyValuePair <Natives.ParserColumn, string> >();
while (true)
{
var column = memory.ReadStructure <Natives.ParserColumn>(currentAddress);
currentAddress += 40;
var name = memory.ReadStringZ(column.NamePointer);
if (column.Type == 0)
{
if (string.IsNullOrEmpty(name) == true)
{
break;
}
}
columns.Add(new KeyValuePair <Natives.ParserColumn, string>(column, name));
}
foreach (var kv in columns)
{
var column = kv.Key;
var name = kv.Value;
xml.WriteStartElement("column");
if (string.IsNullOrEmpty(name) == false)
{
xml.WriteAttributeString("name", name);
}
var token = Parser.GlobalTokens.GetToken(column.Token);
Parser.ColumnFlags flags;
var tokenName = GetTokenName(column, out flags);
xml.WriteAttributeString("type", tokenName);
if (column.Token != 0 &&
column.Token != 1 &&
column.Token != 2)
{
xml.WriteElementString("offset", column.Offset.ToString(CultureInfo.InvariantCulture));
}
var values = new List <string>();
if (column.Flags.HasAnyOptions(Parser.ColumnFlags.PARSETABLE_INFO) == true)
{
xml.WriteStartElement("flags");
xml.WriteElementString("flag", "PARSETABLE_INFO");
xml.WriteEndElement();
/* Star Trek Online replaces its format string with a
* stash table of parsed tokens.
*/
string formatString = null;
if (column.FormatStringPointer != 0)
{
if (memory.ReadU32(column.FormatStringPointer) == 33)
{
formatString = memory.ReadStringZ(memory.ReadU32(column.FormatStringPointer + 4));
}
else
{
formatString = memory.ReadStringZ(column.FormatStringPointer);
}
}
if (string.IsNullOrEmpty(formatString) == false)
{
xml.WriteStartElement("format_strings");
ExportFormatStrings(xml, formatString);
xml.WriteEndElement();
}
}
else
{
if ((flags & _ColumnFlagsMask) != 0)
{
xml.WriteStartElement("flags");
foreach (var flag in _ColumnFlagNames)
{
if ((flags & flag.Key) != 0)
{
xml.WriteElementString("flag", flag.Value);
}
}
xml.WriteEndElement();
}
if (column.Flags.HasAnyOptions(Parser.ColumnFlags.REDUNDANTNAME) == true)
{
var aliased = columns
.Where(c => c.Key != column)
.Where(c => c.Key.Flags.HasAnyOptions(Parser.ColumnFlags.REDUNDANTNAME) == false)
.Where(c => c.Key.Offset == column.Offset)
.Where(c => c.Key.Token == column.Token)
.FirstOrDefault(c => c.Key.Token != 23 || c.Key.Parameter0 == column.Parameter0);
if (aliased.Key != null)
{
xml.WriteElementString("redundant_name", aliased.Value);
}
}
//else
{
if (column.MinBits != 0)
{
if (column.Token == 7)
{
xml.WriteElementString("float_rounding", _FloatRounding[column.MinBits]);
}
else
{
xml.WriteElementString("min_bits", column.MinBits.ToString(CultureInfo.InvariantCulture));
}
}
switch (token.GetParameter(column.Flags, 0))
{
case Parser.ColumnParameter.NumberOfElements:
{
xml.WriteElementString(
"num_elements",
((int)column.Parameter0).ToString(CultureInfo.InvariantCulture));
break;
}
case Parser.ColumnParameter.Default:
{
if (column.Parameter0 != 0)
{
xml.WriteElementString(
"default",
((int)column.Parameter0).ToString(CultureInfo.InvariantCulture));
}
break;
}
case Parser.ColumnParameter.StringLength:
{
xml.WriteElementString(
"string_length",
((int)column.Parameter0).ToString(CultureInfo.InvariantCulture));
break;
}
case Parser.ColumnParameter.DefaultString:
{
if (column.Parameter0 != 0)
{
xml.WriteElementString("default_string", memory.ReadStringZ(column.Parameter0) ?? "");
}
break;
}
case Parser.ColumnParameter.CommandString:
{
if (column.Parameter0 != 0)
{
xml.WriteElementString("command_string", memory.ReadStringZ(column.Parameter0) ?? "");
}
break;
}
case Parser.ColumnParameter.Size:
{
xml.WriteElementString(
"size",
((int)column.Parameter0).ToString(CultureInfo.InvariantCulture));
break;
}
case Parser.ColumnParameter.BitOffset:
{
xml.WriteElementString(
"bit_offset",
((int)column.Parameter0).ToString(CultureInfo.InvariantCulture));
break;
}
}
switch (token.GetParameter(column.Flags, 1))
{
case Parser.ColumnParameter.StaticDefineList:
{
if (column.Parameter1 != 0)
{
xml.WriteStartElement("static_define_list");
var possibleEnum = enums.SingleOrDefault(e => e.Value == column.Parameter1);
if (possibleEnum.Key == null)
{
xml.WriteComment(" dynamic enum? ");
/*xml.WriteStartElement("enum");
* ExportEnum(memory, column.Parameter1, xml);
* xml.WriteEndElement();*/
}
else
{
xml.WriteAttributeString("external", possibleEnum.Key);
}
//xml.WriteComment(string.Format(" {0:X8} ", column.Parameter1));
xml.WriteEndElement();
}
break;
}
case Parser.ColumnParameter.DictionaryName:
{
if (column.Parameter1 != 0)
{
xml.WriteElementString("dictionary_name",
memory.ReadStringZ(column.Parameter1) ?? "");
}
break;
}
case Parser.ColumnParameter.Subtable:
{
if (column.Parameter1 == 0)
{
xml.WriteElementString("subtable", "NULL?");
}
else
{
xml.WriteStartElement("subtable");
var parser = parsers.SingleOrDefault(p => p.Value == column.Parameter1);
if (parser.Key == null)
{
//xml.WriteElementString("subtable", "NOT FOUND? " + column.Parameter1.ToString("X*"));
xml.WriteStartElement("table");
ExportParserTable(memory, column.Parameter1, xml, parsers, enums);
xml.WriteEndElement();
}
else
{
xml.WriteAttributeString("external", parser.Key);
}
xml.WriteEndElement();
}
break;
}
}
var format = column.Format & 0xFF;
if (format != 0)
{
if (format < _FormatNames.Length &&
_FormatNames[format] != null)
{
xml.WriteElementString("format", _FormatNames[format]);
}
else
{
xml.WriteElementString("format_raw", format.ToString(CultureInfo.InvariantCulture));
}
}
/* Star Trek Online replaces its format string with a
* stash table of parsed tokens.
*/
string formatString = null;
if (column.FormatStringPointer != 0)
{
if (memory.ReadU32(column.FormatStringPointer) == 33)
{
formatString = memory.ReadStringZ(memory.ReadU32(column.FormatStringPointer + 4));
}
else
{
formatString = memory.ReadStringZ(column.FormatStringPointer);
}
}
if (string.IsNullOrEmpty(formatString) == false)
{
xml.WriteStartElement("format_strings");
ExportFormatStrings(xml, formatString);
xml.WriteEndElement();
}
}
}
xml.WriteEndElement();
}
}
private static List <KeyValuePair <string, uint> > ExportEnums(ProcessMemory memory, string outputPath)
{
var enums = new List <KeyValuePair <string, uint> >();
uint stashPointer;
if (Locate(memory,
out stashPointer,
Locators.EnumTableLocator1.Locate,
Locators.EnumTableLocator2.Locate,
Locators.EnumTableLocator3.Locate) == false)
{
Console.WriteLine("Warning: failed to locate enum table.");
return(enums);
}
var stashCount = memory.ReadS32(stashPointer + 0x08);
var stashEntryPointer = memory.ReadU32(stashPointer + 0x14);
var stashEntries = memory.ReadAllBytes(stashEntryPointer, stashCount * 8);
var enumLocations = new Dictionary <uint, string>();
for (int i = 0; i < stashCount; i++)
{
var namePointer = BitConverter.ToUInt32(stashEntries, (i * 8) + 0);
var dataPointer = BitConverter.ToUInt32(stashEntries, (i * 8) + 4);
if (namePointer == 0 &&
dataPointer == 0)
{
continue;
}
var name = memory.ReadStringZ(namePointer);
if (enumLocations.ContainsKey(dataPointer) == true)
{
var otherName = enumLocations[dataPointer];
if (name != otherName)
{
// Cryptic pls
if (name != "PowerCategory" && otherName != "PowerCategories")
{
throw new InvalidOperationException();
}
}
continue;
}
enumLocations.Add(dataPointer, name);
}
if (enumLocations.Count > 0)
{
Directory.CreateDirectory(Path.Combine(outputPath, "enums"));
foreach (var kv in enumLocations.OrderBy(kv => kv.Value))
{
var name = kv.Value;
var pointer = kv.Key;
var elements = ReadEnum(memory, pointer);
if (elements == null)
{
continue;
}
enums.Add(new KeyValuePair <string, uint>(name, pointer));
Console.WriteLine("[enum] {0}", name);
var settings = new XmlWriterSettings()
{
Indent = true,
};
using (
var xml = XmlWriter.Create(Path.Combine(outputPath, "enums", name + ".enum.xml"), settings))
{
xml.WriteStartDocument();
xml.WriteStartElement("enum");
xml.WriteAttributeString("name", name);
ExportEnum(memory, pointer, elements, xml);
xml.WriteEndElement();
xml.WriteEndDocument();
}
}
}
return(enums);
}
