private void ParseWDC1(MemoryStream stream, string file)
{
stream.Position = 0;
Dictionary <int, FieldType> FieldTypes = new Dictionary <int, FieldType>()
{
{ 8, FieldType.ULONG },
{ 4, FieldType.INT },
{ 2, FieldType.USHORT },
{ 1, FieldType.BYTE },
};
using (var dbReader = new BinaryReader(stream, Encoding.UTF8))
{
WDC1 header = ExtractHeader(dbReader) as WDC1;
if (header == null)
{
return;
}
if (header.RecordCount == 0 || header.RecordSize == 0)
{
return;
}
long pos = dbReader.BaseStream.Position;
Dictionary <int, string> StringTable = new StringTable().Read(dbReader, pos, pos + header.StringBlockSize);
bool stringtableused = StringTable.Values.Any(x => !string.IsNullOrWhiteSpace(x)) && !header.HasOffsetTable;
List <FieldInfo> fields = new List <FieldInfo>();
var copyTable = header.ReadOffsetData(dbReader, dbReader.BaseStream.Position);
for (int f = 0; f < header.ColumnMeta.Count; f++)
{
FieldType byteType;
if (f == header.IdIndex || (f == 0 && header.HasIndexTable))
{
fields.Add(new FieldInfo()
{
ArraySize = 1, Type = FieldType.INT
});
continue;
}
if (header.ColumnMeta[f].CompressionType == CompressionType.None)
{
int bitSize = header.FieldStructure[f].BitCount;
byteType = FieldTypes[NextPow2(~~(bitSize + 7) / 8)];
}
else if (header.ColumnMeta[f].CompressionType > CompressionType.Immediate)
{
byteType = FieldType.INT;
}
else
{
byteType = FieldTypes[NextPow2(~~(header.ColumnMeta[f].BitWidth + 7) / 8)];
}
fields.Add(new FieldInfo()
{
ArraySize = header.ColumnMeta[f].ArraySize,
Type = byteType == FieldType.INT ? FieldType.UNKNOWN : byteType
});
}
int offset = 0;
for (int i = 0; i < fields.Count; i++)
{
switch (fields[i].Type)
{
case FieldType.BYTE:
offset++;
continue;
case FieldType.USHORT:
offset += 2;
continue;
case FieldType.INT:
offset += 4;
continue;
case FieldType.ULONG:
offset += 8;
continue;
}
List <FieldType> options = new List <FieldType>()
{
FieldType.INT, FieldType.FLOAT, FieldType.STRING
};
if (!stringtableused)
{
options.Remove(FieldType.STRING); //Stringtable not used
}
List <int> ints = new List <int>();
List <float> floats = new List <float>();
foreach (var c in copyTable)
{
for (int x = 0; x < fields[i].ArraySize; x++)
{
int asInt = BitConverter.ToInt32(c.Value.Skip(offset + (4 * x)).Take(4).ToArray(), 0);
if (asInt > 0)
{
ints.Add(asInt);
if (FloatUtil.IsLikelyFloat(asInt))
{
floats.Add(BitConverter.ToSingle(BitConverter.GetBytes(asInt), 0));
}
}
}
}
// remove 0's as they could be anything - if all removed then guess its an int
ints.RemoveAll(x => x == 0);
if (ints.Count == 0)
{
fields[i].Type = FieldType.INT;
offset += (4 * fields[i].ArraySize);
continue;
}
// stringtable doesn't contain string so cant be a string
if (options.Contains(FieldType.STRING) && ints.Any(x => !StringTable.ContainsKey(x)))
{
options.Remove(FieldType.STRING);
}
if (floats.Count / (float)ints.Count >= 0.85)
{
fields[i].Type = FieldType.FLOAT;
}
else if (options.Contains(FieldType.STRING))
{
fields[i].Type = FieldType.STRING;
}
else if (header.ColumnMeta[i].CompressionType == CompressionType.Immediate && header.ColumnMeta[i].Cardinality == 0)
{
fields[i].Type = FieldType.UINT;
}
else
{
fields[i].Type = FieldType.INT;
}
offset += (4 * fields[i].ArraySize);
}
Table table = new Table();
table.Name = Path.GetFileNameWithoutExtension(file);
table.Fields = new List <Field>();
string format = $"X{fields.Count.ToString("X").Length}"; //X2, X3 etc
for (int i = 0; i < fields.Count; i++)
{
if (header.RelationshipCount > 0 && i == fields.Count - 1)
{
continue;
}
Field field = new Field();
field.Name = (i == header.IdIndex ? "ID" : $"field{i.ToString(format)}");
field.IsIndex = (i == header.IdIndex);
field.ArraySize = (field.IsIndex ? 1 : fields[i].ArraySize);
field.Type = fields[i].Type.ToString().ToLower();
table.Fields.Add(field);
Console.WriteLine($"Name: {field.Name} | Array: {field.ArraySize} | Type: {field.Type}");
}
tables.Add(table);
Database.ForceGC();
}
}
public static DBEntry Read(string dbFile, int build, out string error)
{
error = "";
using (var fs = new FileStream(dbFile, FileMode.Open, FileAccess.Read))
using (var br = new BinaryReader(fs, Encoding.UTF8))
{
DBHeader header = ReadHeader(br, dbFile, build);
if (IsKnown(header, out DBEntry known))
{
return(known);
}
if (!ValidationChecks(header, dbFile, out error))
{
return(null);
}
if (header.RecordSize / header.FieldCount != 4) // dbc has byte column
{
error = "Has byte columns.";
return(new DBEntry()
{
Name = Path.GetFileName(dbFile).ToUpper(),
Builds = new List <int>()
{
build
},
Fields = new List <DBField>()
});
}
Dictionary <int, string> stringTable = new Dictionary <int, string>();
List <byte[]> dataTable = new List <byte[]>();
FieldInfo[] fieldInfo = new FieldInfo[header.FieldCount];
bool hasStrings = false;
int nullStrings = 0;
// stringtable stuff
long pos = br.BaseStream.Position;
long stringTableStart = br.BaseStream.Position += header.RecordCount * header.RecordSize;
stringTable = ReadStringTable(br, stringTableStart); //Get stringtable
br.Scrub(pos);
// if 1 or 2 empty strings only then strings aren't unused
hasStrings = !(stringTable.Values.Count <= 2 && stringTable.Values.All(x => string.IsNullOrWhiteSpace(x)));
// count empties
nullStrings = stringTable.Count(x => string.IsNullOrWhiteSpace(x.Value));
// read data
for (int i = 0; i < header.RecordCount; i++)
{
dataTable.Add(br.ReadBytes((int)header.RecordSize));
}
// compute possible types
for (int i = 0; i < header.FieldCount; i++)
{
// no pdb dbc struct has uint fields ITS ALL A LIE!!
List <FieldType> options = new List <FieldType>()
{
FieldType.INT, /*FieldType.UINT,*/ FieldType.FLOAT, FieldType.STRING
};
if (!hasStrings)
{
options.Remove(FieldType.STRING); // strings not used
}
List <int> intVals = new List <int>();
List <string> stringVals = new List <string>();
List <float> floatVals = new List <float>();
for (int r = 0; r < dataTable.Count; r++)
{
byte[] data = dataTable[r].Skip(i * 4).Take(4).ToArray();
// ignore 0 byte columns as they could be anything
if (data.All(x => x == 0))
{
continue;
}
// int value
int asInt = BitConverter.ToInt32(data, 0);
intVals.Add(asInt);
// string check
if (options.Contains(FieldType.STRING))
{
if (!stringTable.ContainsKey(asInt))
{
options.Remove(FieldType.STRING);
stringVals.Clear(); // 100% not a string!
}
else
{
stringVals.Add(stringTable[asInt]);
}
}
// float check
if (options.Contains(FieldType.FLOAT) && FloatUtil.IsLikelyFloat(asInt))
{
floatVals.Add(BitConverter.ToSingle(data, 0));
}
// uint check - prefer signed over unsigned as per the wow client
if (options.Contains(FieldType.UINT) && asInt < 0)
{
options.Remove(FieldType.UINT);
}
}
fieldInfo[i] = new FieldInfo()
{
IsEmpty = intVals.Count == 0,
FloatPercentage = (floatVals.Count / (float)intVals.Count) * 100f, // % of valid floats
Options = options,
UniqueStrings = stringVals.Distinct().Count(),
UniqueInts = intVals.Distinct().Count()
};
}
// calculate field types
List <FieldType> temp = new List <FieldType>();
for (int i = 0; i < fieldInfo.Length; i++)
{
var info = fieldInfo[i];
if (info.IsEmpty) // all records are 0
{
// most likely to be int, less likely to be float, very unlikely to be a string
temp.Add(info.Options.Contains(FieldType.UINT) ? FieldType.UINT : FieldType.INT);
}
else if (info.Options.Contains(FieldType.FLOAT) && info.FloatPercentage > FLOAT_THRESHOLD) // threshold needs tweaking?
{
temp.Add(FieldType.FLOAT); // high % of valid floats
}
else if (info.Options.Contains(FieldType.STRING) && info.UniqueStrings > 0)
{
// 1 string, 1st field is more likely an ID not a string
if (stringTable.Count - nullStrings < header.FieldCount && header.RecordCount == 1)
{
if (i == 0)
{
temp.Add(info.Options.Contains(FieldType.UINT) ? FieldType.UINT : FieldType.INT);
}
else
{
temp.Add(FieldType.STRING);
}
}
else if (info.UniqueStrings == 1)
{
// very unlikely to have a column with the same string in every row if there is
temp.Add(info.Options.Contains(FieldType.UINT) ? FieldType.UINT : FieldType.INT);
}
else
{
temp.Add(FieldType.STRING); // case of 0 = "" and 1 = "" in stringtable
}
}
else
{
temp.Add(info.Options.Contains(FieldType.UINT) ? FieldType.UINT : FieldType.INT); // uint over int
}
// LANGREFSTRING check
if (temp[temp.Count - 1] == FieldType.STRING)
{
if (IsLangStringRef(fieldInfo, i + 1, build, out int offset))
{
temp[temp.Count - 1] = FieldType.LANGSTRINGREF;
i += offset;
}
}
}
return(new DBEntry()
{
Name = Path.GetFileName(dbFile).ToUpper(),
Builds = new List <int>()
{
build
},
Fields = temp.Select(x => new DBField()
{
Name = "", Type = x.ToString().ToUpper()
}).ToList()
});
}
}