public void Read <T>(FieldCache[] fields, T entry, BitReader r, int recordOffset, Dictionary <long, string> stringsTable, FieldMetaData[] fieldMeta, ColumnMetaData[] columnMeta, Value32[][] palletData, Dictionary <int, Value32>[] commonData, int id, int refId, bool isSparse = false) where T : ClientDBRow
{
int fieldIndex = 0;
foreach (var f in fields)
{
if (f.IsIndex && id != -1)
{
((FieldCache <T, int>)f).Setter(entry, id);
continue;
}
if (fieldIndex >= fieldMeta.Length)
{
if (refId != -1)
{
((FieldCache <T, int>)f).Setter(entry, refId);
}
continue;
}
if (f.IsArray)
{
switch (f)
{
case FieldCache <T, int[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <int>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, uint[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <uint>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, byte[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <byte>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, sbyte[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <sbyte>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, short[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <short>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, ushort[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <ushort>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, float[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <float>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, long[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <long>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, ulong[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueArray <ulong>(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], c1.ArraySize));
break;
case FieldCache <T, string[]> c1:
c1.Setter(entry, FieldReader.GetFieldValueStringsArray(r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex], stringsTable, isSparse, recordOffset, c1.ArraySize));
break;
default:
throw new Exception($"Unhandled DbcTable type: {f.Field.FieldType.FullName} in {f.Field.DeclaringType.FullName}.{f.Field.Name}");
}
}
else
{
switch (f)
{
case FieldCache <T, int> c1:
c1.Setter(entry, FieldReader.GetFieldValue <int>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, uint> c1:
c1.Setter(entry, FieldReader.GetFieldValue <uint>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, byte> c1:
c1.Setter(entry, FieldReader.GetFieldValue <byte>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, sbyte> c1:
c1.Setter(entry, FieldReader.GetFieldValue <sbyte>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, short> c1:
c1.Setter(entry, FieldReader.GetFieldValue <short>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, ushort> c1:
c1.Setter(entry, FieldReader.GetFieldValue <ushort>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, float> c1:
c1.Setter(entry, FieldReader.GetFieldValue <float>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, long> c1:
c1.Setter(entry, FieldReader.GetFieldValue <long>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, ulong> c1:
c1.Setter(entry, FieldReader.GetFieldValue <ulong>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex]));
break;
case FieldCache <T, string> c1:
c1.Setter(entry, isSparse ? r.ReadCString() : stringsTable[(recordOffset + (r.Position >> 3)) + FieldReader.GetFieldValue <int>(GetId(), r, fieldMeta[fieldIndex], columnMeta[fieldIndex], palletData[fieldIndex], commonData[fieldIndex])]);
break;
default:
throw new Exception($"Unhandled DbcTable type: {f.Field.FieldType.FullName} in {f.Field.DeclaringType.FullName}.{f.Field.Name}");
}
}
fieldIndex++;
}
}
public WDC3Reader(Stream stream, Func <ulong, bool> hasTactKey = null)
{
if (hasTactKey == null)
{
hasTactKeyFunc = (key) => false;
}
else
{
hasTactKeyFunc = hasTactKey;
}
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
if (reader.BaseStream.Length < HeaderSize)
{
throw new InvalidDataException(String.Format("WDC3 file is corrupted!"));
}
uint magic = reader.ReadUInt32();
if (magic != WDC3FmtSig)
{
throw new InvalidDataException(String.Format("WDC3 file is corrupted!"));
}
RecordsCount = reader.ReadInt32();
FieldsCount = reader.ReadInt32();
RecordSize = reader.ReadInt32();
StringTableSize = reader.ReadInt32();
TableHash = reader.ReadUInt32();
LayoutHash = reader.ReadUInt32();
MinIndex = reader.ReadInt32();
MaxIndex = reader.ReadInt32();
int locale = reader.ReadInt32();
int flags = reader.ReadUInt16();
IdFieldIndex = reader.ReadUInt16();
int totalFieldsCount = reader.ReadInt32();
int packedDataOffset = reader.ReadInt32(); // Offset within the field where packed data starts
int lookupColumnCount = reader.ReadInt32(); // count of lookup columns
int columnMetaDataSize = reader.ReadInt32(); // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
int commonDataSize = reader.ReadInt32();
int palletDataSize = reader.ReadInt32(); // in bytes, sizeof(DBC2PalletValue) == 4
int sectionsCount = reader.ReadInt32();
//if (sectionsCount > 1)
// throw new Exception("sectionsCount > 1");
SectionHeader_WDC3[] sections = reader.ReadArray <SectionHeader_WDC3>(sectionsCount);
// field meta data
m_meta = reader.ReadArray <FieldMetaData>(FieldsCount);
if (sectionsCount == 0 || RecordsCount == 0)
{
return;
}
// column meta data
m_columnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);
// pallet data
m_palletData = new Value32[m_columnMeta.Length][];
for (int i = 0; i < m_columnMeta.Length; i++)
{
if (m_columnMeta[i].CompressionType == CompressionType.Pallet || m_columnMeta[i].CompressionType == CompressionType.PalletArray)
{
m_palletData[i] = reader.ReadArray <Value32>((int)m_columnMeta[i].AdditionalDataSize / 4);
}
}
// common data
m_commonData = new Dictionary <int, Value32> [m_columnMeta.Length];
for (int i = 0; i < m_columnMeta.Length; i++)
{
if (m_columnMeta[i].CompressionType == CompressionType.Common)
{
Dictionary <int, Value32> commonValues = new Dictionary <int, Value32>();
m_commonData[i] = commonValues;
for (int j = 0; j < m_columnMeta[i].AdditionalDataSize / 8; j++)
{
commonValues[reader.ReadInt32()] = reader.Read <Value32>();
}
}
}
Dictionary <long, string> stringsTable = new Dictionary <long, string>();
bool isSparse = (flags & 0x1) != 0;
bool hasIndex = (flags & 0x4) != 0;
int previousStringTableSize = 0, previousRecordCount = 0;
for (int sectionIndex = 0; sectionIndex < sectionsCount; sectionIndex++)
{
if (sections[sectionIndex].TactKeyLookup != 0 && !hasTactKeyFunc(sections[sectionIndex].TactKeyLookup))
{
//Console.WriteLine("Detected db2 with encrypted section! HasKey {0}", CASC.HasKey(Sections[sectionIndex].TactKeyLookup));
previousStringTableSize += sections[sectionIndex].StringTableSize;
previousRecordCount += sections[sectionIndex].NumRecords;
continue;
}
reader.BaseStream.Position = sections[sectionIndex].FileOffset;
byte[] recordsData;
SparseEntry[] sparseEntries = null;
if (isSparse)
{
// sparse data with inlined strings
recordsData = reader.ReadBytes(sections[sectionIndex].SparseDataEndOffset - sections[sectionIndex].FileOffset);
if (reader.BaseStream.Position != sections[sectionIndex].SparseDataEndOffset)
{
throw new Exception("reader.BaseStream.Position != sections[sectionIndex].SparseDataEndOffset");
}
}
else
{
// records data
recordsData = reader.ReadBytes(sections[sectionIndex].NumRecords * RecordSize);
// string data
for (int i = 0; i < sections[sectionIndex].StringTableSize;)
{
long oldPos = reader.BaseStream.Position;
stringsTable[i + previousStringTableSize] = CStringExtensions.ReadCString(reader);
i += (int)(reader.BaseStream.Position - oldPos);
}
}
Array.Resize(ref recordsData, recordsData.Length + 8); // pad with extra zeros so we don't crash when reading
// index data
int[] indexData = reader.ReadArray <int>(sections[sectionIndex].IndexDataSize / 4);
bool isIndexEmpty = hasIndex && indexData.Count(i => i == 0) == sections[sectionIndex].NumRecords;
// duplicate rows data
Dictionary <int, int> copyData = new Dictionary <int, int>();
for (int i = 0; i < sections[sectionIndex].NumCopyRecords; i++)
{
copyData[reader.ReadInt32()] = reader.ReadInt32();
}
if (sections[sectionIndex].NumSparseRecords > 0)
{
sparseEntries = reader.ReadArray <SparseEntry>(sections[sectionIndex].NumSparseRecords);
}
// reference data
ReferenceData refData = null;
if (sections[sectionIndex].ParentLookupDataSize > 0)
{
refData = new ReferenceData
{
NumRecords = reader.ReadInt32(),
MinId = reader.ReadInt32(),
MaxId = reader.ReadInt32()
};
ReferenceEntry[] entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
refData.Entries = new Dictionary <int, int>();
for (int i = 0; i < entries.Length; i++)
{
refData.Entries[entries[i].Index] = entries[i].Id;
}
}
else
{
refData = new ReferenceData
{
Entries = new Dictionary <int, int>()
};
}
if (sections[sectionIndex].NumSparseRecords > 0)
{
// TODO: use this shit
int[] sparseIndexData = reader.ReadArray <int>(sections[sectionIndex].NumSparseRecords);
if (hasIndex && indexData.Length != sparseIndexData.Length)
{
throw new Exception("indexData.Length != sparseIndexData.Length");
}
indexData = sparseIndexData;
}
BitReader bitReader = new BitReader(recordsData);
if (sections[sectionIndex].NumSparseRecords > 0 && sections[sectionIndex].NumRecords != sections[sectionIndex].NumSparseRecords)
{
throw new Exception("sections[sectionIndex].NumSparseRecords > 0 && sections[sectionIndex].NumRecords != sections[sectionIndex].NumSparseRecords");
}
for (int i = 0; i < sections[sectionIndex].NumRecords; ++i)
{
bitReader.Position = 0;
if (isSparse)
{
bitReader.Offset = sparseEntries[i].Offset - sections[sectionIndex].FileOffset;
}
else
{
bitReader.Offset = i * RecordSize;
}
bool hasRef = refData.Entries.TryGetValue(i, out int refId);
int recordIndex = i + previousRecordCount;
int recordOffset = (recordIndex * RecordSize) - (RecordsCount * RecordSize);
WDC3Row rec = new WDC3Row(this, bitReader, recordOffset, hasIndex ? (isIndexEmpty ? i : indexData[i]) : -1, hasRef ? refId : -1, isSparse, stringsTable);
_Records.Add(rec.GetId(), rec);
if (i % 1000 == 0)
{
Console.Write("\r{0} records read", i);
}
}
foreach (var copyRow in copyData)
{
WDC3Row rec = (WDC3Row)_Records[copyRow.Value].Clone();
rec.SetId(copyRow.Key);
_Records.Add(copyRow.Key, rec);
}
previousStringTableSize += sections[sectionIndex].StringTableSize;
previousRecordCount += sections[sectionIndex].NumRecords;
}
}
}
public WDC1Reader(Stream stream)
{
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
if (reader.BaseStream.Length < HeaderSize)
{
throw new InvalidDataException(String.Format("WDC1 file is corrupted!"));
}
uint magic = reader.ReadUInt32();
if (magic != WDC1FmtSig)
{
throw new InvalidDataException(String.Format("WDC1 file is corrupted!"));
}
RecordsCount = reader.ReadInt32();
FieldsCount = reader.ReadInt32();
RecordSize = reader.ReadInt32();
StringTableSize = reader.ReadInt32();
TableHash = reader.ReadUInt32();
LayoutHash = reader.ReadUInt32();
MinIndex = reader.ReadInt32();
MaxIndex = reader.ReadInt32();
int locale = reader.ReadInt32();
int copyTableSize = reader.ReadInt32();
int flags = reader.ReadUInt16();
IdFieldIndex = reader.ReadUInt16();
int totalFieldsCount = reader.ReadInt32();
int packedDataOffset = reader.ReadInt32(); // Offset within the field where packed data starts
int lookupColumnCount = reader.ReadInt32(); // count of lookup columns
int sparseTableOffset = reader.ReadInt32(); // absolute value, {uint offset, ushort size}[MaxId - MinId + 1]
int indexDataSize = reader.ReadInt32(); // int indexData[IndexDataSize / 4]
int columnMetaDataSize = reader.ReadInt32(); // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
int commonDataSize = reader.ReadInt32();
int palletDataSize = reader.ReadInt32(); // in bytes, sizeof(DBC2PalletValue) == 4
int referenceDataSize = reader.ReadInt32(); // uint NumRecords, uint minId, uint maxId, {uint id, uint index}[NumRecords], questionable usefulness...
// field meta data
m_meta = reader.ReadArray <FieldMetaData>(FieldsCount);
byte[] recordsData;
Dictionary <long, string> stringsTable = null;
SparseEntry[] sparseEntries = null;
if ((flags & 0x1) == 0)
{
// records data
recordsData = reader.ReadBytes(RecordsCount * RecordSize);
Array.Resize(ref recordsData, recordsData.Length + 8); // pad with extra zeros so we don't crash when reading
// string data
stringsTable = new Dictionary <long, string>();
for (int i = 0; i < StringTableSize;)
{
long oldPos = reader.BaseStream.Position;
stringsTable[i] = CStringExtensions.ReadCString(reader);
i += (int)(reader.BaseStream.Position - oldPos);
}
}
else
{
// sparse data with inlined strings
recordsData = reader.ReadBytes(sparseTableOffset - HeaderSize - Marshal.SizeOf <FieldMetaData>() * FieldsCount);
if (reader.BaseStream.Position != sparseTableOffset)
{
throw new Exception("r.BaseStream.Position != sparseTableOffset");
}
sparseEntries = reader.ReadArray <SparseEntry>(MaxIndex - MinIndex + 1);
if (sparseTableOffset != 0)
{
throw new Exception("Sparse Table NYI!");
}
else
{
throw new Exception("Sparse Table with zero offset?");
}
}
// index data
int[] indexData = reader.ReadArray <int>(indexDataSize / 4);
// duplicate rows data
Dictionary <int, int> copyData = new Dictionary <int, int>();
for (int i = 0; i < copyTableSize / 8; i++)
{
copyData[reader.ReadInt32()] = reader.ReadInt32();
}
// column meta data
m_columnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);
// pallet data
m_palletData = new Value32[m_columnMeta.Length][];
for (int i = 0; i < m_columnMeta.Length; i++)
{
if (m_columnMeta[i].CompressionType == CompressionType.Pallet || m_columnMeta[i].CompressionType == CompressionType.PalletArray)
{
m_palletData[i] = reader.ReadArray <Value32>((int)m_columnMeta[i].AdditionalDataSize / 4);
}
}
// common data
m_commonData = new Dictionary <int, Value32> [m_columnMeta.Length];
for (int i = 0; i < m_columnMeta.Length; i++)
{
if (m_columnMeta[i].CompressionType == CompressionType.Common)
{
Dictionary <int, Value32> commonValues = new Dictionary <int, Value32>();
m_commonData[i] = commonValues;
for (int j = 0; j < m_columnMeta[i].AdditionalDataSize / 8; j++)
{
commonValues[reader.ReadInt32()] = reader.Read <Value32>();
}
}
}
// reference data
ReferenceData refData = null;
if (referenceDataSize > 0)
{
refData = new ReferenceData
{
NumRecords = reader.ReadInt32(),
MinId = reader.ReadInt32(),
MaxId = reader.ReadInt32()
};
ReferenceEntry[] entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
refData.Entries = entries.ToDictionary(e => e.Index, e => e.Id);
}
else
{
refData = new ReferenceData
{
Entries = new Dictionary <int, int>()
};
}
BitReader bitReader = new BitReader(recordsData);
for (int i = 0; i < RecordsCount; ++i)
{
bitReader.Position = 0;
bitReader.Offset = i * RecordSize;
bool hasRef = refData.Entries.TryGetValue(i, out int refId);
WDC1Row rec = new WDC1Row(this, bitReader, indexDataSize != 0 ? indexData[i] : -1, hasRef ? refId : -1, stringsTable);
if (indexDataSize != 0)
{
_Records.Add(indexData[i], rec);
}
else
{
_Records.Add(rec.GetId(), rec);
}
if (i % 1000 == 0)
{
Console.Write("\r{0} records read", i);
}
}
foreach (var copyRow in copyData)
{
WDC1Row rec = (WDC1Row)_Records[copyRow.Value].Clone();
rec.SetId(copyRow.Key);
_Records.Add(copyRow.Key, rec);
}
}
}