public WDC3Reader(Stream stream)
{
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
if (reader.BaseStream.Length < HeaderSize)
{
throw new InvalidDataException("WDC3 file is corrupted!");
}
uint magic = reader.ReadUInt32();
if (magic != WDC3FmtSig)
{
throw new InvalidDataException("WDC3 file is corrupted!");
}
RecordsCount = reader.ReadInt32();
FieldsCount = reader.ReadInt32();
RecordSize = reader.ReadInt32();
StringTableSize = reader.ReadInt32();
TableHash = reader.ReadUInt32();
LayoutHash = reader.ReadUInt32();
MinIndex = reader.ReadInt32();
MaxIndex = reader.ReadInt32();
int locale = reader.ReadInt32();
Flags = (DB2Flags)reader.ReadUInt16();
IdFieldIndex = reader.ReadUInt16();
int totalFieldsCount = reader.ReadInt32();
PackedDataOffset = reader.ReadInt32(); // Offset within the field where packed data starts
int lookupColumnCount = reader.ReadInt32(); // count of lookup columns
int columnMetaDataSize = reader.ReadInt32(); // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
int commonDataSize = reader.ReadInt32();
int palletDataSize = reader.ReadInt32(); // in bytes, sizeof(DBC2PalletValue) == 4
int sectionsCount = reader.ReadInt32();
if (sectionsCount == 0 || RecordsCount == 0)
{
return;
}
var sections = reader.ReadArray <SectionHeaderWDC3>(sectionsCount).ToList();
m_sections = sections.OfType <IEncryptableDatabaseSection>().ToList();
// field meta data
Meta = reader.ReadArray <FieldMetaData>(FieldsCount);
// column meta data
ColumnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);
// pallet data
PalletData = new Value32[ColumnMeta.Length][];
for (int i = 0; i < ColumnMeta.Length; i++)
{
if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray)
{
PalletData[i] = reader.ReadArray <Value32>((int)ColumnMeta[i].AdditionalDataSize / 4);
}
}
// common data
CommonData = new Dictionary <int, Value32> [ColumnMeta.Length];
for (int i = 0; i < ColumnMeta.Length; i++)
{
if (ColumnMeta[i].CompressionType == CompressionType.Common)
{
var commonValues = new Dictionary <int, Value32>((int)ColumnMeta[i].AdditionalDataSize / 8);
CommonData[i] = commonValues;
for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++)
{
commonValues[reader.ReadInt32()] = reader.Read <Value32>();
}
}
}
int previousStringTableSize = 0, previousRecordCount = 0;
foreach (var section in sections)
{
reader.BaseStream.Position = section.FileOffset;
if (!Flags.HasFlagExt(DB2Flags.Sparse))
{
// records data
RecordsData = reader.ReadBytes(section.NumRecords * RecordSize);
Array.Resize(ref RecordsData, RecordsData.Length + 8); // pad with extra zeros so we don't crash when reading
// string data
if (StringTable == null)
{
StringTable = new Dictionary <long, string>(section.StringTableSize / 0x20);
}
for (int i = 0; i < section.StringTableSize;)
{
long oldPos = reader.BaseStream.Position;
StringTable[i + previousStringTableSize] = reader.ReadCString();
i += (int)(reader.BaseStream.Position - oldPos);
}
previousStringTableSize += section.StringTableSize;
}
else
{
// sparse data with inlined strings
RecordsData = reader.ReadBytes(section.OffsetRecordsEndOffset - section.FileOffset);
if (reader.BaseStream.Position != section.OffsetRecordsEndOffset)
{
throw new Exception("reader.BaseStream.Position != section.OffsetRecordsEndOffset");
}
}
// skip encrypted sections => has tact key + record data is zero filled
if (section.TactKeyLookup != 0 && Array.TrueForAll(RecordsData, x => x == 0))
{
previousRecordCount += section.NumRecords;
continue;
}
// index data
IndexData = reader.ReadArray <int>(section.IndexDataSize / 4);
// fix zero-filled index data
if (IndexData.Length > 0 && IndexData.All(x => x == 0))
{
IndexData = Enumerable.Range(MinIndex + previousRecordCount, section.NumRecords).ToArray();
}
// duplicate rows data
if (section.CopyTableCount > 0)
{
if (CopyData == null)
{
CopyData = new Dictionary <int, int>();
}
for (int i = 0; i < section.CopyTableCount; i++)
{
CopyData[reader.ReadInt32()] = reader.ReadInt32();
}
}
if (section.OffsetMapIDCount > 0)
{
// HACK unittestsparse is malformed and has sparseIndexData first
if (TableHash == 145293629)
{
reader.BaseStream.Position += 4 * section.OffsetMapIDCount;
}
SparseEntries = reader.ReadArray <SparseEntry>(section.OffsetMapIDCount).ToList();
}
// reference data
ReferenceData refData = new ReferenceData();
if (section.ParentLookupDataSize > 0)
{
refData.NumRecords = reader.ReadInt32();
refData.MinId = reader.ReadInt32();
refData.MaxId = reader.ReadInt32();
var entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
for (int i = 0; i < entries.Length; i++)
{
refData.Entries[entries[i].Index] = entries[i].Id;
}
}
if (section.OffsetMapIDCount > 0)
{
int[] sparseIndexData = reader.ReadArray <int>(section.OffsetMapIDCount);
if (section.IndexDataSize > 0 && IndexData.Length != sparseIndexData.Length)
{
throw new Exception("m_indexData.Length != sparseIndexData.Length");
}
IndexData = sparseIndexData;
}
int position = 0;
for (int i = 0; i < section.NumRecords; i++)
{
BitReader bitReader = new BitReader(RecordsData)
{
Position = 0
};
if (Flags.HasFlagExt(DB2Flags.Sparse))
{
bitReader.Position = position;
position += SparseEntries[i].Size * 8;
}
else
{
bitReader.Offset = i * RecordSize;
}
refData.Entries.TryGetValue(i, out int refId);
IDBRow rec = new WDC3Row(this, bitReader, section.IndexDataSize != 0 ? IndexData[i] : -1, refId, i + previousRecordCount);
_Records.Add(_Records.Count, rec);
}
previousRecordCount += section.NumRecords;
}
}
}
public WDC3Reader(Stream stream)
{
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
if (reader.BaseStream.Length < HeaderSize)
{
throw new InvalidDataException("WDC3 file is corrupted!");
}
uint magic = reader.ReadUInt32();
if (magic != WDC3FmtSig)
{
throw new InvalidDataException("WDC3 file is corrupted!");
}
RecordsCount = reader.ReadInt32();
FieldsCount = reader.ReadInt32();
RecordSize = reader.ReadInt32();
StringTableSize = reader.ReadInt32();
TableHash = reader.ReadUInt32();
LayoutHash = reader.ReadUInt32();
MinIndex = reader.ReadInt32();
MaxIndex = reader.ReadInt32();
Locale = reader.ReadInt32();
Flags = (DB2Flags)reader.ReadUInt16();
IdFieldIndex = reader.ReadUInt16();
int totalFieldsCount = reader.ReadInt32();
PackedDataOffset = reader.ReadInt32(); // Offset within the field where packed data starts
lookupColumnCount = reader.ReadInt32(); // count of lookup columns
field_info_size = reader.ReadInt32(); // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
commonDataSize = reader.ReadInt32();
palletDataSize = reader.ReadInt32(); // in bytes, sizeof(DBC2PalletValue) == 4
SectionsCount = reader.ReadInt32();
if (SectionsCount == 0 || RecordsCount == 0)
{
return;
}
//section headers
SectionHeaders = reader.ReadArray <SectionHeaderWDC3>(SectionsCount).ToList();
m_sections = SectionHeaders.OfType <IEncryptableDatabaseSection>().ToList();
// field meta data
field_structure_data = reader.ReadArray <FieldMetaData>(FieldsCount);
// column meta data
ColumnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);
// pallet data
PalletData = new Value32[ColumnMeta.Length][];
for (int i = 0; i < ColumnMeta.Length; i++)
{
if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray)
{
PalletData[i] = reader.ReadArray <Value32>((int)ColumnMeta[i].AdditionalDataSize / 4);
}
}
// common data
CommonData = new Dictionary <int, Value32> [ColumnMeta.Length];
for (int i = 0; i < ColumnMeta.Length; i++)
{
if (ColumnMeta[i].CompressionType == CompressionType.Common)
{
var commonValues = new Dictionary <int, Value32>((int)ColumnMeta[i].AdditionalDataSize / 8);
CommonData[i] = commonValues;
for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++)
{
commonValues[reader.ReadInt32()] = reader.Read <Value32>();
}
}
}
//section_data
int StringTableSizeCounter = 0, RecordCountCounter = 0;
//foreach (var sectionheader in SectionHeaders)
//{
reader.BaseStream.Position = SectionHeaders[0].FileOffset;
if (!Flags.HasFlagExt(DB2Flags.Sparse)) //'no offset map'
{
// records data
RecordsData = reader.ReadBytes(SectionHeaders[0].NumRecords * RecordSize);
Array.Resize(ref RecordsData, RecordsData.Length + 8); // pad with extra zeros so we don't crash when reading
// string data
if (StringTable == null)
{
StringTable = new Dictionary <long, string>(SectionHeaders[0].StringTableSize / 0x20);
}
for (int i = 0; i < SectionHeaders[0].StringTableSize;)
{
long oldPos = reader.BaseStream.Position;
StringTable[i + StringTableSizeCounter] = reader.ReadCString();
i += (int)(reader.BaseStream.Position - oldPos);
}
StringTableSizeCounter += SectionHeaders[0].StringTableSize;
}
else //'Has offset map'
{
// sparse data with inlined strings
RecordsData = reader.ReadBytes(SectionHeaders[0].OffsetRecordsEndOffset - SectionHeaders[0].FileOffset);
if (reader.BaseStream.Position != SectionHeaders[0].OffsetRecordsEndOffset)
{
throw new Exception("reader.BaseStream.Position != section.OffsetRecordsEndOffset");
}
}
// skip encrypted sections => has tact key + record data is zero filled
if (SectionHeaders[0].TactKeyLookup != 0 && Array.TrueForAll(RecordsData, x => x == 0))
{
RecordCountCounter += SectionHeaders[0].NumRecords;
return;
}
// id_list_data
id_list_data = reader.ReadArray <int>(SectionHeaders[0].IndexDataSize / 4);
// fix zero-filled index data
if (id_list_data.Length > 0 && id_list_data.All(x => x == 0))
{
id_list_data = Enumerable.Range(MinIndex + RecordCountCounter, SectionHeaders[0].NumRecords).ToArray();
}
// copy table entry data , duplicate rows data
if (SectionHeaders[0].CopyTableCount > 0)
{
if (CopyData == null)
{
CopyData = new Dictionary <int, int>();
}
for (int i = 0; i < SectionHeaders[0].CopyTableCount; i++)
{
CopyData[reader.ReadInt32()] = reader.ReadInt32(); //id_of_new_row, id_of_copied_row
}
}
// offset map entry data
if (SectionHeaders[0].OffsetMapIDCount > 0)
{
// HACK unittestsparse is malformed and has sparseIndexData first
if (TableHash == 145293629)
{
reader.BaseStream.Position += 4 * SectionHeaders[0].OffsetMapIDCount;
}
offset_map_Entries = reader.ReadArray <offset_map_entry>(SectionHeaders[0].OffsetMapIDCount).ToList();
}
// reference/relationship data
refData = new ReferenceData();
if (SectionHeaders[0].ParentLookupDataSize > 0)
{
refData.NumRecords = reader.ReadInt32();
refData.MinId = reader.ReadInt32();
refData.MaxId = reader.ReadInt32();
var entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
for (int i = 0; i < entries.Length; i++)
{
refData.Entries[entries[i].Index] = entries[i].Id;
}
}
//offset_map_id_list
if (SectionHeaders[0].OffsetMapIDCount > 0)
{
int[] offset_map_id_listdata = reader.ReadArray <int>(SectionHeaders[0].OffsetMapIDCount);
if (SectionHeaders[0].IndexDataSize > 0 && id_list_data.Length != offset_map_id_listdata.Length)
{
throw new Exception("m_indexData.Length != sparseIndexData.Length");
}
id_list_data = offset_map_id_listdata;
}
//loop and add field data to create datarow
int position = 0;
for (int i = 0; i < SectionHeaders[0].NumRecords; i++)
{
BitReader bitReader = new BitReader(RecordsData)
{
Position = 0
};
if (Flags.HasFlagExt(DB2Flags.Sparse))
{
bitReader.Position = position;
position += offset_map_Entries[i].Size * 8;
}
else
{
bitReader.Offset = i * RecordSize;
}
refData.Entries.TryGetValue(i, out int refId);
IDBRow rec = new WDC3Row(this, bitReader, SectionHeaders[0].IndexDataSize != 0 ? id_list_data[i] : -1, refId, i + RecordCountCounter);
_Records.Add(_Records.Count, rec);
}
RecordCountCounter += SectionHeaders[0].NumRecords;
//}
long pos = reader.BaseStream.Position;
long len = reader.BaseStream.Length;
NoParseRecordsData = reader.ReadBytes((int)(len - pos));
}
}