Exemple #1
0
        public WDC3Reader(Stream stream)
        {
            using (var reader = new BinaryReader(stream, Encoding.UTF8))
            {
                if (reader.BaseStream.Length < HeaderSize)
                {
                    throw new InvalidDataException("WDC3 file is corrupted!");
                }

                uint magic = reader.ReadUInt32();

                if (magic != WDC3FmtSig)
                {
                    throw new InvalidDataException("WDC3 file is corrupted!");
                }

                RecordsCount    = reader.ReadInt32();
                FieldsCount     = reader.ReadInt32();
                RecordSize      = reader.ReadInt32();
                StringTableSize = reader.ReadInt32();
                TableHash       = reader.ReadUInt32();
                LayoutHash      = reader.ReadUInt32();
                MinIndex        = reader.ReadInt32();
                MaxIndex        = reader.ReadInt32();
                int locale = reader.ReadInt32();
                Flags        = (DB2Flags)reader.ReadUInt16();
                IdFieldIndex = reader.ReadUInt16();
                int totalFieldsCount = reader.ReadInt32();
                PackedDataOffset = reader.ReadInt32();          // Offset within the field where packed data starts
                int lookupColumnCount  = reader.ReadInt32();    // count of lookup columns
                int columnMetaDataSize = reader.ReadInt32();    // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
                int commonDataSize     = reader.ReadInt32();
                int palletDataSize     = reader.ReadInt32();    // in bytes, sizeof(DBC2PalletValue) == 4
                int sectionsCount      = reader.ReadInt32();

                if (sectionsCount == 0 || RecordsCount == 0)
                {
                    return;
                }

                var sections = reader.ReadArray <SectionHeaderWDC3>(sectionsCount).ToList();
                m_sections = sections.OfType <IEncryptableDatabaseSection>().ToList();

                // field meta data
                Meta = reader.ReadArray <FieldMetaData>(FieldsCount);

                // column meta data
                ColumnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);

                // pallet data
                PalletData = new Value32[ColumnMeta.Length][];
                for (int i = 0; i < ColumnMeta.Length; i++)
                {
                    if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray)
                    {
                        PalletData[i] = reader.ReadArray <Value32>((int)ColumnMeta[i].AdditionalDataSize / 4);
                    }
                }

                // common data
                CommonData = new Dictionary <int, Value32> [ColumnMeta.Length];
                for (int i = 0; i < ColumnMeta.Length; i++)
                {
                    if (ColumnMeta[i].CompressionType == CompressionType.Common)
                    {
                        var commonValues = new Dictionary <int, Value32>((int)ColumnMeta[i].AdditionalDataSize / 8);
                        CommonData[i] = commonValues;

                        for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++)
                        {
                            commonValues[reader.ReadInt32()] = reader.Read <Value32>();
                        }
                    }
                }

                int previousStringTableSize = 0, previousRecordCount = 0;
                foreach (var section in sections)
                {
                    reader.BaseStream.Position = section.FileOffset;

                    if (!Flags.HasFlagExt(DB2Flags.Sparse))
                    {
                        // records data
                        RecordsData = reader.ReadBytes(section.NumRecords * RecordSize);

                        Array.Resize(ref RecordsData, RecordsData.Length + 8); // pad with extra zeros so we don't crash when reading

                        // string data
                        if (StringTable == null)
                        {
                            StringTable = new Dictionary <long, string>(section.StringTableSize / 0x20);
                        }

                        for (int i = 0; i < section.StringTableSize;)
                        {
                            long oldPos = reader.BaseStream.Position;
                            StringTable[i + previousStringTableSize] = reader.ReadCString();
                            i += (int)(reader.BaseStream.Position - oldPos);
                        }

                        previousStringTableSize += section.StringTableSize;
                    }
                    else
                    {
                        // sparse data with inlined strings
                        RecordsData = reader.ReadBytes(section.OffsetRecordsEndOffset - section.FileOffset);

                        if (reader.BaseStream.Position != section.OffsetRecordsEndOffset)
                        {
                            throw new Exception("reader.BaseStream.Position != section.OffsetRecordsEndOffset");
                        }
                    }

                    // skip encrypted sections => has tact key + record data is zero filled
                    if (section.TactKeyLookup != 0 && Array.TrueForAll(RecordsData, x => x == 0))
                    {
                        previousRecordCount += section.NumRecords;
                        continue;
                    }

                    // index data
                    IndexData = reader.ReadArray <int>(section.IndexDataSize / 4);

                    // fix zero-filled index data
                    if (IndexData.Length > 0 && IndexData.All(x => x == 0))
                    {
                        IndexData = Enumerable.Range(MinIndex + previousRecordCount, section.NumRecords).ToArray();
                    }

                    // duplicate rows data
                    if (section.CopyTableCount > 0)
                    {
                        if (CopyData == null)
                        {
                            CopyData = new Dictionary <int, int>();
                        }

                        for (int i = 0; i < section.CopyTableCount; i++)
                        {
                            CopyData[reader.ReadInt32()] = reader.ReadInt32();
                        }
                    }

                    if (section.OffsetMapIDCount > 0)
                    {
                        // HACK unittestsparse is malformed and has sparseIndexData first
                        if (TableHash == 145293629)
                        {
                            reader.BaseStream.Position += 4 * section.OffsetMapIDCount;
                        }

                        SparseEntries = reader.ReadArray <SparseEntry>(section.OffsetMapIDCount).ToList();
                    }

                    // reference data
                    ReferenceData refData = new ReferenceData();
                    if (section.ParentLookupDataSize > 0)
                    {
                        refData.NumRecords = reader.ReadInt32();
                        refData.MinId      = reader.ReadInt32();
                        refData.MaxId      = reader.ReadInt32();

                        var entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
                        for (int i = 0; i < entries.Length; i++)
                        {
                            refData.Entries[entries[i].Index] = entries[i].Id;
                        }
                    }

                    if (section.OffsetMapIDCount > 0)
                    {
                        int[] sparseIndexData = reader.ReadArray <int>(section.OffsetMapIDCount);

                        if (section.IndexDataSize > 0 && IndexData.Length != sparseIndexData.Length)
                        {
                            throw new Exception("m_indexData.Length != sparseIndexData.Length");
                        }

                        IndexData = sparseIndexData;
                    }

                    int position = 0;
                    for (int i = 0; i < section.NumRecords; i++)
                    {
                        BitReader bitReader = new BitReader(RecordsData)
                        {
                            Position = 0
                        };

                        if (Flags.HasFlagExt(DB2Flags.Sparse))
                        {
                            bitReader.Position = position;
                            position          += SparseEntries[i].Size * 8;
                        }
                        else
                        {
                            bitReader.Offset = i * RecordSize;
                        }

                        refData.Entries.TryGetValue(i, out int refId);

                        IDBRow rec = new WDC3Row(this, bitReader, section.IndexDataSize != 0 ? IndexData[i] : -1, refId, i + previousRecordCount);
                        _Records.Add(_Records.Count, rec);
                    }

                    previousRecordCount += section.NumRecords;
                }
            }
        }
Exemple #2
0
        public WDC3Reader(Stream stream)
        {
            using (var reader = new BinaryReader(stream, Encoding.UTF8))
            {
                if (reader.BaseStream.Length < HeaderSize)
                {
                    throw new InvalidDataException("WDC3 file is corrupted!");
                }

                uint magic = reader.ReadUInt32();

                if (magic != WDC3FmtSig)
                {
                    throw new InvalidDataException("WDC3 file is corrupted!");
                }

                RecordsCount    = reader.ReadInt32();
                FieldsCount     = reader.ReadInt32();
                RecordSize      = reader.ReadInt32();
                StringTableSize = reader.ReadInt32();
                TableHash       = reader.ReadUInt32();
                LayoutHash      = reader.ReadUInt32();
                MinIndex        = reader.ReadInt32();
                MaxIndex        = reader.ReadInt32();
                Locale          = reader.ReadInt32();
                Flags           = (DB2Flags)reader.ReadUInt16();
                IdFieldIndex    = reader.ReadUInt16();
                int totalFieldsCount = reader.ReadInt32();
                PackedDataOffset  = reader.ReadInt32();     // Offset within the field where packed data starts
                lookupColumnCount = reader.ReadInt32();     // count of lookup columns
                field_info_size   = reader.ReadInt32();     // 24 * NumFields bytes, describes column bit packing, {ushort recordOffset, ushort size, uint additionalDataSize, uint compressionType, uint packedDataOffset or commonvalue, uint cellSize, uint cardinality}[NumFields], sizeof(DBC2CommonValue) == 8
                commonDataSize    = reader.ReadInt32();
                palletDataSize    = reader.ReadInt32();     // in bytes, sizeof(DBC2PalletValue) == 4
                SectionsCount     = reader.ReadInt32();

                if (SectionsCount == 0 || RecordsCount == 0)
                {
                    return;
                }

                //section headers
                SectionHeaders = reader.ReadArray <SectionHeaderWDC3>(SectionsCount).ToList();
                m_sections     = SectionHeaders.OfType <IEncryptableDatabaseSection>().ToList();

                // field meta data
                field_structure_data = reader.ReadArray <FieldMetaData>(FieldsCount);

                // column meta data
                ColumnMeta = reader.ReadArray <ColumnMetaData>(FieldsCount);

                // pallet data
                PalletData = new Value32[ColumnMeta.Length][];
                for (int i = 0; i < ColumnMeta.Length; i++)
                {
                    if (ColumnMeta[i].CompressionType == CompressionType.Pallet || ColumnMeta[i].CompressionType == CompressionType.PalletArray)
                    {
                        PalletData[i] = reader.ReadArray <Value32>((int)ColumnMeta[i].AdditionalDataSize / 4);
                    }
                }

                // common data
                CommonData = new Dictionary <int, Value32> [ColumnMeta.Length];
                for (int i = 0; i < ColumnMeta.Length; i++)
                {
                    if (ColumnMeta[i].CompressionType == CompressionType.Common)
                    {
                        var commonValues = new Dictionary <int, Value32>((int)ColumnMeta[i].AdditionalDataSize / 8);
                        CommonData[i] = commonValues;

                        for (int j = 0; j < ColumnMeta[i].AdditionalDataSize / 8; j++)
                        {
                            commonValues[reader.ReadInt32()] = reader.Read <Value32>();
                        }
                    }
                }

                //section_data
                int StringTableSizeCounter = 0, RecordCountCounter = 0;
                //foreach (var sectionheader in SectionHeaders)
                //{
                reader.BaseStream.Position = SectionHeaders[0].FileOffset;

                if (!Flags.HasFlagExt(DB2Flags.Sparse)) //'no offset map'
                {
                    // records data
                    RecordsData = reader.ReadBytes(SectionHeaders[0].NumRecords * RecordSize);

                    Array.Resize(ref RecordsData, RecordsData.Length + 8); // pad with extra zeros so we don't crash when reading

                    // string data
                    if (StringTable == null)
                    {
                        StringTable = new Dictionary <long, string>(SectionHeaders[0].StringTableSize / 0x20);
                    }

                    for (int i = 0; i < SectionHeaders[0].StringTableSize;)
                    {
                        long oldPos = reader.BaseStream.Position;
                        StringTable[i + StringTableSizeCounter] = reader.ReadCString();
                        i += (int)(reader.BaseStream.Position - oldPos);
                    }

                    StringTableSizeCounter += SectionHeaders[0].StringTableSize;
                }
                else //'Has offset map'
                {
                    // sparse data with inlined strings
                    RecordsData = reader.ReadBytes(SectionHeaders[0].OffsetRecordsEndOffset - SectionHeaders[0].FileOffset);

                    if (reader.BaseStream.Position != SectionHeaders[0].OffsetRecordsEndOffset)
                    {
                        throw new Exception("reader.BaseStream.Position != section.OffsetRecordsEndOffset");
                    }
                }

                // skip encrypted sections => has tact key + record data is zero filled
                if (SectionHeaders[0].TactKeyLookup != 0 && Array.TrueForAll(RecordsData, x => x == 0))
                {
                    RecordCountCounter += SectionHeaders[0].NumRecords;
                    return;
                }

                // id_list_data
                id_list_data = reader.ReadArray <int>(SectionHeaders[0].IndexDataSize / 4);

                // fix zero-filled index data
                if (id_list_data.Length > 0 && id_list_data.All(x => x == 0))
                {
                    id_list_data = Enumerable.Range(MinIndex + RecordCountCounter, SectionHeaders[0].NumRecords).ToArray();
                }

                // copy table entry data    , duplicate rows data
                if (SectionHeaders[0].CopyTableCount > 0)
                {
                    if (CopyData == null)
                    {
                        CopyData = new Dictionary <int, int>();
                    }

                    for (int i = 0; i < SectionHeaders[0].CopyTableCount; i++)
                    {
                        CopyData[reader.ReadInt32()] = reader.ReadInt32(); //id_of_new_row, id_of_copied_row
                    }
                }

                // offset map entry data
                if (SectionHeaders[0].OffsetMapIDCount > 0)
                {
                    // HACK unittestsparse is malformed and has sparseIndexData first
                    if (TableHash == 145293629)
                    {
                        reader.BaseStream.Position += 4 * SectionHeaders[0].OffsetMapIDCount;
                    }

                    offset_map_Entries = reader.ReadArray <offset_map_entry>(SectionHeaders[0].OffsetMapIDCount).ToList();
                }

                // reference/relationship data
                refData = new ReferenceData();
                if (SectionHeaders[0].ParentLookupDataSize > 0)
                {
                    refData.NumRecords = reader.ReadInt32();
                    refData.MinId      = reader.ReadInt32();
                    refData.MaxId      = reader.ReadInt32();

                    var entries = reader.ReadArray <ReferenceEntry>(refData.NumRecords);
                    for (int i = 0; i < entries.Length; i++)
                    {
                        refData.Entries[entries[i].Index] = entries[i].Id;
                    }
                }

                //offset_map_id_list
                if (SectionHeaders[0].OffsetMapIDCount > 0)
                {
                    int[] offset_map_id_listdata = reader.ReadArray <int>(SectionHeaders[0].OffsetMapIDCount);

                    if (SectionHeaders[0].IndexDataSize > 0 && id_list_data.Length != offset_map_id_listdata.Length)
                    {
                        throw new Exception("m_indexData.Length != sparseIndexData.Length");
                    }

                    id_list_data = offset_map_id_listdata;
                }

                //loop and add field data to create datarow
                int position = 0;
                for (int i = 0; i < SectionHeaders[0].NumRecords; i++)
                {
                    BitReader bitReader = new BitReader(RecordsData)
                    {
                        Position = 0
                    };

                    if (Flags.HasFlagExt(DB2Flags.Sparse))
                    {
                        bitReader.Position = position;
                        position          += offset_map_Entries[i].Size * 8;
                    }
                    else
                    {
                        bitReader.Offset = i * RecordSize;
                    }

                    refData.Entries.TryGetValue(i, out int refId);

                    IDBRow rec = new WDC3Row(this, bitReader, SectionHeaders[0].IndexDataSize != 0 ? id_list_data[i] : -1, refId, i + RecordCountCounter);
                    _Records.Add(_Records.Count, rec);
                }

                RecordCountCounter += SectionHeaders[0].NumRecords;
                //}

                long pos = reader.BaseStream.Position;
                long len = reader.BaseStream.Length;
                NoParseRecordsData = reader.ReadBytes((int)(len - pos));
            }
        }